From 46cd41c57be7c9bce871ab80b15166d3da15c036 Mon Sep 17 00:00:00 2001 From: clee2000 <44682903+clee2000@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:13:52 -0800 Subject: [PATCH] [CI] Enable TABS formatter/linter (#6160) Copy over grep_linter from https://togithub.com/pytorch/pytorch/pull/144589. Enable TABS linter in CI Add paths to ignore list that don't currently work Run formatter on clickhouse_queries folder --- .github/workflows/lint.yml | 2 +- .lintrunner.toml | 3 ++ tools/linter/adapters/grep_linter.py | 44 +++++++++---------- .../test_time_per_class/query.sql | 4 +- .../query.sql | 4 +- .../test_time_per_file/query.sql | 4 +- .../query.sql | 4 +- .../workflow_duration_percentile/query.sql | 32 +++++++------- 8 files changed, 50 insertions(+), 47 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f00e197208..e9d0cadb7f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,7 +37,7 @@ jobs: - name: Run lintrunner on all files - Linux run: | set +e - if ! lintrunner -v --force-color --all-files --tee-json=lint.json --take ACTIONLINT,MYPY,RUSTFMT,COPYRIGHT,LINTRUNNER_VERSION,UFMT,NEWLINE; then + if ! lintrunner -v --force-color --all-files --tee-json=lint.json --take ACTIONLINT,MYPY,RUSTFMT,COPYRIGHT,LINTRUNNER_VERSION,UFMT,NEWLINE,TABS; then echo "" echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner -m main\`.\e[0m" exit 1 diff --git a/.lintrunner.toml b/.lintrunner.toml index db94d33516..f66e9e7526 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -158,6 +158,9 @@ exclude_patterns = [ '.lintrunner.toml', '**/fixtures/**', '**/snapshots/**', + '.github/actions/setup-ssh/index.js', + 'terraform-aws-github-runner/**', + 'aws/ami/windows/**', ] command = [ 'python3', diff --git a/tools/linter/adapters/grep_linter.py b/tools/linter/adapters/grep_linter.py index 21c8a210b2..003ff2b49f 100644 --- a/tools/linter/adapters/grep_linter.py +++ b/tools/linter/adapters/grep_linter.py @@ -2,6 +2,8 @@ Generic linter that greps for a pattern and optionally suggests replacements. """ +from __future__ import annotations + import argparse import json import logging @@ -10,7 +12,7 @@ import sys import time from enum import Enum -from typing import Any, List, NamedTuple, Optional +from typing import Any, NamedTuple IS_WINDOWS: bool = os.name == "nt" @@ -28,15 +30,15 @@ class LintSeverity(str, Enum): class LintMessage(NamedTuple): - path: Optional[str] - line: Optional[int] - char: Optional[int] + path: str | None + line: int | None + char: int | None code: str severity: LintSeverity name: str - original: Optional[str] - replacement: Optional[str] - description: Optional[str] + original: str | None + replacement: str | None + description: str | None def as_posix(name: str) -> str: @@ -44,8 +46,8 @@ def as_posix(name: str) -> str: def run_command( - args: List[str], -) -> "subprocess.CompletedProcess[bytes]": + args: list[str], +) -> subprocess.CompletedProcess[bytes]: logging.debug("$ %s", " ".join(args)) start_time = time.monotonic() try: @@ -59,18 +61,13 @@ def run_command( def lint_file( - matching_line: str, + filename: str, allowlist_pattern: str, replace_pattern: str, linter_name: str, error_name: str, error_description: str, -) -> Optional[LintMessage]: - # matching_line looks like: - # tools/linter/clangtidy_linter.py:13:import foo.bar.baz - split = matching_line.split(":") - filename = split[0] - +) -> LintMessage | None: if allowlist_pattern: try: proc = run_command(["grep", "-nEHI", allowlist_pattern, filename]) @@ -108,7 +105,7 @@ def lint_file( original = None replacement = None if replace_pattern: - with open(filename, "r") as f: + with open(filename) as f: original = f.read() try: @@ -142,8 +139,8 @@ def lint_file( ) return LintMessage( - path=split[0], - line=int(split[1]) if len(split) > 1 else None, + path=filename, + line=None, char=None, code=linter_name, severity=LintSeverity.ERROR, @@ -252,12 +249,15 @@ def main() -> None: ), ) print(json.dumps(err_msg._asdict()), flush=True) - exit(0) + sys.exit(0) lines = proc.stdout.decode().splitlines() - for line in lines: + # matching_line looks like: + # tools/linter/clangtidy_linter.py:13:import foo.bar.baz + files = {line.split(":")[0] for line in lines} + for file in files: lint_message = lint_file( - line, + file, args.allowlist_pattern, args.replace_pattern, args.linter_name, diff --git a/torchci/clickhouse_queries/test_time_per_class/query.sql b/torchci/clickhouse_queries/test_time_per_class/query.sql index aab54f585e..bc19426920 100644 --- a/torchci/clickhouse_queries/test_time_per_class/query.sql +++ b/torchci/clickhouse_queries/test_time_per_class/query.sql @@ -49,8 +49,8 @@ class_duration_per_job AS ( test_run.invoking_file, test_run.classname, base_name, - test_config, - job.run_id + test_config, + job.run_id ) SELECT REPLACE(file, '.', '/') AS file, diff --git a/torchci/clickhouse_queries/test_time_per_class_periodic_jobs/query.sql b/torchci/clickhouse_queries/test_time_per_class_periodic_jobs/query.sql index 559b35191d..1333b86404 100644 --- a/torchci/clickhouse_queries/test_time_per_class_periodic_jobs/query.sql +++ b/torchci/clickhouse_queries/test_time_per_class_periodic_jobs/query.sql @@ -66,8 +66,8 @@ class_duration_per_job AS ( test_run.invoking_file, test_run.classname, base_name, - test_config, - job.run_id + test_config, + job.run_id ) SELECT REPLACE(file, '.', '/') AS file, diff --git a/torchci/clickhouse_queries/test_time_per_file/query.sql b/torchci/clickhouse_queries/test_time_per_file/query.sql index 2ced4a60c9..37c7ae2a7d 100644 --- a/torchci/clickhouse_queries/test_time_per_file/query.sql +++ b/torchci/clickhouse_queries/test_time_per_file/query.sql @@ -47,8 +47,8 @@ file_duration_per_job AS ( GROUP BY test_run.invoking_file, base_name, - test_config, - job.run_id + test_config, + job.run_id ) SELECT REPLACE(file, '.', '/') AS file, diff --git a/torchci/clickhouse_queries/test_time_per_file_periodic_jobs/query.sql b/torchci/clickhouse_queries/test_time_per_file_periodic_jobs/query.sql index 33de90fdb6..19f36b71d2 100644 --- a/torchci/clickhouse_queries/test_time_per_file_periodic_jobs/query.sql +++ b/torchci/clickhouse_queries/test_time_per_file_periodic_jobs/query.sql @@ -64,8 +64,8 @@ file_duration_per_job AS ( GROUP BY test_run.invoking_file, base_name, - test_config, - job.run_id + test_config, + job.run_id ) SELECT REPLACE(file, '.', '/') AS file, diff --git a/torchci/clickhouse_queries/workflow_duration_percentile/query.sql b/torchci/clickhouse_queries/workflow_duration_percentile/query.sql index fcce2e51ba..21316b3e4c 100644 --- a/torchci/clickhouse_queries/workflow_duration_percentile/query.sql +++ b/torchci/clickhouse_queries/workflow_duration_percentile/query.sql @@ -2,24 +2,24 @@ -- workflow names grouped by sha and then take percentile, but this matches the -- Rockset query most closely with tts as ( - SELECT - DATE_DIFF( - 'second', - workflow.created_at, - workflow.updated_at - ) as duration_sec, - name, - FROM - default.workflow_run workflow final - WHERE - conclusion = 'success' - AND lower(workflow.name) in {workflowNames: Array(String)} - AND workflow.created_at >= {startTime: DateTime64(3)} - AND workflow.created_at < {stopTime: DateTime64(3)} - AND workflow.run_attempt = 1 + SELECT + DATE_DIFF( + 'second', + workflow.created_at, + workflow.updated_at + ) as duration_sec, + name, + FROM + default.workflow_run workflow final + WHERE + conclusion = 'success' + AND lower(workflow.name) in {workflowNames: Array(String)} + AND workflow.created_at >= {startTime: DateTime64(3)} + AND workflow.created_at < {stopTime: DateTime64(3)} + AND workflow.run_attempt = 1 ), tts_by_name as ( SELECT - quantileExact({percentile: Float32})(tts.duration_sec) as duration_sec + quantileExact({percentile: Float32})(tts.duration_sec) as duration_sec FROM tts group by name )