diff --git a/.lintrunner.toml b/.lintrunner.toml index 5cb8e563db9114..8e21dd311597b8 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -25,6 +25,19 @@ args = [ '--', '@{{PATHSFILE}}' ] +init_args = [ + 'python3', + 'tools/linter/adapters/pip_init.py', + '--dry-run={{DRYRUN}}', + 'flake8==3.8.2', + 'flake8-bugbear==20.1.4', + 'flake8-comprehensions==3.3.0', + 'flake8-executable==2.0.4', + 'flake8-pyi==20.5.0', + 'mccabe==0.6.1', + 'pycodestyle==2.6.0', + 'pyflakes==2.2.0', +] [[linter]] @@ -129,3 +142,19 @@ args = [ '--regen-script=generate_config_yml.py', ] bypass_matched_file_filter = true + +[[linter]] +name = 'NATIVEFUNCTIONS' +include_patterns=['aten/src/ATen/native/native_functions.yaml'] +args = [ + 'python3', + 'tools/linter/adapters/nativefunctions_linter.py', + '--native-functions-yml=aten/src/ATen/native/native_functions.yaml', +] +init_args = [ + 'python3', + 'tools/linter/adapters/pip_init.py', + '--dry-run={{DRYRUN}}', + 'ruamel.yaml==0.17.4', +] +bypass_matched_file_filter = true diff --git a/requirements-flake8.txt b/requirements-flake8.txt index 4f521b30c48106..08c432ad4eb3cb 100644 --- a/requirements-flake8.txt +++ b/requirements-flake8.txt @@ -4,6 +4,6 @@ flake8-comprehensions==3.3.0 flake8-executable==2.0.4 git+https://github.com/malfet/flake8-coding.git flake8-pyi==20.5.0 -mccabe +mccabe==0.6.1 pycodestyle==2.6.0 pyflakes==2.2.0 diff --git a/tools/linter/adapters/nativefunctions_linter.py b/tools/linter/adapters/nativefunctions_linter.py new file mode 100644 index 00000000000000..018714499744fc --- /dev/null +++ b/tools/linter/adapters/nativefunctions_linter.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +Verify that it is possible to round-trip native_functions.yaml via ruamel under some +configuration. Keeping native_functions.yaml consistent in this way allows us to +run codemods on the file using ruamel without introducing line noise. Note that we don't +want to normalize the YAML file, as that would to lots of spurious lint failures. Anything +that ruamel understands how to roundtrip, e.g., whitespace and comments, is OK! + +ruamel is a bit picky about inconsistent indentation, so you will have to indent your +file properly. Also, if you are working on changing the syntax of native_functions.yaml, +you may find that you want to use some format that is not what ruamel prefers. If so, +it is OK to modify this script (instead of reformatting native_functions.yaml)--the point +is simply to make sure that there is *some* configuration of ruamel that can round trip +the YAML, not to be prescriptive about it. +""" + +import ruamel.yaml # type: ignore[import] +import argparse +import json +import sys +from io import StringIO +from enum import Enum +from typing import NamedTuple, Optional + + +class LintSeverity(str, Enum): + ERROR = "error" + WARNING = "warning" + ADVICE = "advice" + DISABLED = "disabled" + + +class LintMessage(NamedTuple): + path: Optional[str] + line: Optional[int] + char: Optional[int] + code: str + severity: LintSeverity + name: str + original: Optional[str] + replacement: Optional[str] + description: Optional[str] + bypassChangedLineFiltering: Optional[bool] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="native functions linter", fromfile_prefix_chars="@", + ) + parser.add_argument( + "--native-functions-yml", + required=True, + help="location of native_functions.yaml", + ) + + args = parser.parse_args() + + with open(args.native_functions_yml) as f: + contents = f.read() + + yaml = ruamel.yaml.YAML() # type: ignore[attr-defined] + yaml.preserve_quotes = True # type: ignore[assignment] + yaml.width = 1000 # type: ignore[assignment] + yaml.boolean_representation = ["False", "True"] # type: ignore[attr-defined] + try: + r = yaml.load(contents) + except Exception as err: + msg = LintMessage( + path=None, + line=None, + char=None, + code="NATIVEFUNCTIONS", + severity=LintSeverity.ERROR, + name="YAML load failure", + original=None, + replacement=None, + description=f"Failed due to {err.__class__.__name__}:\n{err}", + bypassChangedLineFiltering=None, + ) + + print(json.dumps(msg._asdict()), flush=True) + sys.exit(0) + + # Cuz ruamel's author intentionally didn't include conversion to string + # https://stackoverflow.com/questions/47614862/best-way-to-use-ruamel-yaml-to-dump-to-string-not-to-stream + string_stream = StringIO() + yaml.dump(r, string_stream) + new_contents = string_stream.getvalue() + string_stream.close() + + if contents != new_contents: + msg = LintMessage( + path=args.native_functions_yml, + line=1, + char=1, + code="NATIVEFUNCTIONS", + severity=LintSeverity.ERROR, + name="roundtrip inconsistency", + original=contents, + replacement=new_contents, + description=( + "YAML roundtrip failed; run `lintrunner --take NATIVEFUNCTIONS -a` to apply the suggested changes. " + "If you think this is in error, please see tools/linter/adapters/nativefunctions_linter.py" + ), + bypassChangedLineFiltering=None, + ) + + print(json.dumps(msg._asdict()), flush=True) diff --git a/tools/linter/adapters/pip_init.py b/tools/linter/adapters/pip_init.py new file mode 100644 index 00000000000000..b4451beac644dc --- /dev/null +++ b/tools/linter/adapters/pip_init.py @@ -0,0 +1,56 @@ +""" +Initializer script that installs stuff to pip. +""" +import argparse +import logging +import subprocess +import sys +import time + +from typing import List + + +def run_command(args: List[str]) -> "subprocess.CompletedProcess[bytes]": + logging.debug("$ %s", " ".join(args)) + start_time = time.monotonic() + try: + return subprocess.run(args, check=True) + finally: + end_time = time.monotonic() + logging.debug("took %dms", (end_time - start_time) * 1000) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="pip initializer") + parser.add_argument( + "packages", nargs="+", help="pip packages to install", + ) + parser.add_argument( + "--verbose", action="store_true", help="verbose logging", + ) + parser.add_argument("--dry-run", help="do not install anything, just print what would be done.") + + args = parser.parse_args() + + logging.basicConfig( + format="<%(threadName)s:%(levelname)s> %(message)s", + level=logging.NOTSET if args.verbose else logging.DEBUG, + stream=sys.stderr, + ) + + for package in args.packages: + package_name, _, version = package.partition("=") + if version == "": + raise RuntimeError( + "Package {package_name} did not have a version specified. " + "Please specify a version to product a consistent linting experience." + ) + pip_args = ["pip3", "install", "--user"] + pip_args.extend(args.packages) + + dry_run = args.dry_run == "1" + if dry_run: + print(f"Would have run: {pip_args}") + sys.exit(0) + + run_command(pip_args)