Add black support as a v2 rule

BUILD

@@ -36,6 +36,11 @@ files(
   source = 'pants.ini',
 )
 
+files(
+  name = 'pyproject',
+  source = 'pyproject.toml',
+)
+
 # NB: This is used for integration tests. This is generated automatically via `./pants` and
 # `build-support/bin/bootstrap_pants_pex.sh`.
 files(

examples/src/python/example/hello/greet/greet.py

@@ -5,5 +5,5 @@
 
 
 def greet(greetee):
-  """Given the name, return a greeting for a person of that name."""
-  return green('Hello, {}!'.format(greetee))
+    """Given the name, return a greeting for a person of that name."""
+    return green("Hello, {}!".format(greetee))

examples/src/python/example/hello/main/main.py

@@ -6,7 +6,7 @@
 from example.hello.greet.greet import greet
 
 
-if __name__ == '__main__':
-  greetees = sys.argv[1:] or ['world']
-  for greetee in greetees:
-    print(greet(greetee))
+if __name__ == "__main__":
+    greetees = sys.argv[1:] or ["world"]
+    for greetee in greetees:
+        print(greet(greetee))

pants.ini

@@ -82,6 +82,8 @@ pants_ignore: +[
     '/build-support/bin/native/src',
   ]
 
+[black]
+config: pyproject.toml
 
 [cache]
 # Caching is on globally by default, but we disable it here for development purposes.
@@ -258,6 +260,9 @@ skip: True
 # --closure option is removed. See comment in the task code for details.
 transitive: True
 
+[lint.pythonstyle]
+skip: True
+
 [lint.scalafmt]
 skip: True
 

pyproject.toml

@@ -0,0 +1,15 @@
+[tool.black]
+line-length = 100
+exclude = '''
+/(
+  # These would already be ignored by pants, but having them here allows for manually running Black if one so whishes.
+  | \.git
+  | \.mypy_cache
+  | dist
+  | \.pants\.d
+  | virtualenvs
+  # This file intentionally contains invalid syntax
+  # It trips black up.
+  | compilation_failure
+)/
+'''

src/python/pants/backend/python/register.py

@@ -5,7 +5,13 @@
 from pants.backend.python.python_artifact import PythonArtifact
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.python_requirements import PythonRequirements
-from pants.backend.python.rules import download_pex_bin, inject_init, pex, python_test_runner
+from pants.backend.python.rules import (
+  download_pex_bin,
+  inject_init,
+  pex,
+  python_fmt,
+  python_test_runner,
+)
 from pants.backend.python.subsystems.python_native_code import PythonNativeCode
 from pants.backend.python.subsystems.python_native_code import rules as python_native_code_rules
 from pants.backend.python.subsystems.subprocess_environment import SubprocessEnvironment
@@ -93,6 +99,7 @@ def rules():
   return (
     download_pex_bin.rules() +
     inject_init.rules() +
+    python_fmt.rules() +
     python_test_runner.rules() +
     python_native_code_rules() +
     pex.rules() +

src/python/pants/backend/python/rules/python_fmt.py

@@ -0,0 +1,141 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Set
+
+from pants.backend.python.rules.pex import CreatePex, Pex
+from pants.backend.python.subsystems.black import Black
+from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
+from pants.engine.fs import Digest, DirectoriesToMerge, PathGlobs, Snapshot
+from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult
+from pants.engine.legacy.structs import (
+  PantsPluginAdaptor,
+  PythonAppAdaptor,
+  PythonBinaryAdaptor,
+  PythonTargetAdaptor,
+  PythonTestsAdaptor,
+)
+from pants.engine.rules import UnionRule, optionable_rule, rule
+from pants.engine.selectors import Get
+from pants.rules.core.fmt import FmtResult, FmtTarget
+
+
+# Note: this is a workaround until https://github.com/pantsbuild/pants/issues/8343 is addressed
+# We have to write this type which basically represents a union of all various kinds of targets
+# containing python files so we can have one single type used as an input in the run_black rule.
+@dataclass(frozen=True)
+class FormattablePythonTarget:
+    target: Any
+
+
+@rule
+def run_black(
+  wrapped_target: FormattablePythonTarget,
+  black: Black,
+  python_setup: PythonSetup,
+  subprocess_encoding_environment: SubprocessEncodingEnvironment,
+  ) -> FmtResult:
+  config_path = black.get_options().config
+  config_snapshot = yield Get(Snapshot, PathGlobs(include=(config_path,)))
+
+  resolved_requirements_pex = yield Get(
+    Pex, CreatePex(
+      output_filename="black.pex",
+      requirements=tuple(black.get_requirement_specs()),
+      interpreter_constraints=tuple(black.default_interpreter_constraints),
+      entry_point=black.get_entry_point(),
+    )
+  )
+  target = wrapped_target.target
+  sources_digest = target.sources.snapshot.directory_digest
+
+  all_input_digests = [
+    sources_digest,
+    resolved_requirements_pex.directory_digest,
+    config_snapshot.directory_digest,
+  ]
+  merged_input_files = yield Get(
+    Digest,
+    DirectoriesToMerge(directories=tuple(all_input_digests)),
+  )
+
+  # The exclude option from Black only works on recursive invocations,
+  # so call black with the directories in which the files are present
+  # and passing the full file names with the include option
+  dirs: Set[str] = set()
+  for filename in target.sources.snapshot.files:
+    dirs.add(f"{Path(filename).parent}")
+  pex_args= tuple(sorted(dirs))
+  if config_path:
+    pex_args += ("--config", config_path)
+  if target.sources.snapshot.files:
+    pex_args += ("--include", "|".join(re.escape(f) for f in target.sources.snapshot.files))
+
+  request = resolved_requirements_pex.create_execute_request(
+    python_setup=python_setup,
+    subprocess_encoding_environment=subprocess_encoding_environment,
+    pex_path="./black.pex",
+    pex_args=pex_args,
+    input_files=merged_input_files,
+    output_files=target.sources.snapshot.files,
+    description=f'Run Black for {target.address.reference()}',
+  )
+
+  result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, request)
+
+  yield FmtResult(
+    digest=result.output_directory_digest,
+    stdout=result.stdout.decode(),
+    stderr=result.stderr.decode(),
+  )
+
+# TODO: remove this workaround once https://github.com/pantsbuild/pants/issues/8343 is addressed
+@rule
+def target_adaptor(target: PythonTargetAdaptor) -> FormattablePythonTarget:
+  yield FormattablePythonTarget(target)
+
+
+# TODO: remove this workaround once https://github.com/pantsbuild/pants/issues/8343 is addressed
+@rule
+def app_adaptor(target: PythonAppAdaptor) -> FormattablePythonTarget:
+  yield FormattablePythonTarget(target)
+
+
+# TODO: remove this workaround once https://github.com/pantsbuild/pants/issues/8343 is addressed
+@rule
+def binary_adaptor(target: PythonBinaryAdaptor) -> FormattablePythonTarget:
+  yield FormattablePythonTarget(target)
+
+
+# TODO: remove this workaround once https://github.com/pantsbuild/pants/issues/8343 is addressed
+@rule
+def tests_adaptor(target: PythonTestsAdaptor) -> FormattablePythonTarget:
+  yield FormattablePythonTarget(target)
+
+
+# TODO: remove this workaround once https://github.com/pantsbuild/pants/issues/8343 is addressed
+@rule
+def plugin_adaptor(target: PantsPluginAdaptor) -> FormattablePythonTarget:
+  yield FormattablePythonTarget(target)
+
+
+def rules():
+  return [
+    target_adaptor,
+    app_adaptor,
+    binary_adaptor,
+    tests_adaptor,
+    plugin_adaptor,
+    run_black,
+    UnionRule(FmtTarget, PythonTargetAdaptor),
+    UnionRule(FmtTarget, PythonAppAdaptor),
+    UnionRule(FmtTarget, PythonBinaryAdaptor),
+    UnionRule(FmtTarget, PythonTestsAdaptor),
+    UnionRule(FmtTarget, PantsPluginAdaptor),
+    optionable_rule(Black),
+    optionable_rule(PythonSetup),
+  ]

src/python/pants/backend/python/subsystems/black.py

@@ -0,0 +1,18 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.option.custom_types import file_option
+
+
+class Black(PythonToolBase):
+  options_scope = 'black'
+  default_requirements = ['black==19.3b0', 'setuptools']
+  default_entry_point = 'black:patched_main'
+  default_interpreter_constraints = ["CPython>=3.6"]
+
+  @classmethod
+  def register_options(cls, register):
+    super().register_options(register)
+    register('--config', advanced=True, type=file_option, fingerprint=True,
+              help="Path to formatting tool's config file")

src/python/pants/bin/BUILD

@@ -77,6 +77,7 @@ python_binary(
     '//:build_tools',
     '//:pants_ini',
     '//:3rdparty_directory',
+    '//:pyproject',
     'build-support/checkstyle',
     'build-support/eslint',
     'build-support/ivy',

src/python/pants/rules/core/fmt.py

@@ -0,0 +1,79 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from pants.base.build_environment import get_buildroot
+from pants.engine.console import Console
+from pants.engine.fs import Digest, FilesContent
+from pants.engine.goal import Goal
+from pants.engine.legacy.graph import HydratedTargets
+from pants.engine.legacy.structs import (
+  PythonAppAdaptor,
+  PythonBinaryAdaptor,
+  PythonTargetAdaptor,
+  PythonTestsAdaptor,
+)
+from pants.engine.rules import console_rule, union
+from pants.engine.selectors import Get
+from pants.util.objects import datatype
+
+
+@dataclass(frozen=True)
+class FmtResult:
+  digest: Digest
+  stdout: str
+  stderr: str
+
+
+@union
+class FmtTarget:
+  """A union for registration of a formattable target type."""
+
+
+class Fmt(Goal):
+  """Autoformat source code."""
+
+  # TODO: make this "fmt"
+  # Blocked on https://github.com/pantsbuild/pants/issues/8351
+  name = 'fmt-v2'
+
+
+@console_rule
+def fmt(console: Console, targets: HydratedTargets) -> Fmt:
+  results = yield [
+          Get(FmtResult, FmtTarget, target.adaptor)
+          for target in targets
+          # @union assumes that all targets passed implement the union, so we manually
+          # filter the targets we know do; this should probably no-op or log or something
+          # configurable for non-matching targets.
+          # We also would want to remove the workaround that filters adaptors which have a
+          # `sources` attribute.
+          # See https://github.com/pantsbuild/pants/issues/4535
+          if isinstance(target.adaptor, (PythonAppAdaptor, PythonTargetAdaptor, PythonTestsAdaptor, PythonBinaryAdaptor)) and hasattr(target.adaptor, "sources")
+          ]
+
+  for result in results:
+    files_content = yield Get(FilesContent, Digest, result.digest)
+    # TODO: This is hacky and inefficient, and should be replaced by using the Workspace type
+    # once that is available on master.
+    # Blocked on: https://github.com/pantsbuild/pants/pull/8329
+    for file_content in files_content:
+      with Path(get_buildroot(), file_content.path).open('wb') as f:
+        f.write(file_content.content)
+
+    if result.stdout:
+      console.print_stdout(result.stdout)
+    if result.stderr:
+      console.print_stderr(result.stderr)
+
+  # Since we ran an ExecuteRequest, any failure would already have interrupted our flow
+  exit_code = 0
+  yield Fmt(exit_code)
+
+
+def rules():
+  return [
+      fmt,
+    ]

src/python/pants/rules/core/register.py

@@ -1,11 +1,12 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from pants.rules.core import filedeps, list_roots, list_targets, strip_source_root, test
+from pants.rules.core import filedeps, fmt, list_roots, list_targets, strip_source_root, test
 
 
 def rules():
   return [
+    *fmt.rules(),
     *list_roots.rules(),
     *list_targets.rules(),
     *filedeps.rules(),

tests/python/pants_test/BUILD

@@ -57,6 +57,7 @@ target(
     # on changes to those undeclared dependencies.
     'src/python/pants/bin:pants_local_binary',
     'src/rust/engine',
+    '//:pyproject',
   ],
 )
 

tests/python/pants_test/backend/jvm/tasks/test_checkstyle_integration.py

@@ -95,6 +95,7 @@ def _temporary_buildroot(self, files_to_copy, current_root=None):
       'contrib',
       'pants-plugins',
       'src',
+      'pyproject.toml',
     ))
     with temporary_dir() as temp_root:
       temp_root = os.path.normpath(temp_root)

tests/python/pants_test/backend/python/rules/BUILD

@@ -40,3 +40,14 @@ python_tests(
   timeout = 90,
 )
 
+python_tests(
+  name='python_fmt_integration',
+  source='test_python_fmt_integration.py',
+  dependencies=[
+    'tests/python/pants_test:int-test',
+    'testprojects/src/python:unicode_directory',
+    'examples/src/python/example:hello_directory',
+  ],
+  tags={'integration'},
+)
+