From cb17e91744ee292b4f7bae65b9f605404a8a3b0c Mon Sep 17 00:00:00 2001 From: Greg Shuflin Date: Fri, 4 Oct 2019 13:01:27 -0700 Subject: [PATCH] Python binary creation --- src/python/pants/backend/python/register.py | 4 + .../python/rules/interpreter_constraints.py | 42 ++++++++ src/python/pants/backend/python/rules/pex.py | 21 ++-- .../python/rules/python_create_binary.py | 96 +++++++++++++++++++ .../python/rules/python_test_runner.py | 15 ++- .../backend/python/targets/python_binary.py | 7 +- src/python/pants/rules/core/binary.py | 68 +++++++++++++ src/python/pants/rules/core/register.py | 11 ++- .../backend/python/rules/test_pex.py | 15 ++- 9 files changed, 254 insertions(+), 25 deletions(-) create mode 100644 src/python/pants/backend/python/rules/interpreter_constraints.py create mode 100644 src/python/pants/backend/python/rules/python_create_binary.py create mode 100644 src/python/pants/rules/core/binary.py diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 72fc71d3734f..2f02bc21f669 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -8,7 +8,9 @@ from pants.backend.python.rules import ( download_pex_bin, inject_init, + interpreter_constraints, pex, + python_create_binary, python_fmt, python_test_runner, ) @@ -100,7 +102,9 @@ def rules(): download_pex_bin.rules() + inject_init.rules() + python_fmt.rules() + + interpreter_constraints.rules() + python_test_runner.rules() + + python_create_binary.rules() + python_native_code_rules() + pex.rules() + subprocess_environment_rules() diff --git a/src/python/pants/backend/python/rules/interpreter_constraints.py b/src/python/pants/backend/python/rules/interpreter_constraints.py new file mode 100644 index 000000000000..bccb5865c1ca --- /dev/null +++ b/src/python/pants/backend/python/rules/interpreter_constraints.py @@ -0,0 +1,42 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass +from typing import FrozenSet, List, Tuple + +from pants.backend.python.subsystems.python_setup import PythonSetup +from pants.engine.legacy.structs import PythonTargetAdaptor +from pants.engine.rules import RootRule, rule + + +@dataclass(frozen=True) +class BuildConstraintsForAdaptors: + adaptors: Tuple[PythonTargetAdaptor] + + +@dataclass(frozen=True) +class PexInterpreterContraints: + constraint_set: FrozenSet[str] = frozenset() + + def generate_pex_arg_list(self) -> List[str]: + args = [] + for constraint in self.constraint_set: + args.extend(["--interpreter-constraint", constraint]) + return args + + +@rule +def handle_constraints(build_constraints_for_adaptors: BuildConstraintsForAdaptors, python_setup: PythonSetup) -> PexInterpreterContraints: + interpreter_constraints = frozenset( + [constraint + for target_adaptor in build_constraints_for_adaptors.adaptors + for constraint in python_setup.compatibility_or_constraints( + getattr(target_adaptor, 'compatibility', None) + )] + ) + + yield PexInterpreterContraints(constraint_set=interpreter_constraints) + + +def rules(): + return [handle_constraints, RootRule(BuildConstraintsForAdaptors)] diff --git a/src/python/pants/backend/python/rules/pex.py b/src/python/pants/backend/python/rules/pex.py index 3d1d25a90b75..e1111a1b3516 100644 --- a/src/python/pants/backend/python/rules/pex.py +++ b/src/python/pants/backend/python/rules/pex.py @@ -6,6 +6,7 @@ from pants.backend.python.rules.download_pex_bin import DownloadedPexBin from pants.backend.python.rules.hermetic_pex import HermeticPex +from pants.backend.python.rules.interpreter_constraints import PexInterpreterContraints from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment @@ -15,10 +16,15 @@ DirectoriesToMerge, DirectoryWithPrefixToAdd, ) -from pants.engine.isolated_process import ExecuteProcessResult, MultiPlatformExecuteProcessRequest +from pants.engine.isolated_process import ( + ExecuteProcessRequest, + ExecuteProcessResult, + MultiPlatformExecuteProcessRequest, +) from pants.engine.platform import Platform, PlatformConstraint -from pants.engine.rules import optionable_rule, rule +from pants.engine.rules import RootRule, optionable_rule, rule from pants.engine.selectors import Get +from pants.util.strutil import create_path_env_var @dataclass(frozen=True) @@ -26,7 +32,7 @@ class CreatePex: """Represents a generic request to create a PEX from its inputs.""" output_filename: str requirements: Tuple[str] = () - interpreter_constraints: Tuple[str] = () + interpreter_constraints: PexInterpreterContraints = PexInterpreterContraints() entry_point: Optional[str] = None input_files_digest: Optional[Digest] = None @@ -35,6 +41,7 @@ class CreatePex: class Pex(HermeticPex): """Wrapper for a digest containing a pex file created with some filename.""" directory_digest: Digest + output_filename: str # TODO: This is non-hermetic because the requirements will be resolved on the fly by @@ -51,9 +58,7 @@ def create_pex( """Returns a PEX with the given requirements, optional entry point, and optional interpreter constraints.""" - interpreter_constraint_args = [] - for constraint in request.interpreter_constraints: - interpreter_constraint_args.extend(["--interpreter-constraint", constraint]) + interpreter_constraint_args = request.interpreter_constraints.generate_pex_arg_list() argv = ["--output-file", request.output_filename] if request.entry_point is not None: @@ -69,7 +74,7 @@ def create_pex( merged_digest = yield Get(Digest, DirectoriesToMerge(directories=all_inputs)) # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without - # cross-building we specify that out PEX command be run on the current local platform. When we + # cross-building, we specify that out PEX command be run on the current local platform. When we # support cross-building through CLI flags we can configure requests that build a PEX for out # local platform that are able to execute on a different platform, but for now in order to # guarantee correct build we need to restrict this command to execute on the same platform type @@ -97,7 +102,7 @@ def create_pex( MultiPlatformExecuteProcessRequest, execute_process_request ) - yield Pex(directory_digest=result.output_directory_digest) + yield Pex(directory_digest=result.output_directory_digest, output_filename=request.output_filename) def rules(): diff --git a/src/python/pants/backend/python/rules/python_create_binary.py b/src/python/pants/backend/python/rules/python_create_binary.py new file mode 100644 index 000000000000..50dfa3988d8e --- /dev/null +++ b/src/python/pants/backend/python/rules/python_create_binary.py @@ -0,0 +1,96 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pants.backend.python.rules.inject_init import InjectedInitDigest +from pants.backend.python.rules.interpreter_constraints import ( + BuildConstraintsForAdaptors, + PexInterpreterContraints, +) +from pants.backend.python.rules.pex import CreatePex, Pex +from pants.backend.python.subsystems.python_setup import PythonSetup +from pants.backend.python.targets.python_binary import PythonBinary +from pants.engine.fs import Digest, DirectoriesToMerge +from pants.engine.isolated_process import ExecuteProcessRequest, FallibleExecuteProcessResult +from pants.engine.legacy.graph import BuildFileAddresses, HydratedTarget, TransitiveHydratedTargets +from pants.engine.legacy.structs import PythonBinaryAdaptor +from pants.engine.rules import UnionRule, rule +from pants.engine.selectors import Get +from pants.rules.core.binary import BinaryTarget, CreatedBinary +from pants.rules.core.strip_source_root import SourceRootStrippedSources + + +@rule +def create_python_binary(python_binary_adaptor: PythonBinaryAdaptor, + python_setup: PythonSetup) -> Pex: + transitive_hydrated_targets = yield Get( + TransitiveHydratedTargets, BuildFileAddresses((python_binary_adaptor.address,)) + ) + all_targets = transitive_hydrated_targets.closure + all_target_adaptors = [t.adaptor for t in all_targets] + + + interpreter_constraints = yield Get(PexInterpreterContraints, + BuildConstraintsForAdaptors(adaptors=tuple(all_target_adaptors))) + + source_root_stripped_sources = yield [ + Get(SourceRootStrippedSources, HydratedTarget, target_adaptor) + for target_adaptor in all_targets + ] + + #TODO This way of calculating the entry point works but is a bit hackish. + entry_point = None + if hasattr(python_binary_adaptor, 'entry_point'): + entry_point = python_binary_adaptor.entry_point + else: + sources_snapshot = python_binary_adaptor.sources.snapshot + if len(sources_snapshot.files) == 1: + target = transitive_hydrated_targets.roots[0] + output = yield Get(SourceRootStrippedSources, HydratedTarget, target) + root_filename = output.snapshot.files[0] + entry_point = PythonBinary.translate_source_path_to_py_module_specifier(root_filename) + + stripped_sources_digests = [stripped_sources.snapshot.directory_digest for stripped_sources in source_root_stripped_sources] + sources_digest = yield Get(Digest, DirectoriesToMerge(directories=tuple(stripped_sources_digests))) + inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest) + all_input_digests = [sources_digest, inits_digest.directory_digest] + merged_input_files = yield Get(Digest, DirectoriesToMerge, DirectoriesToMerge(directories=tuple(all_input_digests))) + + #TODO This chunk of code should be made into an @rule and used both here and in + # python_test_runner.py. + # Produce a pex containing pytest and all transitive 3rdparty requirements. + all_target_requirements = [] + for maybe_python_req_lib in all_target_adaptors: + # This is a python_requirement()-like target. + if hasattr(maybe_python_req_lib, 'requirement'): + all_target_requirements.append(str(maybe_python_req_lib.requirement)) + # This is a python_requirement_library()-like target. + if hasattr(maybe_python_req_lib, 'requirements'): + for py_req in maybe_python_req_lib.requirements: + all_target_requirements.append(str(py_req.requirement)) + + output_filename = f"{python_binary_adaptor.address.target_name}.pex" + + all_requirements = all_target_requirements + create_requirements_pex = CreatePex( + output_filename=output_filename, + requirements=tuple(sorted(all_requirements)), + interpreter_constraints=interpreter_constraints, + entry_point=entry_point, + input_files_digest=merged_input_files, + ) + + pex = yield Get(Pex, CreatePex, create_requirements_pex) + yield pex + + +@rule +def pex_to_created_binary(pex: Pex) -> CreatedBinary: + yield CreatedBinary(_inner=pex) + + +def rules(): + return [ + UnionRule(BinaryTarget, PythonBinaryAdaptor), + pex_to_created_binary, + create_python_binary, + ] diff --git a/src/python/pants/backend/python/rules/python_test_runner.py b/src/python/pants/backend/python/rules/python_test_runner.py index 764ed5158a6d..47dc2c9018d7 100644 --- a/src/python/pants/backend/python/rules/python_test_runner.py +++ b/src/python/pants/backend/python/rules/python_test_runner.py @@ -2,6 +2,10 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.python.rules.inject_init import InjectedInitDigest +from pants.backend.python.rules.interpreter_constraints import ( + BuildConstraintsForAdaptors, + PexInterpreterContraints, +) from pants.backend.python.rules.pex import CreatePex, Pex from pants.backend.python.subsystems.pytest import PyTest from pants.backend.python.subsystems.python_setup import PythonSetup @@ -33,13 +37,8 @@ def run_python_test( ) all_targets = transitive_hydrated_targets.closure - interpreter_constraints = { - constraint - for target_adaptor in all_targets - for constraint in python_setup.compatibility_or_constraints( - getattr(target_adaptor, 'compatibility', None) - ) - } + interpreter_constraints = yield Get(PexInterpreterContraints, + BuildConstraintsForAdaptors(adaptors=tuple(all_targets))) # Produce a pex containing pytest and all transitive 3rdparty requirements. output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex' @@ -58,7 +57,7 @@ def run_python_test( Pex, CreatePex( output_filename=output_pytest_requirements_pex_filename, requirements=tuple(sorted(all_requirements)), - interpreter_constraints=tuple(sorted(interpreter_constraints)), + interpreter_constraints=interpreter_constraints, entry_point="pytest:main", ) ) diff --git a/src/python/pants/backend/python/targets/python_binary.py b/src/python/pants/backend/python/targets/python_binary.py index f19a9a682a95..8baf75859d6c 100644 --- a/src/python/pants/backend/python/targets/python_binary.py +++ b/src/python/pants/backend/python/targets/python_binary.py @@ -116,7 +116,7 @@ def __init__(self, if sources and sources.files and entry_point: entry_point_module = entry_point.split(':', 1)[0] entry_source = list(self.sources_relative_to_source_root())[0] - source_entry_point = self._translate_to_entry_point(entry_source) + source_entry_point = self.translate_source_path_to_py_module_specifier(entry_source) if entry_point_module != source_entry_point: raise TargetDefinitionException(self, 'Specified both source and entry_point but they do not agree: {} vs {}'.format( @@ -136,7 +136,8 @@ def repositories(self): def indices(self): return self.payload.indices - def _translate_to_entry_point(self, source): + @classmethod + def translate_source_path_to_py_module_specifier(self, source: str) -> str: source_base, _ = os.path.splitext(source) return source_base.replace(os.path.sep, '.') @@ -147,7 +148,7 @@ def entry_point(self): elif self.payload.sources.source_paths: assert len(self.payload.sources.source_paths) == 1 entry_source = list(self.sources_relative_to_source_root())[0] - return self._translate_to_entry_point(entry_source) + return self.translate_source_path_to_py_module_specifier(entry_source) else: return None diff --git a/src/python/pants/rules/core/binary.py b/src/python/pants/rules/core/binary.py new file mode 100644 index 000000000000..d88605d6e22a --- /dev/null +++ b/src/python/pants/rules/core/binary.py @@ -0,0 +1,68 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass +from typing import Any + +from pants.backend.python.rules.pex import Pex +from pants.build_graph.address import Address +from pants.engine.addressable import BuildFileAddresses +from pants.engine.console import Console +from pants.engine.fs import DirectoryToMaterialize, Workspace +from pants.engine.goal import Goal, LineOriented +from pants.engine.legacy.graph import HydratedTarget +from pants.engine.legacy.structs import PythonBinaryAdaptor +from pants.engine.rules import console_rule, optionable_rule, rule, union +from pants.engine.selectors import Get + + +@dataclass(frozen=True) +class Binary(LineOriented, Goal): + name = 'binary' + + +@union +@dataclass(frozen=True) +class BinaryTarget: + pass + + +@union +@dataclass(frozen=True) +class CreatedBinary: + _inner: Any + + def write_to_dist(self, print_stdout: Any, workspace: Workspace) -> None: + if isinstance(self._inner, Pex): + dtm = DirectoryToMaterialize( + path = 'dist/', + directory_digest = self._inner.directory_digest, + ) + output = workspace.materialize_directories((dtm,)) + for path in output.dependencies[0].output_paths: + print_stdout(f"Wrote {path}") + else: + raise ValueError(f"CreatedBinary should not contain a value of type: {type(self._inner)}") + + +@console_rule +def create_binary(addresses: BuildFileAddresses, console: Console, workspace: Workspace, options: Binary.Options) -> Binary: + with Binary.line_oriented(options, console) as (print_stdout, print_stderr): + print_stdout("Generating binaries in `dist/`") + binaries = yield [Get(CreatedBinary, Address, address.to_address()) for address in addresses] + for binary in binaries: + binary.write_to_dist(print_stdout, workspace) + yield Binary(exit_code=0) + + +@rule +def coordinator_of_binaries(target: HydratedTarget) -> CreatedBinary: + binary = yield Get(CreatedBinary, BinaryTarget, target.adaptor) + yield binary + + +def rules(): + return [ + create_binary, + coordinator_of_binaries, + ] diff --git a/src/python/pants/rules/core/register.py b/src/python/pants/rules/core/register.py index f5df079fa764..20d89fdd387a 100644 --- a/src/python/pants/rules/core/register.py +++ b/src/python/pants/rules/core/register.py @@ -1,11 +1,20 @@ # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.rules.core import filedeps, fmt, list_roots, list_targets, strip_source_root, test +from pants.rules.core import ( + binary, + filedeps, + fmt, + list_roots, + list_targets, + strip_source_root, + test, +) def rules(): return [ + *binary.rules(), *fmt.rules(), *list_roots.rules(), *list_targets.rules(), diff --git a/tests/python/pants_test/backend/python/rules/test_pex.py b/tests/python/pants_test/backend/python/rules/test_pex.py index 914967513e0e..e6c2a72c54f3 100644 --- a/tests/python/pants_test/backend/python/rules/test_pex.py +++ b/tests/python/pants_test/backend/python/rules/test_pex.py @@ -7,6 +7,7 @@ from typing import Dict, List from pants.backend.python.rules.download_pex_bin import download_pex_bin +from pants.backend.python.rules.interpreter_constraints import PexInterpreterContraints from pants.backend.python.rules.pex import CreatePex, Pex, create_pex from pants.backend.python.subsystems.python_native_code import ( PythonNativeCode, @@ -47,7 +48,9 @@ def setUp(self): super().setUp() init_subsystems([PythonSetup, PythonNativeCode, SubprocessEnvironment]) - def create_pex_and_get_all_data(self, *, requirements=None, entry_point=None, interpreter_constraints=None, + def create_pex_and_get_all_data(self, *, requirements=None, + entry_point=None, + interpreter_constraints=PexInterpreterContraints(), input_files: Digest = None) -> (Dict, List[str]): def hashify_optional_collection(iterable): return tuple(sorted(iterable)) if iterable is not None else tuple() @@ -55,7 +58,7 @@ def hashify_optional_collection(iterable): request = CreatePex( output_filename="test.pex", requirements=hashify_optional_collection(requirements), - interpreter_constraints=hashify_optional_collection(interpreter_constraints), + interpreter_constraints=interpreter_constraints, entry_point=entry_point, input_files_digest=input_files, ) @@ -78,7 +81,8 @@ def hashify_optional_collection(iterable): return {'pex': requirements_pex, 'info': json.loads(pex_info_content), 'files': pex_list} def create_pex_and_get_pex_info( - self, *, requirements=None, entry_point=None, interpreter_constraints=None, + self, *, requirements=None, entry_point=None, + interpreter_constraints=PexInterpreterContraints(), input_files: Digest = None) -> Dict: return self.create_pex_and_get_all_data(requirements=requirements, entry_point=entry_point, interpreter_constraints=interpreter_constraints, input_files=input_files)['info'] @@ -119,6 +123,7 @@ def test_entry_point(self) -> None: self.assertEqual(pex_info["entry_point"], entry_point) def test_interpreter_constraints(self) -> None: - constraints = {"CPython>=2.7,<3", "CPython>=3.6,<4"} + constraints = PexInterpreterContraints( + constraint_set=frozenset(sorted({"CPython>=2.7,<3", "CPython>=3.6,<4"}))) pex_info = self.create_pex_and_get_pex_info(interpreter_constraints=constraints) - self.assertEqual(set(pex_info["interpreter_constraints"]), constraints) + self.assertEqual(frozenset(pex_info["interpreter_constraints"]), constraints.constraint_set)