Python binary creation

src/python/pants/backend/python/register.py

@@ -8,7 +8,9 @@
 from pants.backend.python.rules import (
   download_pex_bin,
   inject_init,
+  interpreter_constraints,
   pex,
+  python_create_binary,
   python_fmt,
   python_test_runner,
 )
@@ -100,7 +102,9 @@ def rules():
     download_pex_bin.rules() +
     inject_init.rules() +
     python_fmt.rules() +
+    interpreter_constraints.rules() +
     python_test_runner.rules() +
+    python_create_binary.rules() +
     python_native_code_rules() +
     pex.rules() +
     subprocess_environment_rules()

src/python/pants/backend/python/rules/interpreter_constraints.py

@@ -0,0 +1,42 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from dataclasses import dataclass
+from typing import FrozenSet, List, Tuple
+
+from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.engine.legacy.structs import PythonTargetAdaptor
+from pants.engine.rules import RootRule, rule
+
+
+@dataclass(frozen=True)
+class BuildConstraintsForAdaptors:
+  adaptors: Tuple[PythonTargetAdaptor]
+
+
+@dataclass(frozen=True)
+class PexInterpreterContraints:
+  constraint_set: FrozenSet[str] = frozenset()
+
+  def generate_pex_arg_list(self) -> List[str]:
+    args = []
+    for constraint in self.constraint_set:
+      args.extend(["--interpreter-constraint", constraint])
+    return args
+
+
+@rule
+def handle_constraints(build_constraints_for_adaptors: BuildConstraintsForAdaptors, python_setup: PythonSetup) -> PexInterpreterContraints:
+  interpreter_constraints = frozenset(
+    [constraint
+    for target_adaptor in build_constraints_for_adaptors.adaptors
+    for constraint in python_setup.compatibility_or_constraints(
+      getattr(target_adaptor, 'compatibility', None)
+    )]
+  )
+
+  yield PexInterpreterContraints(constraint_set=interpreter_constraints)
+
+
+def rules():
+  return [handle_constraints, RootRule(BuildConstraintsForAdaptors)]

src/python/pants/backend/python/rules/pex.py

@@ -6,6 +6,7 @@
 
 from pants.backend.python.rules.download_pex_bin import DownloadedPexBin
 from pants.backend.python.rules.hermetic_pex import HermeticPex
+from pants.backend.python.rules.interpreter_constraints import PexInterpreterContraints
 from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
@@ -15,18 +16,23 @@
   DirectoriesToMerge,
   DirectoryWithPrefixToAdd,
 )
-from pants.engine.isolated_process import ExecuteProcessResult, MultiPlatformExecuteProcessRequest
+from pants.engine.isolated_process import (
+  ExecuteProcessRequest,
+  ExecuteProcessResult,
+  MultiPlatformExecuteProcessRequest,
+)
 from pants.engine.platform import Platform, PlatformConstraint
-from pants.engine.rules import optionable_rule, rule
+from pants.engine.rules import RootRule, optionable_rule, rule
 from pants.engine.selectors import Get
+from pants.util.strutil import create_path_env_var
 
 
 @dataclass(frozen=True)
 class CreatePex:
   """Represents a generic request to create a PEX from its inputs."""
   output_filename: str
   requirements: Tuple[str] = ()
-  interpreter_constraints: Tuple[str] = ()
+  interpreter_constraints: PexInterpreterContraints = PexInterpreterContraints()
   entry_point: Optional[str] = None
   input_files_digest: Optional[Digest] = None
 
@@ -35,6 +41,7 @@ class CreatePex:
 class Pex(HermeticPex):
   """Wrapper for a digest containing a pex file created with some filename."""
   directory_digest: Digest
+  output_filename: str
 
 
 # TODO: This is non-hermetic because the requirements will be resolved on the fly by
@@ -51,9 +58,7 @@ def create_pex(
   """Returns a PEX with the given requirements, optional entry point, and optional
   interpreter constraints."""
 
-  interpreter_constraint_args = []
-  for constraint in request.interpreter_constraints:
-    interpreter_constraint_args.extend(["--interpreter-constraint", constraint])
+  interpreter_constraint_args = request.interpreter_constraints.generate_pex_arg_list()
 
   argv = ["--output-file", request.output_filename]
   if request.entry_point is not None:
@@ -69,7 +74,7 @@ def create_pex(
   merged_digest = yield Get(Digest, DirectoriesToMerge(directories=all_inputs))
 
   # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without
-  # cross-building we specify that out PEX command be run on the current local platform. When we
+  # cross-building, we specify that out PEX command be run on the current local platform. When we
   # support cross-building through CLI flags we can configure requests that build a PEX for out
   # local platform that are able to execute on a different platform, but for now in order to
   # guarantee correct build we need to restrict this command to execute on the same platform type
@@ -97,7 +102,7 @@ def create_pex(
     MultiPlatformExecuteProcessRequest,
     execute_process_request
   )
-  yield Pex(directory_digest=result.output_directory_digest)
+  yield Pex(directory_digest=result.output_directory_digest, output_filename=request.output_filename)
 
 
 def rules():

src/python/pants/backend/python/rules/python_create_binary.py

@@ -0,0 +1,96 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.python.rules.inject_init import InjectedInitDigest
+from pants.backend.python.rules.interpreter_constraints import (
+  BuildConstraintsForAdaptors,
+  PexInterpreterContraints,
+)
+from pants.backend.python.rules.pex import CreatePex, Pex
+from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.targets.python_binary import PythonBinary
+from pants.engine.fs import Digest, DirectoriesToMerge
+from pants.engine.isolated_process import ExecuteProcessRequest, FallibleExecuteProcessResult
+from pants.engine.legacy.graph import BuildFileAddresses, HydratedTarget, TransitiveHydratedTargets
+from pants.engine.legacy.structs import PythonBinaryAdaptor
+from pants.engine.rules import UnionRule, rule
+from pants.engine.selectors import Get
+from pants.rules.core.binary import BinaryTarget, CreatedBinary
+from pants.rules.core.strip_source_root import SourceRootStrippedSources
+
+
+@rule
+def create_python_binary(python_binary_adaptor: PythonBinaryAdaptor,
+  python_setup: PythonSetup) -> Pex:
+  transitive_hydrated_targets = yield Get(
+    TransitiveHydratedTargets, BuildFileAddresses((python_binary_adaptor.address,))
+  )
+  all_targets = transitive_hydrated_targets.closure
+  all_target_adaptors = [t.adaptor for t in all_targets]
+
+
+  interpreter_constraints = yield Get(PexInterpreterContraints,
+      BuildConstraintsForAdaptors(adaptors=tuple(all_target_adaptors)))
+
+  source_root_stripped_sources = yield [
+    Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
+    for target_adaptor in all_targets
+  ]
+
+  #TODO This way of calculating the entry point works but is a bit hackish.
+  entry_point = None
+  if hasattr(python_binary_adaptor, 'entry_point'):
+    entry_point = python_binary_adaptor.entry_point
+  else:
+    sources_snapshot = python_binary_adaptor.sources.snapshot
+    if len(sources_snapshot.files) == 1:
+      target = transitive_hydrated_targets.roots[0]
+      output = yield Get(SourceRootStrippedSources, HydratedTarget, target)
+      root_filename = output.snapshot.files[0]
+      entry_point = PythonBinary.translate_source_path_to_py_module_specifier(root_filename)
+
+  stripped_sources_digests = [stripped_sources.snapshot.directory_digest for stripped_sources in source_root_stripped_sources]
+  sources_digest = yield Get(Digest, DirectoriesToMerge(directories=tuple(stripped_sources_digests)))
+  inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
+  all_input_digests = [sources_digest, inits_digest.directory_digest]
+  merged_input_files = yield Get(Digest, DirectoriesToMerge, DirectoriesToMerge(directories=tuple(all_input_digests)))
+
+  #TODO This chunk of code should be made into an @rule and used both here and in
+  # python_test_runner.py.
+  # Produce a pex containing pytest and all transitive 3rdparty requirements.
+  all_target_requirements = []
+  for maybe_python_req_lib in all_target_adaptors:
+    # This is a python_requirement()-like target.
+    if hasattr(maybe_python_req_lib, 'requirement'):
+      all_target_requirements.append(str(maybe_python_req_lib.requirement))
+    # This is a python_requirement_library()-like target.
+    if hasattr(maybe_python_req_lib, 'requirements'):
+      for py_req in maybe_python_req_lib.requirements:
+        all_target_requirements.append(str(py_req.requirement))
+
+  output_filename = f"{python_binary_adaptor.address.target_name}.pex"
+
+  all_requirements = all_target_requirements
+  create_requirements_pex = CreatePex(
+    output_filename=output_filename,
+    requirements=tuple(sorted(all_requirements)),
+    interpreter_constraints=interpreter_constraints,
+    entry_point=entry_point,
+    input_files_digest=merged_input_files,
+  )
+
+  pex = yield Get(Pex, CreatePex, create_requirements_pex)
+  yield pex
+
+
+@rule
+def pex_to_created_binary(pex: Pex) -> CreatedBinary:
+  yield CreatedBinary(_inner=pex)
+
+
+def rules():
+  return [
+    UnionRule(BinaryTarget, PythonBinaryAdaptor),
+    pex_to_created_binary,
+    create_python_binary,
+  ]

src/python/pants/backend/python/rules/python_test_runner.py

@@ -2,6 +2,10 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.backend.python.rules.inject_init import InjectedInitDigest
+from pants.backend.python.rules.interpreter_constraints import (
+  BuildConstraintsForAdaptors,
+  PexInterpreterContraints,
+)
 from pants.backend.python.rules.pex import CreatePex, Pex
 from pants.backend.python.subsystems.pytest import PyTest
 from pants.backend.python.subsystems.python_setup import PythonSetup
@@ -33,13 +37,8 @@ def run_python_test(
   )
   all_targets = transitive_hydrated_targets.closure
 
-  interpreter_constraints = {
-    constraint
-    for target_adaptor in all_targets
-    for constraint in python_setup.compatibility_or_constraints(
-      getattr(target_adaptor, 'compatibility', None)
-    )
-  }
+  interpreter_constraints = yield Get(PexInterpreterContraints,
+      BuildConstraintsForAdaptors(adaptors=tuple(all_targets)))
 
   # Produce a pex containing pytest and all transitive 3rdparty requirements.
   output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
@@ -58,7 +57,7 @@ def run_python_test(
     Pex, CreatePex(
       output_filename=output_pytest_requirements_pex_filename,
       requirements=tuple(sorted(all_requirements)),
-      interpreter_constraints=tuple(sorted(interpreter_constraints)),
+      interpreter_constraints=interpreter_constraints,
       entry_point="pytest:main",
     )
   )

src/python/pants/backend/python/targets/python_binary.py

@@ -116,7 +116,7 @@ def __init__(self,
     if sources and sources.files and entry_point:
       entry_point_module = entry_point.split(':', 1)[0]
       entry_source = list(self.sources_relative_to_source_root())[0]
-      source_entry_point = self._translate_to_entry_point(entry_source)
+      source_entry_point = self.translate_source_path_to_py_module_specifier(entry_source)
       if entry_point_module != source_entry_point:
         raise TargetDefinitionException(self,
             'Specified both source and entry_point but they do not agree: {} vs {}'.format(
@@ -136,7 +136,8 @@ def repositories(self):
   def indices(self):
     return self.payload.indices
 
-  def _translate_to_entry_point(self, source):
+  @classmethod
+  def translate_source_path_to_py_module_specifier(self, source: str) -> str:
     source_base, _ = os.path.splitext(source)
     return source_base.replace(os.path.sep, '.')
 
@@ -147,7 +148,7 @@ def entry_point(self):
     elif self.payload.sources.source_paths:
       assert len(self.payload.sources.source_paths) == 1
       entry_source = list(self.sources_relative_to_source_root())[0]
-      return self._translate_to_entry_point(entry_source)
+      return self.translate_source_path_to_py_module_specifier(entry_source)
     else:
       return None
 

src/python/pants/rules/core/binary.py

@@ -0,0 +1,68 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from dataclasses import dataclass
+from typing import Any
+
+from pants.backend.python.rules.pex import Pex
+from pants.build_graph.address import Address
+from pants.engine.addressable import BuildFileAddresses
+from pants.engine.console import Console
+from pants.engine.fs import DirectoryToMaterialize, Workspace
+from pants.engine.goal import Goal, LineOriented
+from pants.engine.legacy.graph import HydratedTarget
+from pants.engine.legacy.structs import PythonBinaryAdaptor
+from pants.engine.rules import console_rule, optionable_rule, rule, union
+from pants.engine.selectors import Get
+
+
+@dataclass(frozen=True)
+class Binary(LineOriented, Goal):
+  name = 'binary'
+
+
+@union
+@dataclass(frozen=True)
+class BinaryTarget:
+  pass
+
+
+@union
+@dataclass(frozen=True)
+class CreatedBinary:
+  _inner: Any
+
+  def write_to_dist(self, print_stdout: Any, workspace: Workspace) -> None:
+    if isinstance(self._inner, Pex):
+      dtm = DirectoryToMaterialize(
+        path = 'dist/',
+        directory_digest = self._inner.directory_digest,
+      )
+      output = workspace.materialize_directories((dtm,))
+      for path in output.dependencies[0].output_paths:
+        print_stdout(f"Wrote {path}")
+    else:
+      raise ValueError(f"CreatedBinary should not contain a value of type: {type(self._inner)}")
+
+
+@console_rule
+def create_binary(addresses: BuildFileAddresses, console: Console, workspace: Workspace, options: Binary.Options) -> Binary:
+  with Binary.line_oriented(options, console) as (print_stdout, print_stderr):
+    print_stdout("Generating binaries in `dist/`")
+    binaries = yield [Get(CreatedBinary, Address, address.to_address()) for address in addresses]
+    for binary in binaries:
+      binary.write_to_dist(print_stdout, workspace)
+  yield Binary(exit_code=0)
+
+
+@rule
+def coordinator_of_binaries(target: HydratedTarget) -> CreatedBinary:
+  binary = yield Get(CreatedBinary, BinaryTarget, target.adaptor)
+  yield binary
+
+
+def rules():
+  return [
+    create_binary,
+    coordinator_of_binaries,
+  ]

src/python/pants/rules/core/register.py

@@ -1,11 +1,20 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from pants.rules.core import filedeps, fmt, list_roots, list_targets, strip_source_root, test
+from pants.rules.core import (
+  binary,
+  filedeps,
+  fmt,
+  list_roots,
+  list_targets,
+  strip_source_root,
+  test,
+)
 
 
 def rules():
   return [
+    *binary.rules(),
     *fmt.rules(),
     *list_roots.rules(),
     *list_targets.rules(),