From 10bab8ed4b8bc75b0af6483a90fbb3f3a7b7809a Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Tue, 29 Sep 2020 17:59:25 -0700 Subject: [PATCH 1/5] Deprecate `awslambda` and `binary` goals in favor of new `build` goal [ci skip-rust] [ci skip-build-wheels] --- .../{awslambda_common_rules.py => rules.py} | 9 ++- .../backend/awslambda/python/register.py | 6 +- .../{awslambda_python_rules.py => rules.py} | 19 ++++-- ...bda_python_rules_test.py => rules_test.py} | 11 ++- ...ython_binary.py => build_python_binary.py} | 22 ++++-- .../backend/python/goals/pytest_runner.py | 26 ++++--- .../goals/pytest_runner_integration_test.py | 6 +- .../backend/python/goals/run_python_binary.py | 2 +- src/python/pants/backend/python/register.py | 4 +- .../pants/backend/python/target_types.py | 35 ++++++++-- src/python/pants/core/goals/binary.py | 6 +- src/python/pants/core/goals/build.py | 68 +++++++++++++++++++ src/python/pants/core/register.py | 3 +- 13 files changed, 178 insertions(+), 39 deletions(-) rename src/python/pants/backend/awslambda/common/{awslambda_common_rules.py => rules.py} (89%) rename src/python/pants/backend/awslambda/python/{awslambda_python_rules.py => rules.py} (89%) rename src/python/pants/backend/awslambda/python/{awslambda_python_rules_test.py => rules_test.py} (82%) rename src/python/pants/backend/python/goals/{create_python_binary.py => build_python_binary.py} (88%) create mode 100644 src/python/pants/core/goals/build.py diff --git a/src/python/pants/backend/awslambda/common/awslambda_common_rules.py b/src/python/pants/backend/awslambda/common/rules.py similarity index 89% rename from src/python/pants/backend/awslambda/common/awslambda_common_rules.py rename to src/python/pants/backend/awslambda/common/rules.py index 2e2ab23976e..5768c6bdf4b 100644 --- a/src/python/pants/backend/awslambda/common/awslambda_common_rules.py +++ b/src/python/pants/backend/awslambda/common/rules.py @@ -1,6 +1,7 @@ # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import logging from abc import ABCMeta from dataclasses import dataclass from textwrap import dedent @@ -13,6 +14,8 @@ from pants.engine.target import FieldSet, TargetRootsToFieldSets, TargetRootsToFieldSetsRequest from pants.engine.unions import union +logger = logging.getLogger(__name__) + class AWSLambdaError(Exception): pass @@ -32,7 +35,7 @@ class AWSLambdaFieldSet(FieldSet, metaclass=ABCMeta): class AWSLambdaSubsystem(LineOriented, GoalSubsystem): - """Generate an AWS Lambda.""" + """Deprecated in favor of the `build` goal.""" name = "awslambda" @@ -48,6 +51,10 @@ async def create_awslambda( distdir: DistDir, workspace: Workspace, ) -> AWSLambdaGoal: + logger.warning( + "The `awslambda` goal is deprecated in favor of the `build` goal, which behaves " + "identically. `awslambda` will be removed in 2.1.0.dev0." + ) targets_to_valid_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( diff --git a/src/python/pants/backend/awslambda/python/register.py b/src/python/pants/backend/awslambda/python/register.py index 353c4d0b839..3acf64545cf 100644 --- a/src/python/pants/backend/awslambda/python/register.py +++ b/src/python/pants/backend/awslambda/python/register.py @@ -6,13 +6,13 @@ See https://www.pantsbuild.org/docs/awslambda-python. """ -from pants.backend.awslambda.common import awslambda_common_rules -from pants.backend.awslambda.python import awslambda_python_rules +from pants.backend.awslambda.common import rules as common_rules +from pants.backend.awslambda.python import rules as python_rules from pants.backend.awslambda.python.target_types import PythonAWSLambda def rules(): - return [*awslambda_common_rules.rules(), *awslambda_python_rules.rules()] + return [*common_rules.rules(), *python_rules.rules()] def target_types(): diff --git a/src/python/pants/backend/awslambda/python/awslambda_python_rules.py b/src/python/pants/backend/awslambda/python/rules.py similarity index 89% rename from src/python/pants/backend/awslambda/python/awslambda_python_rules.py rename to src/python/pants/backend/awslambda/python/rules.py index d17f2b2a023..375ec4ed6dd 100644 --- a/src/python/pants/backend/awslambda/python/awslambda_python_rules.py +++ b/src/python/pants/backend/awslambda/python/rules.py @@ -5,10 +5,7 @@ import os from dataclasses import dataclass -from pants.backend.awslambda.common.awslambda_common_rules import ( - AWSLambdaFieldSet, - CreatedAWSLambda, -) +from pants.backend.awslambda.common.rules import AWSLambdaFieldSet, CreatedAWSLambda from pants.backend.awslambda.python.lambdex import Lambdex from pants.backend.awslambda.python.target_types import ( PythonAwsLambdaHandler, @@ -28,6 +25,7 @@ PexFromTargetsRequest, TwoStepPexFromTargetsRequest, ) +from pants.core.goals.build import BuildFieldSet, BuiltAsset from pants.engine.fs import Digest, MergeDigests from pants.engine.process import ProcessResult from pants.engine.rules import Get, collect_rules, rule @@ -39,7 +37,7 @@ @dataclass(frozen=True) -class PythonAwsLambdaFieldSet(AWSLambdaFieldSet): +class PythonAwsLambdaFieldSet(BuildFieldSet, AWSLambdaFieldSet): required_fields = (PythonAwsLambdaHandler, PythonAwsLambdaRuntime) handler: PythonAwsLambdaHandler @@ -123,6 +121,16 @@ async def create_python_awslambda( ) +@rule +async def build_python_awslambda(field_set: PythonAwsLambdaFieldSet) -> BuiltAsset: + awslambda = await Get(CreatedAWSLambda, AWSLambdaFieldSet, field_set) + return BuiltAsset( + awslambda.digest, + relpath=awslambda.zip_file_relpath, + extra_log_info=f" Runtime: {awslambda.runtime}\n Handler: {awslambda.handler}", + ) + + @rule(desc="Set up lambdex") async def setup_lambdex(lambdex: Lambdex) -> LambdexSetup: requirements_pex = await Get( @@ -142,5 +150,6 @@ def rules(): return [ *collect_rules(), UnionRule(AWSLambdaFieldSet, PythonAwsLambdaFieldSet), + UnionRule(BuildFieldSet, PythonAwsLambdaFieldSet), *pex_from_targets.rules(), ] diff --git a/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py b/src/python/pants/backend/awslambda/python/rules_test.py similarity index 82% rename from src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py rename to src/python/pants/backend/awslambda/python/rules_test.py index b3a36957a1e..6c3d615b4f0 100644 --- a/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py +++ b/src/python/pants/backend/awslambda/python/rules_test.py @@ -8,11 +8,12 @@ import pytest -from pants.backend.awslambda.common.awslambda_common_rules import CreatedAWSLambda -from pants.backend.awslambda.python.awslambda_python_rules import PythonAwsLambdaFieldSet -from pants.backend.awslambda.python.awslambda_python_rules import rules as awslambda_python_rules +from pants.backend.awslambda.common.rules import CreatedAWSLambda +from pants.backend.awslambda.python.rules import PythonAwsLambdaFieldSet +from pants.backend.awslambda.python.rules import rules as awslambda_python_rules from pants.backend.awslambda.python.target_types import PythonAWSLambda from pants.backend.python.target_types import PythonLibrary +from pants.core.goals.build import BuiltAsset from pants.engine.addresses import Address from pants.engine.fs import DigestContents from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -23,6 +24,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *awslambda_python_rules(), + QueryRule(BuiltAsset, (PythonAwsLambdaFieldSet,)), QueryRule(CreatedAWSLambda, (PythonAwsLambdaFieldSet,)), ], target_types=[PythonAWSLambda, PythonLibrary], @@ -41,6 +43,9 @@ def create_python_awslambda(rule_runner: RuleRunner, addr: str) -> Tuple[str, by created_awslambda = rule_runner.request( CreatedAWSLambda, [PythonAwsLambdaFieldSet.create(target)] ) + built_asset = rule_runner.request(BuiltAsset, [PythonAwsLambdaFieldSet.create(target)]) + assert created_awslambda.digest == built_asset.digest + assert created_awslambda.zip_file_relpath == built_asset.relpath created_awslambda_digest_contents = rule_runner.request( DigestContents, [created_awslambda.digest] ) diff --git a/src/python/pants/backend/python/goals/create_python_binary.py b/src/python/pants/backend/python/goals/build_python_binary.py similarity index 88% rename from src/python/pants/backend/python/goals/create_python_binary.py rename to src/python/pants/backend/python/goals/build_python_binary.py index 0d6235f057c..cf2159eff71 100644 --- a/src/python/pants/backend/python/goals/create_python_binary.py +++ b/src/python/pants/backend/python/goals/build_python_binary.py @@ -24,6 +24,7 @@ TwoStepPexFromTargetsRequest, ) from pants.core.goals.binary import BinaryFieldSet, CreatedBinary +from pants.core.goals.build import BuildFieldSet, BuiltAsset from pants.core.goals.run import RunFieldSet from pants.engine.fs import PathGlobs, Paths from pants.engine.rules import Get, collect_rules, rule @@ -37,7 +38,7 @@ @dataclass(frozen=True) -class PythonBinaryFieldSet(BinaryFieldSet, RunFieldSet): +class PythonBinaryFieldSet(BuildFieldSet, BinaryFieldSet, RunFieldSet): required_fields = (PythonEntryPoint, PythonBinarySources) sources: PythonBinarySources @@ -71,11 +72,11 @@ def generate_additional_args( @rule(level=LogLevel.DEBUG) -async def create_python_binary( +async def build_python_binary( field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults, global_options: GlobalOptions, -) -> CreatedBinary: +) -> BuiltAsset: entry_point = field_set.entry_point.value if entry_point is None: binary_source_paths = await Get( @@ -124,9 +125,18 @@ async def create_python_binary( ) ), ) - pex = two_step_pex.pex - return CreatedBinary(digest=pex.digest, binary_name=pex.name) + return BuiltAsset(two_step_pex.pex.digest, relpath=output_filename) + + +@rule(level=LogLevel.DEBUG) +async def create_python_binary(field_set: PythonBinaryFieldSet) -> CreatedBinary: + pex = await Get(BuiltAsset, BuildFieldSet, field_set) + return CreatedBinary(pex.digest, pex.relpath) def rules(): - return [*collect_rules(), UnionRule(BinaryFieldSet, PythonBinaryFieldSet)] + return [ + *collect_rules(), + UnionRule(BuildFieldSet, PythonBinaryFieldSet), + UnionRule(BinaryFieldSet, PythonBinaryFieldSet), + ] diff --git a/src/python/pants/backend/python/goals/pytest_runner.py b/src/python/pants/backend/python/goals/pytest_runner.py index 70b952c10e2..7ba62558c3a 100644 --- a/src/python/pants/backend/python/goals/pytest_runner.py +++ b/src/python/pants/backend/python/goals/pytest_runner.py @@ -16,6 +16,7 @@ from pants.backend.python.target_types import ( PythonInterpreterCompatibility, PythonRuntimeBinaryDependencies, + PythonRuntimeBuildDependencies, PythonTestsSources, PythonTestsTimeout, ) @@ -31,7 +32,7 @@ PythonSourceFiles, PythonSourceFilesRequest, ) -from pants.core.goals.binary import BinaryFieldSet, CreatedBinary +from pants.core.goals.build import BuildFieldSet, BuiltAsset from pants.core.goals.test import ( TestDebugRequest, TestExtraEnv, @@ -64,6 +65,7 @@ class PythonTestFieldSet(TestFieldSet): sources: PythonTestsSources timeout: PythonTestsTimeout + runtime_build_dependencies: PythonRuntimeBuildDependencies runtime_binary_dependencies: PythonRuntimeBinaryDependencies def is_conftest_or_type_stub(self) -> bool: @@ -152,24 +154,30 @@ async def setup_pytest_for_target( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True) ) - # Create any binaries that the test depends on through the `runtime_binary_dependencies` field. - binaries: Tuple[CreatedBinary, ...] = () - if request.field_set.runtime_binary_dependencies.value: + # Create any assets that the test depends on through the `runtime_build_dependencies` field. + assets: Tuple[BuiltAsset, ...] = () + if ( + request.field_set.runtime_build_dependencies.value + or request.field_set.runtime_binary_dependencies.value + ): runtime_binary_addresses = await MultiGet( Get( Address, AddressInput, AddressInput.parse(v, relative_to=request.field_set.address.spec_path), ) - for v in request.field_set.runtime_binary_dependencies.value + for v in ( + *(request.field_set.runtime_build_dependencies.value or ()), + *(request.field_set.runtime_binary_dependencies.value or ()), + ) ) runtime_binary_targets = await Get(Targets, Addresses(runtime_binary_addresses)) field_sets_per_target = await Get( FieldSetsPerTarget, - FieldSetsPerTargetRequest(BinaryFieldSet, runtime_binary_targets), + FieldSetsPerTargetRequest(BuildFieldSet, runtime_binary_targets), ) - binaries = await MultiGet( - Get(CreatedBinary, BinaryFieldSet, field_set) + assets = await MultiGet( + Get(BuiltAsset, BuildFieldSet, field_set) for field_set in field_sets_per_target.field_sets ) @@ -194,7 +202,7 @@ async def setup_pytest_for_target( prepared_sources.source_files.snapshot.digest, requirements_pex.digest, pytest_pex.digest, - *(binary.digest for binary in binaries), + *(binary.digest for binary in assets), ) ), ) diff --git a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py index e7681dce5d8..b203e9f1de8 100644 --- a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py +++ b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py @@ -10,7 +10,7 @@ import pytest from pants.backend.python.dependency_inference import rules as dependency_inference_rules -from pants.backend.python.goals import create_python_binary, pytest_runner +from pants.backend.python.goals import build_python_binary, pytest_runner from pants.backend.python.goals.coverage_py import create_coverage_config from pants.backend.python.goals.pytest_runner import PythonTestFieldSet from pants.backend.python.target_types import ( @@ -40,7 +40,7 @@ def rule_runner() -> RuleRunner: *dependency_inference_rules.rules(), # For conftest detection. *distdir.rules(), *binary.rules(), - *create_python_binary.rules(), + *build_python_binary.rules(), get_filtered_environment, QueryRule(TestResult, (PythonTestFieldSet,)), QueryRule(TestDebugRequest, (PythonTestFieldSet,)), @@ -454,7 +454,7 @@ def test_embedded_binary(): """ ), ) - rule_runner.add_to_build_file(PACKAGE, "python_tests(runtime_binary_dependencies=[':bin'])") + rule_runner.add_to_build_file(PACKAGE, "python_tests(runtime_build_dependencies=[':bin'])") tgt = rule_runner.get_target(Address(PACKAGE, relative_file_path="test_binary_call.py")) assert isinstance(tgt, PythonTests) result = run_pytest(rule_runner, tgt, passthrough_args="-s") diff --git a/src/python/pants/backend/python/goals/run_python_binary.py b/src/python/pants/backend/python/goals/run_python_binary.py index 253a7d7b8c6..ca17bd0b970 100644 --- a/src/python/pants/backend/python/goals/run_python_binary.py +++ b/src/python/pants/backend/python/goals/run_python_binary.py @@ -3,7 +3,7 @@ import os -from pants.backend.python.goals.create_python_binary import PythonBinaryFieldSet +from pants.backend.python.goals.build_python_binary import PythonBinaryFieldSet from pants.backend.python.target_types import PythonBinaryDefaults, PythonBinarySources from pants.backend.python.util_rules.pex import Pex, PexRequest from pants.backend.python.util_rules.pex_environment import PexEnvironment diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 42f5c01b475..1b435a44ecf 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -8,8 +8,8 @@ from pants.backend.python.dependency_inference import rules as dependency_inference_rules from pants.backend.python.goals import ( + build_python_binary, coverage_py, - create_python_binary, pytest_runner, repl, run_python_binary, @@ -69,7 +69,7 @@ def rules(): *pex_environment.rules(), *pex_from_targets.rules(), *pytest_runner.rules(), - *create_python_binary.rules(), + *build_python_binary.rules(), *python_native_code.rules(), *repl.rules(), *run_python_binary.rules(), diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index e185b402ffe..4bcde1a64e8 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import collections.abc +import logging import os.path from textwrap import dedent from typing import Iterable, Optional, Tuple, Union, cast @@ -37,6 +38,8 @@ from pants.python.python_requirement import PythonRequirement from pants.python.python_setup import PythonSetup +logger = logging.getLogger(__name__) + # ----------------------------------------------------------------------------------------------- # Common fields # ----------------------------------------------------------------------------------------------- @@ -264,15 +267,39 @@ class PythonTestsDependencies(Dependencies): supports_transitive_excludes = True -class PythonRuntimeBinaryDependencies(StringSequenceField): - """Addresses to binary targets that will be built and included in this target at runtime. +class PythonRuntimeBuildDependencies(StringSequenceField): + """Addresses to targets that can be built with the `./pants build` goal and whose resulting + assets should be included in the test run. - These binary targets do not necessarily need to be `python_binary` targets; they can be any - targets accepted by `./pants binary` + Pants will build the asset as if it had run `./pants build`, and will include the result in + your test environment using the same name it would normally have, but without the `dist/` + prefix. + + These targets do not necessarily need to be `python_binary` targets; they can be any + targets accepted by `./pants build`, as `python_awslambda` and `archive`. """ + alias = "runtime_build_dependencies" + + +class PythonRuntimeBinaryDependencies(StringSequenceField): + """Deprecated in favor of the `runtime_build_dependencies` field, which works with more target + types like `archive` and `python_awslambda`.""" + alias = "runtime_binary_dependencies" + @classmethod + def compute_value( + cls, raw_value: Optional[Iterable[str]], *, address: Address + ) -> Optional[Tuple[str, ...]]: + if raw_value is not None: + logger.warning( + f"Using the `runtime_binary_dependencies` field in the target {address}. This " + "field is now deprecated in favor of the more flexible " + "`runtime_build_dependencies` field, and it will be removed in 2.1.0.dev0." + ) + return super().compute_value(raw_value, address=address) + class PythonTestsTimeout(IntField): """A timeout (in seconds) which covers the total runtime of all tests in this target. diff --git a/src/python/pants/core/goals/binary.py b/src/python/pants/core/goals/binary.py index c02676e4e02..5b650c68eea 100644 --- a/src/python/pants/core/goals/binary.py +++ b/src/python/pants/core/goals/binary.py @@ -27,7 +27,7 @@ class CreatedBinary: class BinarySubsystem(GoalSubsystem): - """Create a runnable binary.""" + """Deprecated in favor of the `build` goal.""" name = "binary" @@ -40,6 +40,10 @@ class Binary(Goal): @goal_rule async def create_binary(workspace: Workspace, dist_dir: DistDir) -> Binary: + logger.warning( + "The `binary` goal is deprecated in favor of the `build` goal, which behaves identically. " + "`binary` will be removed in 2.1.0.dev0.", + ) target_roots_to_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( diff --git a/src/python/pants/core/goals/build.py b/src/python/pants/core/goals/build.py new file mode 100644 index 00000000000..b99638f51a9 --- /dev/null +++ b/src/python/pants/core/goals/build.py @@ -0,0 +1,68 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import logging +from abc import ABCMeta +from dataclasses import dataclass +from typing import Optional + +from pants.core.util_rules.distdir import DistDir +from pants.engine.fs import Digest, MergeDigests, Snapshot, Workspace +from pants.engine.goal import Goal, GoalSubsystem +from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule +from pants.engine.target import FieldSet, TargetRootsToFieldSets, TargetRootsToFieldSetsRequest +from pants.engine.unions import union + +logger = logging.getLogger(__name__) + + +@union +class BuildFieldSet(FieldSet, metaclass=ABCMeta): + """The fields necessary to build an asset from a target.""" + + +@dataclass(frozen=True) +class BuiltAsset: + digest: Digest + relpath: str + extra_log_info: Optional[str] = None + + +class BuildSubsystem(GoalSubsystem): + """Build an asset, such as an archive, JAR, PEX, AWS Lambda, etc.""" + + name = "build" + + required_union_implementations = (BuildFieldSet,) + + +class Build(Goal): + subsystem_cls = BuildSubsystem + + +@goal_rule +async def build_asset(workspace: Workspace, dist_dir: DistDir) -> Build: + target_roots_to_field_sets = await Get( + TargetRootsToFieldSets, + TargetRootsToFieldSetsRequest( + BuildFieldSet, + goal_description="the `build` goal", + error_if_no_applicable_targets=True, + ), + ) + assets = await MultiGet( + Get(BuiltAsset, BuildFieldSet, field_set) + for field_set in target_roots_to_field_sets.field_sets + ) + merged_snapshot = await Get(Snapshot, MergeDigests(asset.digest for asset in assets)) + workspace.write_digest(merged_snapshot.digest, path_prefix=str(dist_dir.relpath)) + for asset in assets: + msg = f"Wrote {dist_dir.relpath / asset.relpath}" + if asset.extra_log_info: + msg += f"\n{asset.extra_log_info}" + logger.info(msg) + return Build(exit_code=0) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/core/register.py b/src/python/pants/core/register.py index b06a9077e28..b76a861c25b 100644 --- a/src/python/pants/core/register.py +++ b/src/python/pants/core/register.py @@ -6,7 +6,7 @@ These are always activated and cannot be disabled. """ -from pants.core.goals import binary, fmt, lint, repl, run, test, typecheck +from pants.core.goals import binary, build, fmt, lint, repl, run, test, typecheck from pants.core.target_types import Files, GenericTarget, Resources from pants.core.util_rules import ( archive, @@ -25,6 +25,7 @@ def rules(): return [ # goals + *build.rules(), *binary.rules(), *fmt.rules(), *lint.rules(), From c5306712296d30273285ba2ea5f7ae06db840ac9 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 30 Sep 2020 18:18:29 -0700 Subject: [PATCH 2/5] Rename `build` to `package` Thanks for the suggestion, JP! # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/awslambda/common/rules.py | 2 +- .../pants/backend/awslambda/python/rules.py | 10 +++---- .../backend/awslambda/python/rules_test.py | 6 ++--- .../python/goals/build_python_binary.py | 12 ++++----- .../backend/python/goals/pytest_runner.py | 22 ++++++++-------- .../goals/pytest_runner_integration_test.py | 4 +-- .../pants/backend/python/target_types.py | 12 ++++----- src/python/pants/core/goals/binary.py | 4 +-- .../pants/core/goals/{build.py => package.py} | 26 +++++++++---------- src/python/pants/core/register.py | 4 +-- 10 files changed, 51 insertions(+), 51 deletions(-) rename src/python/pants/core/goals/{build.py => package.py} (72%) diff --git a/src/python/pants/backend/awslambda/common/rules.py b/src/python/pants/backend/awslambda/common/rules.py index 5768c6bdf4b..1d8dd2cf290 100644 --- a/src/python/pants/backend/awslambda/common/rules.py +++ b/src/python/pants/backend/awslambda/common/rules.py @@ -52,7 +52,7 @@ async def create_awslambda( workspace: Workspace, ) -> AWSLambdaGoal: logger.warning( - "The `awslambda` goal is deprecated in favor of the `build` goal, which behaves " + "The `awslambda` goal is deprecated in favor of the `package` goal, which behaves " "identically. `awslambda` will be removed in 2.1.0.dev0." ) targets_to_valid_field_sets = await Get( diff --git a/src/python/pants/backend/awslambda/python/rules.py b/src/python/pants/backend/awslambda/python/rules.py index 375ec4ed6dd..e20be07c8d1 100644 --- a/src/python/pants/backend/awslambda/python/rules.py +++ b/src/python/pants/backend/awslambda/python/rules.py @@ -25,7 +25,7 @@ PexFromTargetsRequest, TwoStepPexFromTargetsRequest, ) -from pants.core.goals.build import BuildFieldSet, BuiltAsset +from pants.core.goals.package import BuiltPackage, PackageFieldSet from pants.engine.fs import Digest, MergeDigests from pants.engine.process import ProcessResult from pants.engine.rules import Get, collect_rules, rule @@ -37,7 +37,7 @@ @dataclass(frozen=True) -class PythonAwsLambdaFieldSet(BuildFieldSet, AWSLambdaFieldSet): +class PythonAwsLambdaFieldSet(PackageFieldSet, AWSLambdaFieldSet): required_fields = (PythonAwsLambdaHandler, PythonAwsLambdaRuntime) handler: PythonAwsLambdaHandler @@ -122,9 +122,9 @@ async def create_python_awslambda( @rule -async def build_python_awslambda(field_set: PythonAwsLambdaFieldSet) -> BuiltAsset: +async def package_python_awslambda(field_set: PythonAwsLambdaFieldSet) -> BuiltPackage: awslambda = await Get(CreatedAWSLambda, AWSLambdaFieldSet, field_set) - return BuiltAsset( + return BuiltPackage( awslambda.digest, relpath=awslambda.zip_file_relpath, extra_log_info=f" Runtime: {awslambda.runtime}\n Handler: {awslambda.handler}", @@ -150,6 +150,6 @@ def rules(): return [ *collect_rules(), UnionRule(AWSLambdaFieldSet, PythonAwsLambdaFieldSet), - UnionRule(BuildFieldSet, PythonAwsLambdaFieldSet), + UnionRule(PackageFieldSet, PythonAwsLambdaFieldSet), *pex_from_targets.rules(), ] diff --git a/src/python/pants/backend/awslambda/python/rules_test.py b/src/python/pants/backend/awslambda/python/rules_test.py index 6c3d615b4f0..835aae2e8ca 100644 --- a/src/python/pants/backend/awslambda/python/rules_test.py +++ b/src/python/pants/backend/awslambda/python/rules_test.py @@ -13,7 +13,7 @@ from pants.backend.awslambda.python.rules import rules as awslambda_python_rules from pants.backend.awslambda.python.target_types import PythonAWSLambda from pants.backend.python.target_types import PythonLibrary -from pants.core.goals.build import BuiltAsset +from pants.core.goals.package import BuiltPackage from pants.engine.addresses import Address from pants.engine.fs import DigestContents from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -24,7 +24,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *awslambda_python_rules(), - QueryRule(BuiltAsset, (PythonAwsLambdaFieldSet,)), + QueryRule(BuiltPackage, (PythonAwsLambdaFieldSet,)), QueryRule(CreatedAWSLambda, (PythonAwsLambdaFieldSet,)), ], target_types=[PythonAWSLambda, PythonLibrary], @@ -43,7 +43,7 @@ def create_python_awslambda(rule_runner: RuleRunner, addr: str) -> Tuple[str, by created_awslambda = rule_runner.request( CreatedAWSLambda, [PythonAwsLambdaFieldSet.create(target)] ) - built_asset = rule_runner.request(BuiltAsset, [PythonAwsLambdaFieldSet.create(target)]) + built_asset = rule_runner.request(BuiltPackage, [PythonAwsLambdaFieldSet.create(target)]) assert created_awslambda.digest == built_asset.digest assert created_awslambda.zip_file_relpath == built_asset.relpath created_awslambda_digest_contents = rule_runner.request( diff --git a/src/python/pants/backend/python/goals/build_python_binary.py b/src/python/pants/backend/python/goals/build_python_binary.py index cf2159eff71..4fd257f25cb 100644 --- a/src/python/pants/backend/python/goals/build_python_binary.py +++ b/src/python/pants/backend/python/goals/build_python_binary.py @@ -24,7 +24,7 @@ TwoStepPexFromTargetsRequest, ) from pants.core.goals.binary import BinaryFieldSet, CreatedBinary -from pants.core.goals.build import BuildFieldSet, BuiltAsset +from pants.core.goals.package import BuiltPackage, PackageFieldSet from pants.core.goals.run import RunFieldSet from pants.engine.fs import PathGlobs, Paths from pants.engine.rules import Get, collect_rules, rule @@ -38,7 +38,7 @@ @dataclass(frozen=True) -class PythonBinaryFieldSet(BuildFieldSet, BinaryFieldSet, RunFieldSet): +class PythonBinaryFieldSet(PackageFieldSet, BinaryFieldSet, RunFieldSet): required_fields = (PythonEntryPoint, PythonBinarySources) sources: PythonBinarySources @@ -76,7 +76,7 @@ async def build_python_binary( field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults, global_options: GlobalOptions, -) -> BuiltAsset: +) -> BuiltPackage: entry_point = field_set.entry_point.value if entry_point is None: binary_source_paths = await Get( @@ -125,18 +125,18 @@ async def build_python_binary( ) ), ) - return BuiltAsset(two_step_pex.pex.digest, relpath=output_filename) + return BuiltPackage(two_step_pex.pex.digest, relpath=output_filename) @rule(level=LogLevel.DEBUG) async def create_python_binary(field_set: PythonBinaryFieldSet) -> CreatedBinary: - pex = await Get(BuiltAsset, BuildFieldSet, field_set) + pex = await Get(BuiltPackage, PackageFieldSet, field_set) return CreatedBinary(pex.digest, pex.relpath) def rules(): return [ *collect_rules(), - UnionRule(BuildFieldSet, PythonBinaryFieldSet), + UnionRule(PackageFieldSet, PythonBinaryFieldSet), UnionRule(BinaryFieldSet, PythonBinaryFieldSet), ] diff --git a/src/python/pants/backend/python/goals/pytest_runner.py b/src/python/pants/backend/python/goals/pytest_runner.py index 7ba62558c3a..8caf8159780 100644 --- a/src/python/pants/backend/python/goals/pytest_runner.py +++ b/src/python/pants/backend/python/goals/pytest_runner.py @@ -16,7 +16,7 @@ from pants.backend.python.target_types import ( PythonInterpreterCompatibility, PythonRuntimeBinaryDependencies, - PythonRuntimeBuildDependencies, + PythonRuntimePackageDependencies, PythonTestsSources, PythonTestsTimeout, ) @@ -32,7 +32,7 @@ PythonSourceFiles, PythonSourceFilesRequest, ) -from pants.core.goals.build import BuildFieldSet, BuiltAsset +from pants.core.goals.package import BuiltPackage, PackageFieldSet from pants.core.goals.test import ( TestDebugRequest, TestExtraEnv, @@ -65,8 +65,8 @@ class PythonTestFieldSet(TestFieldSet): sources: PythonTestsSources timeout: PythonTestsTimeout - runtime_build_dependencies: PythonRuntimeBuildDependencies runtime_binary_dependencies: PythonRuntimeBinaryDependencies + runtime_package_dependencies: PythonRuntimePackageDependencies def is_conftest_or_type_stub(self) -> bool: """We skip both `conftest.py` and `.pyi` stubs, even though though they often belong to a @@ -154,30 +154,30 @@ async def setup_pytest_for_target( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True) ) - # Create any assets that the test depends on through the `runtime_build_dependencies` field. - assets: Tuple[BuiltAsset, ...] = () + # Create any assets that the test depends on through the `runtime_package_dependencies` field. + assets: Tuple[BuiltPackage, ...] = () if ( - request.field_set.runtime_build_dependencies.value + request.field_set.runtime_package_dependencies.value or request.field_set.runtime_binary_dependencies.value ): - runtime_binary_addresses = await MultiGet( + runtime_package_addresses = await MultiGet( Get( Address, AddressInput, AddressInput.parse(v, relative_to=request.field_set.address.spec_path), ) for v in ( - *(request.field_set.runtime_build_dependencies.value or ()), + *(request.field_set.runtime_package_dependencies.value or ()), *(request.field_set.runtime_binary_dependencies.value or ()), ) ) - runtime_binary_targets = await Get(Targets, Addresses(runtime_binary_addresses)) + runtime_package_targets = await Get(Targets, Addresses(runtime_package_addresses)) field_sets_per_target = await Get( FieldSetsPerTarget, - FieldSetsPerTargetRequest(BuildFieldSet, runtime_binary_targets), + FieldSetsPerTargetRequest(PackageFieldSet, runtime_package_targets), ) assets = await MultiGet( - Get(BuiltAsset, BuildFieldSet, field_set) + Get(BuiltPackage, PackageFieldSet, field_set) for field_set in field_sets_per_target.field_sets ) diff --git a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py index b203e9f1de8..ce7ae88ccf8 100644 --- a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py +++ b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py @@ -441,7 +441,7 @@ def test_args(): assert result.exit_code == 0 -def test_runtime_binary_dependency(rule_runner: RuleRunner) -> None: +def test_runtime_package_dependency(rule_runner: RuleRunner) -> None: create_python_binary_target(rule_runner, BINARY_SOURCE) rule_runner.create_file( f"{PACKAGE}/test_binary_call.py", @@ -454,7 +454,7 @@ def test_embedded_binary(): """ ), ) - rule_runner.add_to_build_file(PACKAGE, "python_tests(runtime_build_dependencies=[':bin'])") + rule_runner.add_to_build_file(PACKAGE, "python_tests(runtime_package_dependencies=[':bin'])") tgt = rule_runner.get_target(Address(PACKAGE, relative_file_path="test_binary_call.py")) assert isinstance(tgt, PythonTests) result = run_pytest(rule_runner, tgt, passthrough_args="-s") diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index 4bcde1a64e8..c71fa7f9b31 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -267,19 +267,19 @@ class PythonTestsDependencies(Dependencies): supports_transitive_excludes = True -class PythonRuntimeBuildDependencies(StringSequenceField): - """Addresses to targets that can be built with the `./pants build` goal and whose resulting +class PythonRuntimePackageDependencies(StringSequenceField): + """Addresses to targets that can be built with the `./pants package` goal and whose resulting assets should be included in the test run. - Pants will build the asset as if it had run `./pants build`, and will include the result in + Pants will build the asset as if it had run `./pants package`, and will include the result in your test environment using the same name it would normally have, but without the `dist/` prefix. These targets do not necessarily need to be `python_binary` targets; they can be any - targets accepted by `./pants build`, as `python_awslambda` and `archive`. + targets accepted by `./pants package`, as `python_awslambda` and `archive`. """ - alias = "runtime_build_dependencies" + alias = "runtime_package_dependencies" class PythonRuntimeBinaryDependencies(StringSequenceField): @@ -296,7 +296,7 @@ def compute_value( logger.warning( f"Using the `runtime_binary_dependencies` field in the target {address}. This " "field is now deprecated in favor of the more flexible " - "`runtime_build_dependencies` field, and it will be removed in 2.1.0.dev0." + "`runtime_package_dependencies` field, and it will be removed in 2.1.0.dev0." ) return super().compute_value(raw_value, address=address) diff --git a/src/python/pants/core/goals/binary.py b/src/python/pants/core/goals/binary.py index 5b650c68eea..2133e272219 100644 --- a/src/python/pants/core/goals/binary.py +++ b/src/python/pants/core/goals/binary.py @@ -41,8 +41,8 @@ class Binary(Goal): @goal_rule async def create_binary(workspace: Workspace, dist_dir: DistDir) -> Binary: logger.warning( - "The `binary` goal is deprecated in favor of the `build` goal, which behaves identically. " - "`binary` will be removed in 2.1.0.dev0.", + "The `binary` goal is deprecated in favor of the `package` goal, which behaves " + "identically. `binary` will be removed in 2.1.0.dev0.", ) target_roots_to_field_sets = await Get( TargetRootsToFieldSets, diff --git a/src/python/pants/core/goals/build.py b/src/python/pants/core/goals/package.py similarity index 72% rename from src/python/pants/core/goals/build.py rename to src/python/pants/core/goals/package.py index b99638f51a9..6376f0ac075 100644 --- a/src/python/pants/core/goals/build.py +++ b/src/python/pants/core/goals/package.py @@ -17,41 +17,41 @@ @union -class BuildFieldSet(FieldSet, metaclass=ABCMeta): +class PackageFieldSet(FieldSet, metaclass=ABCMeta): """The fields necessary to build an asset from a target.""" @dataclass(frozen=True) -class BuiltAsset: +class BuiltPackage: digest: Digest relpath: str extra_log_info: Optional[str] = None -class BuildSubsystem(GoalSubsystem): - """Build an asset, such as an archive, JAR, PEX, AWS Lambda, etc.""" +class PackageSubsystem(GoalSubsystem): + """Package an asset and put in `--distdir`, such as an archive, PEX, wheel, AWS Lambda, etc.""" - name = "build" + name = "package" - required_union_implementations = (BuildFieldSet,) + required_union_implementations = (PackageFieldSet,) -class Build(Goal): - subsystem_cls = BuildSubsystem +class Package(Goal): + subsystem_cls = PackageSubsystem @goal_rule -async def build_asset(workspace: Workspace, dist_dir: DistDir) -> Build: +async def package_asset(workspace: Workspace, dist_dir: DistDir) -> Package: target_roots_to_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( - BuildFieldSet, - goal_description="the `build` goal", + PackageFieldSet, + goal_description="the `package` goal", error_if_no_applicable_targets=True, ), ) assets = await MultiGet( - Get(BuiltAsset, BuildFieldSet, field_set) + Get(BuiltPackage, PackageFieldSet, field_set) for field_set in target_roots_to_field_sets.field_sets ) merged_snapshot = await Get(Snapshot, MergeDigests(asset.digest for asset in assets)) @@ -61,7 +61,7 @@ async def build_asset(workspace: Workspace, dist_dir: DistDir) -> Build: if asset.extra_log_info: msg += f"\n{asset.extra_log_info}" logger.info(msg) - return Build(exit_code=0) + return Package(exit_code=0) def rules(): diff --git a/src/python/pants/core/register.py b/src/python/pants/core/register.py index b76a861c25b..2f5224bf2f2 100644 --- a/src/python/pants/core/register.py +++ b/src/python/pants/core/register.py @@ -6,7 +6,7 @@ These are always activated and cannot be disabled. """ -from pants.core.goals import binary, build, fmt, lint, repl, run, test, typecheck +from pants.core.goals import binary, fmt, lint, package, repl, run, test, typecheck from pants.core.target_types import Files, GenericTarget, Resources from pants.core.util_rules import ( archive, @@ -25,7 +25,7 @@ def rules(): return [ # goals - *build.rules(), + *package.rules(), *binary.rules(), *fmt.rules(), *lint.rules(), From 7f3a235b19ed4eb17a9d264e260bbe35b8a49fe3 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 1 Oct 2020 09:03:15 -0700 Subject: [PATCH 3/5] Rename file too --- .../{build_python_binary.py => package_python_binary.py} | 2 +- .../backend/python/goals/pytest_runner_integration_test.py | 4 ++-- src/python/pants/backend/python/goals/run_python_binary.py | 2 +- src/python/pants/backend/python/register.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) rename src/python/pants/backend/python/goals/{build_python_binary.py => package_python_binary.py} (99%) diff --git a/src/python/pants/backend/python/goals/build_python_binary.py b/src/python/pants/backend/python/goals/package_python_binary.py similarity index 99% rename from src/python/pants/backend/python/goals/build_python_binary.py rename to src/python/pants/backend/python/goals/package_python_binary.py index 4fd257f25cb..f435f39252f 100644 --- a/src/python/pants/backend/python/goals/build_python_binary.py +++ b/src/python/pants/backend/python/goals/package_python_binary.py @@ -72,7 +72,7 @@ def generate_additional_args( @rule(level=LogLevel.DEBUG) -async def build_python_binary( +async def package_python_binary( field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults, global_options: GlobalOptions, diff --git a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py index ce7ae88ccf8..cf31601b8a2 100644 --- a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py +++ b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py @@ -10,7 +10,7 @@ import pytest from pants.backend.python.dependency_inference import rules as dependency_inference_rules -from pants.backend.python.goals import build_python_binary, pytest_runner +from pants.backend.python.goals import package_python_binary, pytest_runner from pants.backend.python.goals.coverage_py import create_coverage_config from pants.backend.python.goals.pytest_runner import PythonTestFieldSet from pants.backend.python.target_types import ( @@ -40,7 +40,7 @@ def rule_runner() -> RuleRunner: *dependency_inference_rules.rules(), # For conftest detection. *distdir.rules(), *binary.rules(), - *build_python_binary.rules(), + *package_python_binary.rules(), get_filtered_environment, QueryRule(TestResult, (PythonTestFieldSet,)), QueryRule(TestDebugRequest, (PythonTestFieldSet,)), diff --git a/src/python/pants/backend/python/goals/run_python_binary.py b/src/python/pants/backend/python/goals/run_python_binary.py index ca17bd0b970..838703be7a7 100644 --- a/src/python/pants/backend/python/goals/run_python_binary.py +++ b/src/python/pants/backend/python/goals/run_python_binary.py @@ -3,7 +3,7 @@ import os -from pants.backend.python.goals.build_python_binary import PythonBinaryFieldSet +from pants.backend.python.goals.package_python_binary import PythonBinaryFieldSet from pants.backend.python.target_types import PythonBinaryDefaults, PythonBinarySources from pants.backend.python.util_rules.pex import Pex, PexRequest from pants.backend.python.util_rules.pex_environment import PexEnvironment diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 1b435a44ecf..3120c22f366 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -8,8 +8,8 @@ from pants.backend.python.dependency_inference import rules as dependency_inference_rules from pants.backend.python.goals import ( - build_python_binary, coverage_py, + package_python_binary, pytest_runner, repl, run_python_binary, @@ -69,7 +69,7 @@ def rules(): *pex_environment.rules(), *pex_from_targets.rules(), *pytest_runner.rules(), - *build_python_binary.rules(), + *package_python_binary.rules(), *python_native_code.rules(), *repl.rules(), *run_python_binary.rules(), From 5ee8888777fe17858c52420dbd7e9bba07e582cb Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 1 Oct 2020 09:17:56 -0700 Subject: [PATCH 4/5] Review feedback + add a TODO --- src/python/pants/backend/python/target_types.py | 7 +++---- src/python/pants/core/target_types.py | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index c71fa7f9b31..41c9d51338d 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -267,16 +267,15 @@ class PythonTestsDependencies(Dependencies): supports_transitive_excludes = True +# TODO(#10888): Teach project introspection goals that this is a special type of the `Dependencies` +# field. class PythonRuntimePackageDependencies(StringSequenceField): """Addresses to targets that can be built with the `./pants package` goal and whose resulting assets should be included in the test run. Pants will build the asset as if it had run `./pants package`, and will include the result in your test environment using the same name it would normally have, but without the `dist/` - prefix. - - These targets do not necessarily need to be `python_binary` targets; they can be any - targets accepted by `./pants package`, as `python_awslambda` and `archive`. + prefix. For example, you can include a `python_binary`, `python_awslambda`, or `archive`. """ alias = "runtime_package_dependencies" diff --git a/src/python/pants/core/target_types.py b/src/python/pants/core/target_types.py index 22574fb39d0..4a99b703899 100644 --- a/src/python/pants/core/target_types.py +++ b/src/python/pants/core/target_types.py @@ -55,6 +55,8 @@ class RelocatedFilesSources(Sources): expected_num_files = 0 +# TODO(#10888): Teach project introspection goals that this is a special type of the `Dependencies` +# field. class RelocatedFilesOriginalTargets(StringSequenceField): """Addresses to the original `files()` targets that you want to relocate, such as `['//:json_files']`. From 4c0a8d238af49014812d7505df559d329d363fc1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 1 Oct 2020 09:20:21 -0700 Subject: [PATCH 5/5] Actually register the field # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/python/target_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index 41c9d51338d..265620b78e9 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -349,6 +349,7 @@ class PythonTests(Target): *COMMON_PYTHON_FIELDS, PythonTestsSources, PythonTestsDependencies, + PythonRuntimePackageDependencies, PythonRuntimeBinaryDependencies, PythonTestsTimeout, )