From 84bb69d47080c32f2f03b49f0ee025e5dfd3a4c1 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 9 Sep 2021 14:37:39 -0700 Subject: [PATCH] Upgrade to Pex 2.1.48 and leverage packed layout. (#12808) Pex 2.1.48 brings `--layout {packed,loose}` alternate layouts for PEXes. These are both more friendly to remote caching, leading to smaller artifacts to cache and greater cache hit ratios in the face of requirement changes. Since the loose layout still does not perform well with the local CAS scheme, we use packed for now. Fixes #12548 Fixes #12688 Fixes #12803 [ci skip-rust] [ci skip-build-wheels] --- 3rdparty/python/lockfiles/user_reqs.txt | 8 +- 3rdparty/python/requirements.txt | 2 +- build-support/bin/_release_helper.py | 1 - .../python/goals/package_pex_binary.py | 6 - .../backend/python/goals/pytest_runner.py | 10 +- src/python/pants/backend/python/goals/repl.py | 4 +- .../backend/python/goals/run_pex_binary.py | 4 +- .../goals/run_pex_binary_integration_test.py | 1 - .../pants/backend/python/lint/pylint/rules.py | 10 +- .../pants/backend/python/target_types.py | 50 +++- .../backend/python/typecheck/mypy/rules.py | 10 +- .../pants/backend/python/util_rules/pex.py | 277 ++++++------------ .../backend/python/util_rules/pex_cli.py | 9 +- .../python/util_rules/pex_from_targets.py | 51 ++-- .../util_rules/pex_from_targets_test.py | 75 +---- .../backend/python/util_rules/pex_test.py | 243 ++++++++------- src/python/pants/bin/BUILD | 1 - src/python/pants/init/plugin_resolver.py | 7 +- .../pants_test/init/test_plugin_resolver.py | 13 +- 19 files changed, 327 insertions(+), 455 deletions(-) diff --git a/3rdparty/python/lockfiles/user_reqs.txt b/3rdparty/python/lockfiles/user_reqs.txt index adcca514bf5..3ad594c917c 100644 --- a/3rdparty/python/lockfiles/user_reqs.txt +++ b/3rdparty/python/lockfiles/user_reqs.txt @@ -5,7 +5,7 @@ # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { # "version": 1, -# "requirements_invalidation_digest": "19770133e0608f747845bc429b61942d2f79b3cad46790962ade28db07e2b4fd", +# "requirements_invalidation_digest": "6adf6599ccc90c6aad4cc62333b089eea937311f47ae9eab025203d7a0e0eb2c", # "valid_for_interpreter_constraints": [ # "CPython<3.10,>=3.7" # ] @@ -113,9 +113,9 @@ iniconfig==1.1.1; python_version >= "3.6" \ packaging==21.0; python_version >= "3.6" \ --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 \ --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 -pex==2.1.47; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "3.10") \ - --hash=sha256:2341bad1146136237f2d7a7440fe706f010879febe0d8a714e6cd92b8ba47ce9 \ - --hash=sha256:0928d0316caac840db528030fc741930e8be22a3fa6a8635308fb8443a0a0c6a +pex==2.1.48; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "3.10") \ + --hash=sha256:903ab1e2781dbf077e9936d6a5dada9fa868c5416423065eee7bd00d94f2ea06 \ + --hash=sha256:5f6a489075c5bbecdb36a42249cd52cfd882e205242f80a1f1e2294951ab46e7 pluggy==1.0.0; python_version >= "3.6" \ --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 \ --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 17f84e4a030..9529ea203a2 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -8,7 +8,7 @@ humbug==0.2.6 ijson==3.1.4 packaging==21.0 -pex==2.1.47 +pex==2.1.48 psutil==5.8.0 pystache==0.5.4 # This should be kept in sync with `pytest.py`. diff --git a/build-support/bin/_release_helper.py b/build-support/bin/_release_helper.py index 0f0f13795ee..8b86344cd4f 100644 --- a/build-support/bin/_release_helper.py +++ b/build-support/bin/_release_helper.py @@ -598,7 +598,6 @@ def build_pex(fetch: bool) -> None: str(CONSTANTS.deploy_3rdparty_wheel_dir / CONSTANTS.pants_unstable_version), "--no-strip-pex-env", "--console-script=pants", - "--unzip", *extra_pex_args, f"pantsbuild.pants=={CONSTANTS.pants_unstable_version}", ], diff --git a/src/python/pants/backend/python/goals/package_pex_binary.py b/src/python/pants/backend/python/goals/package_pex_binary.py index 4987d83dcff..fc41b1e41f1 100644 --- a/src/python/pants/backend/python/goals/package_pex_binary.py +++ b/src/python/pants/backend/python/goals/package_pex_binary.py @@ -71,8 +71,6 @@ def _execution_mode(self) -> PexExecutionMode: def generate_additional_args(self, pex_binary_defaults: PexBinaryDefaults) -> Tuple[str, ...]: args = [] - if self.always_write_cache.value is True: - args.append("--always-write-cache") if self.emit_warnings.value_or_global_default(pex_binary_defaults) is False: args.append("--no-emit-warnings") if self.ignore_errors.value is True: @@ -81,12 +79,8 @@ def generate_additional_args(self, pex_binary_defaults: PexBinaryDefaults) -> Tu args.append(f"--inherit-path={self.inherit_path.value}") if self.shebang.value is not None: args.append(f"--python-shebang={self.shebang.value}") - if self.zip_safe.value is False: - args.append("--not-zip-safe") if self.strip_env.value is False: args.append("--no-strip-pex-env") - if self._execution_mode is PexExecutionMode.UNZIP: - args.append("--unzip") if self._execution_mode is PexExecutionMode.VENV: args.extend(("--venv", "prepend")) if self.include_tools.value is True: diff --git a/src/python/pants/backend/python/goals/pytest_runner.py b/src/python/pants/backend/python/goals/pytest_runner.py index 27cafbfa8de..b8923d55596 100644 --- a/src/python/pants/backend/python/goals/pytest_runner.py +++ b/src/python/pants/backend/python/goals/pytest_runner.py @@ -13,13 +13,7 @@ ) from pants.backend.python.subsystems.pytest import PyTest, PythonTestFieldSet from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.backend.python.util_rules.pex import ( - Pex, - PexRequest, - VenvPex, - VenvPexProcess, - pex_path_closure, -) +from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest from pants.backend.python.util_rules.python_sources import ( PythonSourceFiles, @@ -222,7 +216,7 @@ async def setup_pytest_for_target( interpreter_constraints=interpreter_constraints, main=pytest.main, internal_only=True, - pex_path=pex_path_closure([pytest_pex, requirements_pex]), + pex_path=[pytest_pex, requirements_pex], ), ) config_files_get = Get( diff --git a/src/python/pants/backend/python/goals/repl.py b/src/python/pants/backend/python/goals/repl.py index c0e1a8688cd..8514e0e68f1 100644 --- a/src/python/pants/backend/python/goals/repl.py +++ b/src/python/pants/backend/python/goals/repl.py @@ -2,7 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.python.subsystems.ipython import IPython -from pants.backend.python.util_rules.pex import Pex, PexRequest, pex_path_closure +from pants.backend.python.util_rules.pex import Pex, PexRequest from pants.backend.python.util_rules.pex_environment import PexEnvironment from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest from pants.backend.python.util_rules.python_sources import ( @@ -106,7 +106,7 @@ async def create_ipython_repl_request( chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None), - "PEX_PATH": ":".join(repl.in_chroot(p.name) for p in pex_path_closure([requirements_pex])), + "PEX_PATH": repl.in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } diff --git a/src/python/pants/backend/python/goals/run_pex_binary.py b/src/python/pants/backend/python/goals/run_pex_binary.py index 7cf0c2eb27f..fb68253ad35 100644 --- a/src/python/pants/backend/python/goals/run_pex_binary.py +++ b/src/python/pants/backend/python/goals/run_pex_binary.py @@ -9,7 +9,7 @@ ResolvedPexEntryPoint, ResolvePexEntryPointRequest, ) -from pants.backend.python.util_rules.pex import Pex, PexRequest, pex_path_closure +from pants.backend.python.util_rules.pex import Pex, PexRequest from pants.backend.python.util_rules.pex_environment import PexEnvironment from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest from pants.backend.python.util_rules.python_sources import ( @@ -94,7 +94,7 @@ def in_chroot(relpath: str) -> str: chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=runner_pex.python is not None), - "PEX_PATH": ":".join(in_chroot(p.name) for p in pex_path_closure([requirements])), + "PEX_PATH": in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } diff --git a/src/python/pants/backend/python/goals/run_pex_binary_integration_test.py b/src/python/pants/backend/python/goals/run_pex_binary_integration_test.py index affa587a819..a3358642aa0 100644 --- a/src/python/pants/backend/python/goals/run_pex_binary_integration_test.py +++ b/src/python/pants/backend/python/goals/run_pex_binary_integration_test.py @@ -88,7 +88,6 @@ def run(*extra_args: str, **extra_env: str) -> PantsResult: result = run("--", "info", PEX_TOOLS="1") assert result.exit_code == 0 pex_info = json.loads(result.stdout) - assert (execution_mode is PexExecutionMode.UNZIP) == pex_info["unzip"] assert (execution_mode is PexExecutionMode.VENV) == pex_info["venv"] assert ("prepend" if execution_mode is PexExecutionMode.VENV else "false") == pex_info[ "venv_bin_path" diff --git a/src/python/pants/backend/python/lint/pylint/rules.py b/src/python/pants/backend/python/lint/pylint/rules.py index d2015e9635a..75db43fcb53 100644 --- a/src/python/pants/backend/python/lint/pylint/rules.py +++ b/src/python/pants/backend/python/lint/pylint/rules.py @@ -13,13 +13,7 @@ from pants.backend.python.target_types import InterpreterConstraintsField from pants.backend.python.util_rules import pex_from_targets from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.backend.python.util_rules.pex import ( - Pex, - PexRequest, - VenvPex, - VenvPexProcess, - pex_path_closure, -) +from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest from pants.backend.python.util_rules.python_sources import ( PythonSourceFiles, @@ -134,7 +128,7 @@ async def pylint_lint_partition( interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, - pex_path=pex_path_closure([pylint_pex, requirements_pex]), + pex_path=[pylint_pex, requirements_pex], ), ), Get( diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index 6c6052fc081..7eff4f62d1c 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from enum import Enum from textwrap import dedent -from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Optional, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, Optional, Tuple, Union, cast from packaging.utils import canonicalize_name as canonicalize_project_name from pkg_resources import Requirement @@ -20,6 +20,7 @@ DEFAULT_TYPE_STUB_MODULE_MAPPING, ) from pants.backend.python.macros.python_artifact import PythonArtifact +from pants.base.deprecated import warn_or_error from pants.core.goals.package import OutputPathField from pants.core.goals.test import RuntimePackageDependenciesField from pants.engine.addresses import Address, Addresses @@ -281,6 +282,7 @@ def compute_value( return super().compute_value(raw_value, address) +# TODO(John Sirois): Deprecate: https://github.com/pantsbuild/pants/issues/12803 class PexZipSafeField(BoolField): alias = "zip_safe" default = True @@ -289,6 +291,11 @@ class PexZipSafeField(BoolField): "not zip safe, it will be written to disk prior to execution. You may need to mark " "`zip_safe=False` if you're having issues loading your code." ) + removal_version = "2.9.0.dev0" + removal_hint = ( + "All PEX binaries now unpack your code to disk prior to first execution; so this option no " + "longer needs to be specified." + ) class PexStripEnvField(BoolField): @@ -311,6 +318,12 @@ class PexAlwaysWriteCacheField(BoolField): "Whether PEX should always write the .deps cache of the .pex file to disk or not. This " "can use less memory in RAM-constrained environments." ) + removal_version = "2.9.0.dev0" + removal_hint = ( + "This option never had any effect when passed to Pex and the Pex option is now removed " + "altogether. PEXes always write all their internal dependencies out to disk as part of " + "first execution bootstrapping." + ) class PexIgnoreErrorsField(BoolField): @@ -354,18 +367,33 @@ class PexExecutionModeField(StringField): expected_type = str default = PexExecutionMode.ZIPAPP.value help = ( - "The mode the generated PEX file will run in.\n\nThe traditional PEX file runs in " - f"{PexExecutionMode.ZIPAPP.value!r} mode (See: https://www.python.org/dev/peps/pep-0441/). " - f"In general, faster cold start times can be attained using the " - f"{PexExecutionMode.UNZIP.value!r} mode which also has the benefit of allowing standard " - "use of `__file__` and filesystem APIs to access code and resources in the PEX.\n\nThe " - f"fastest execution mode in the steady state is {PexExecutionMode.VENV.value!r}, which " - "generates a virtual environment from the PEX file on first run, but then achieves near " - "native virtual environment start times. This mode also benefits from a traditional " - "virtual environment `sys.path`, giving maximum compatibility with stdlib and third party " - "APIs." + "The mode the generated PEX file will run in.\n\nThe traditional PEX file runs in a " + f"modified {PexExecutionMode.ZIPAPP.value!r} mode (See: " + "https://www.python.org/dev/peps/pep-0441/) where zipped internal code and dependencies " + "are first unpacked to disk. This mode achieves the fastest cold start times and may, for " + "example be the best choice for cloud lambda functions.\n\nThe fastest execution mode in " + f"the steady state is {PexExecutionMode.VENV.value!r}, which generates a virtual " + "environment from the PEX file on first run, but then achieves near native virtual " + "environment start times. This mode also benefits from a traditional virtual environment " + "`sys.path`, giving maximum compatibility with stdlib and third party APIs.\n\nThe " + f"{PexExecutionMode.UNZIP.value!r} mode is deprecated since the default " + f"{PexExecutionMode.ZIPAPP.value!r} mode now executes this way." ) + @classmethod + def _check_deprecated(cls, raw_value: Optional[Any], address_: Address) -> None: + if PexExecutionMode.UNZIP.value == raw_value: + warn_or_error( + removal_version="2.9.0.dev0", + entity=f"the {cls.alias!r} field {PexExecutionMode.UNZIP.value!r} value", + hint=( + f"The {PexExecutionMode.UNZIP.value!r} mode is now the default PEX execution " + "mode; so you can remove this field setting or explicitly choose the default " + f"of {PexExecutionMode.ZIPAPP.value!r} and get the same benefits you already " + "enjoy from this mode." + ), + ) + class PexIncludeToolsField(BoolField): alias = "include_tools" diff --git a/src/python/pants/backend/python/typecheck/mypy/rules.py b/src/python/pants/backend/python/typecheck/mypy/rules.py index 029de7df82f..6c2a91c9fb3 100644 --- a/src/python/pants/backend/python/typecheck/mypy/rules.py +++ b/src/python/pants/backend/python/typecheck/mypy/rules.py @@ -15,13 +15,7 @@ ) from pants.backend.python.util_rules import pex_from_targets from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.backend.python.util_rules.pex import ( - Pex, - PexRequest, - VenvPex, - VenvPexProcess, - pex_path_closure, -) +from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest from pants.backend.python.util_rules.python_sources import ( PythonSourceFiles, @@ -178,7 +172,7 @@ async def mypy_typecheck_partition( PexRequest( output_filename="requirements_venv.pex", internal_only=True, - pex_path=pex_path_closure([requirements_pex]), + pex_path=[requirements_pex], interpreter_constraints=partition.interpreter_constraints, ), ) diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py index 1bb61a6b3e7..65e3fad3595 100644 --- a/src/python/pants/backend/python/util_rules/pex.py +++ b/src/python/pants/backend/python/util_rules/pex.py @@ -8,7 +8,6 @@ import logging import os import shlex -from collections import deque from dataclasses import dataclass from pathlib import PurePath from textwrap import dedent @@ -55,15 +54,15 @@ ProcessCacheScope, ProcessResult, ) -from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.rules import Get, collect_rules, rule from pants.python.python_repos import PythonRepos from pants.python.python_setup import InvalidLockfileBehavior, PythonSetup from pants.util.docutil import doc_url from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.meta import frozen_after_init -from pants.util.ordered_set import FrozenOrderedSet, OrderedSet -from pants.util.strutil import path_safe, pluralize +from pants.util.ordered_set import FrozenOrderedSet +from pants.util.strutil import pluralize @dataclass(frozen=True) @@ -100,26 +99,18 @@ class ToolCustomLockfile(Lockfile, _ToolLockfileMixin): @dataclass(unsafe_hash=True) class PexRequirements: req_strings: FrozenOrderedSet[str] - apply_constraints: bool - resolved_dists: ResolvedDistributions | None + repository_pex: Pex | None def __init__( - self, - req_strings: Iterable[str] = (), - *, - apply_constraints: bool = False, - resolved_dists: ResolvedDistributions | None = None, + self, req_strings: Iterable[str] = (), *, repository_pex: Pex | None = None ) -> None: """ :param req_strings: The requirement strings to resolve. - :param apply_constraints: Whether to apply any configured - requirement_constraints while building this PEX. - :param resolved_dists: An optional ResolvedDistributions instance containing the - closed universe of wheels that this PEX should be built from.. + :param repository_pex: An optional PEX to resolve requirements from via the Pex CLI + `--pex-repository` option. """ self.req_strings = FrozenOrderedSet(sorted(req_strings)) - self.apply_constraints = apply_constraints - self.resolved_dists = resolved_dists + self.repository_pex = repository_pex @classmethod def create_from_requirement_fields( @@ -127,12 +118,9 @@ def create_from_requirement_fields( fields: Iterable[PythonRequirementsField], *, additional_requirements: Iterable[str] = (), - apply_constraints: bool = True, ) -> PexRequirements: field_requirements = {str(python_req) for field in fields for python_req in field.value} - return PexRequirements( - {*field_requirements, *additional_requirements}, apply_constraints=apply_constraints - ) + return PexRequirements({*field_requirements, *additional_requirements}) def __bool__(self) -> bool: return bool(self.req_strings) @@ -166,6 +154,7 @@ class PexRequest(EngineAwareParameter): main: MainSpecification | None additional_args: Tuple[str, ...] pex_path: Tuple[Pex, ...] + apply_requirement_constraints: bool description: str | None = dataclasses.field(compare=False) def __init__( @@ -182,6 +171,7 @@ def __init__( main: MainSpecification | None = None, additional_args: Iterable[str] = (), pex_path: Iterable[Pex] = (), + apply_requirement_constraints: bool = False, description: str | None = None, ) -> None: """A request to create a PEX from its inputs. @@ -206,6 +196,8 @@ def __init__( left off, the Pex will open up as a REPL. :param additional_args: Any additional Pex flags. :param pex_path: Pex files to add to the PEX_PATH. + :param apply_requirement_constraints: Whether to apply any configured + requirement_constraints while building this PEX. :param description: A human-readable description to render in the dynamic UI when building the Pex. """ @@ -220,6 +212,7 @@ def __init__( self.main = main self.additional_args = tuple(additional_args) self.pex_path = tuple(pex_path) + self.apply_requirement_constraints = apply_requirement_constraints self.description = description self.__post_init__() @@ -252,25 +245,11 @@ class Pex: digest: Digest name: str python: PythonExecutable | None - pex_path: Tuple[Pex, ...] logger = logging.getLogger(__name__) -def pex_path_closure(pexes: Iterable[Pex]) -> OrderedSet[Pex]: - """Return all distinct Pex files in the transitive pex_path of the given Pexes.""" - output: OrderedSet[Pex] = OrderedSet() - to_visit = deque(pexes) - while to_visit: - pex = to_visit.popleft() - if pex in output: - continue - output.add(pex) - to_visit.extend(pex.pex_path) - return output - - @rule(desc="Find Python interpreter for constraints", level=LogLevel.DEBUG) async def find_interpreter( interpreter_constraints: InterpreterConstraints, pex_runtime_env: PexRuntimeEnvironment @@ -333,90 +312,29 @@ class BuildPexResult: pex_filename: str digest: Digest python: PythonExecutable | None - pex_path: Tuple[Pex, ...] def create_pex(self) -> Pex: - return Pex( - digest=self.digest, name=self.pex_filename, python=self.python, pex_path=self.pex_path - ) - - -@dataclass(frozen=True) -class BuildPexComponentResult: - """A wrapper around BuildPexResult to enable iterativately building a PEX from multiple PEXes. - - TODO: The `BuildPexResult` rule is not able to recurse on itself due to a bad @rule graph - interplay with the mypy+protobuf rules (which request a PEX during the generation of sources). - So instead, this rule adjusts the PexRequest and requests the dependencies first. See if this - trampoline can be removed once https://github.com/pantsbuild/pants/issues/11269 is fixed. - """ - - result: BuildPexResult + return Pex(digest=self.digest, name=self.pex_filename, python=self.python) @rule(level=LogLevel.DEBUG) async def build_pex( request: PexRequest, -) -> BuildPexResult: - # If there are requirements and we're resolving from ResolvedDistributions, request - # individual PEX files for each requirement, and then compose them using the - # PEX_PATH. This is much friendlier to the cache, because unlike a monolithic PEX, - # per-requirement PEX files can be deduped in the CAS across many consumers. - # - # TODO: Note that due to https://github.com/pantsbuild/pex/issues/1423, the PEX files - # resolved here are each transitive, meaning that when the root requirements have - # overlapping transitive dependencies, the PEXes will contain redundant-but-identical - # content. This is still much less redundant than a direct subset though: - # see https://github.com/pantsbuild/pants/issues/12688 - reqs = request.requirements - if ( - request.internal_only - and isinstance(reqs, PexRequirements) - and reqs.resolved_dists - and reqs.req_strings - ): - partial_results = await MultiGet( - Get( - BuildPexComponentResult, - PexRequest, - dataclasses.replace( - request, - requirements=dataclasses.replace( - request.requirements, req_strings=(req_string,) - ), - output_filename=f"__reqs/{path_safe(req_string)}.pex", - ), - ) - for req_string in reqs.req_strings - ) - request = dataclasses.replace( - request, - requirements=dataclasses.replace(request.requirements, req_strings=()), - pex_path=request.pex_path + tuple(p.result.create_pex() for p in partial_results), - ) - - partial = await Get(BuildPexComponentResult, PexRequest, request) - return partial.result - - -@rule(level=LogLevel.DEBUG) -async def build_pex_component( - request: PexRequest, python_setup: PythonSetup, python_repos: PythonRepos, platform: Platform, pex_runtime_env: PexRuntimeEnvironment, -) -> BuildPexComponentResult: +) -> BuildPexResult: """Returns a PEX with the given settings.""" argv = ["--output-file", request.output_filename, *request.additional_args] - resolved_dists = ( - request.requirements.resolved_dists + repository_pex = ( + request.requirements.repository_pex if isinstance(request.requirements, PexRequirements) else None ) - if resolved_dists: - argv.extend(["--pex-repository", resolved_dists.pex.name]) + if repository_pex: + argv.extend(["--pex-repository", repository_pex.name]) else: # NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes` # including PyPI, which will override `--no-pypi` and result in using PyPI in the default @@ -432,8 +350,11 @@ async def build_pex_component( ] ) + is_lockfile = isinstance(request.requirements, (Lockfile, LockfileContent)) + if is_lockfile: + argv.append("--no-transitive") + python: PythonExecutable | None = None - pex_path = list(request.pex_path) # NB: If `--platform` is specified, this signals that the PEX should not be built locally. # `--interpreter-constraint` only makes sense in the context of building locally. These two @@ -472,6 +393,12 @@ async def build_pex_component( if request.main is not None: argv.extend(request.main.iter_pex_args()) + # TODO(John Sirois): Right now any request requirements will shadow corresponding pex path + # requirements, which could lead to problems. Support shading python binaries. + # See: https://github.com/pantsbuild/pants/issues/9206 + if request.pex_path: + argv.extend(["--pex-path", ":".join(pex.name for pex in request.pex_path)]) + source_dir_name = "source_files" argv.append(f"--sources-directory={source_dir_name}") sources_digest_as_subdir = await Get( @@ -479,8 +406,22 @@ async def build_pex_component( ) additional_inputs_digest = request.additional_inputs or EMPTY_DIGEST - resolved_dists_digest = resolved_dists.pex.digest if resolved_dists else EMPTY_DIGEST + repository_pex_digest = repository_pex.digest if repository_pex else EMPTY_DIGEST + constraint_file_digest = EMPTY_DIGEST + if ( + not is_lockfile and request.apply_requirement_constraints + ) and python_setup.requirement_constraints is not None: + argv.extend(["--constraints", python_setup.requirement_constraints]) + constraint_file_digest = await Get( + Digest, + PathGlobs( + [python_setup.requirement_constraints], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin="the option `[python-setup].requirement_constraints`", + ), + ) + requirements_file_digest = EMPTY_DIGEST # TODO(#12314): Capture the resolve name for multiple user lockfiles. @@ -492,7 +433,6 @@ async def build_pex_component( if isinstance(request.requirements, Lockfile): argv.extend(["--requirement", request.requirements.file_path]) - argv.append("--no-transitive") globs = PathGlobs( [request.requirements.file_path], @@ -514,38 +454,14 @@ async def build_pex_component( elif isinstance(request.requirements, LockfileContent): file_content = request.requirements.file_content argv.extend(["--requirement", file_content.path]) - argv.append("--no-transitive") metadata = LockfileMetadata.from_lockfile(file_content.content, resolve_name=resolve_name) _validate_metadata(metadata, request, request.requirements, python_setup) requirements_file_digest = await Get(Digest, CreateDigest([file_content])) else: - assert isinstance(request.requirements, PexRequirements) - - # If constraints should be applied and are set, capture them. - if ( - request.requirements.apply_constraints - and python_setup.requirement_constraints is not None - ): - argv.extend(["--constraints", python_setup.requirement_constraints]) - constraint_file_digest = await Get( - Digest, - PathGlobs( - [python_setup.requirement_constraints], - glob_match_error_behavior=GlobMatchErrorBehavior.error, - description_of_origin="the option `[python-setup].requirement_constraints`", - ), - ) - argv.extend(request.requirements.req_strings) - # TODO(John Sirois): Right now any request requirements will shadow corresponding pex path - # requirements, which could lead to problems. Support shading python binaries. - # See: https://github.com/pantsbuild/pants/issues/9206 - if pex_path: - argv.extend(["--pex-path", ":".join(pex.name for pex in pex_path_closure(pex_path))]) - merged_digest = await Get( Digest, MergeDigests( @@ -554,12 +470,21 @@ async def build_pex_component( additional_inputs_digest, constraint_file_digest, requirements_file_digest, - resolved_dists_digest, - *(pex.digest for pex in pex_path), + repository_pex_digest, + *(pex.digest for pex in request.pex_path), ) ), ) + output_files: Iterable[str] | None = None + output_directories: Iterable[str] | None = None + if request.internal_only: + # This is a much friendlier layout for the CAS than the default zipapp. + argv.extend(["--layout", "packed"]) + output_directories = [request.output_filename] + else: + output_files = [request.output_filename] + process = await Get( Process, PexCliProcess( @@ -567,7 +492,8 @@ async def build_pex_component( argv=argv, additional_input_digest=merged_digest, description=_build_pex_description(request), - output_files=[request.output_filename], + output_files=output_files, + output_directories=output_directories, ), ) @@ -583,21 +509,14 @@ async def build_pex_component( digest = ( await Get( - Digest, - MergeDigests((result.output_digest, *(pex.digest for pex in pex_path))), + Digest, MergeDigests((result.output_digest, *(pex.digest for pex in request.pex_path))) ) - if pex_path + if request.pex_path else result.output_digest ) - return BuildPexComponentResult( - BuildPexResult( - result=result, - pex_filename=request.output_filename, - digest=digest, - python=python, - pex_path=tuple(pex_path), - ) + return BuildPexResult( + result=result, pex_filename=request.output_filename, digest=digest, python=python ) @@ -704,27 +623,26 @@ def _build_pex_description(request: PexRequest) -> str: if request.description: return request.description - reqs = request.requirements - if isinstance(reqs, Lockfile): - return f"Resolving {request.output_filename} from {reqs.file_path}" - elif isinstance(reqs, LockfileContent): - return f"Resolving {request.output_filename} from {reqs.file_content.path}" - elif request.internal_only and reqs.resolved_dists: - repo_pex = reqs.resolved_dists.pex - if reqs.req_strings: - return f"Extracting {', '.join(reqs.req_strings)} from {repo_pex.name}" - else: + if isinstance(request.requirements, Lockfile): + desc_suffix = f"from {request.requirements.file_path}" + elif isinstance(request.requirements, LockfileContent): + desc_suffix = f"from {request.requirements.file_content.path}" + else: + if not request.requirements.req_strings: + return f"Building {request.output_filename}" + elif request.requirements.repository_pex: + repo_pex = request.requirements.repository_pex.name return ( - f"Composing {pluralize(len(request.pex_path), 'requirement')} to build " - f"{request.output_filename} from {repo_pex.name}" + f"Extracting {pluralize(len(request.requirements.req_strings), 'requirement')} " + f"to build {request.output_filename} from {repo_pex}: " + f"{', '.join(request.requirements.req_strings)}" ) - elif not reqs.req_strings: - return f"Building {request.output_filename}" - else: - return ( - f"Building {request.output_filename} with " - f"{pluralize(len(reqs.req_strings), 'requirement')}: {', '.join(reqs.req_strings)}" - ) + else: + desc_suffix = ( + f"with {pluralize(len(request.requirements.req_strings), 'requirement')}: " + f"{', '.join(request.requirements.req_strings)}" + ) + return f"Building {request.output_filename} {desc_suffix}" @rule @@ -898,10 +816,10 @@ async def create_venv_pex( # file startup overhead. # # To achieve the minimal overhead (on the order of 1ms) we discard: - # 1. Using Pex `--unzip` mode: - # Although this does reduce steady-state overhead, it still leaves a minimum O(100ms) of - # overhead per tool invocation. Fundamentally, Pex still needs to execute its `sys.path` - # isolation bootstrap code in this case. + # 1. Using Pex default mode: + # Although this does reduce initial tool execution overhead, it still leaves a minimum + # O(100ms) of overhead per subsequent tool invocation. Fundamentally, Pex still needs to + # execute its `sys.path` isolation bootstrap code in this case. # 2. Using the Pex `venv` tool: # The idea here would be to create a tool venv as a Process output and then use the tool # venv as an input digest for all tool invocations. This was tried and netted ~500ms of @@ -1175,9 +1093,7 @@ async def determine_pex_resolve_info(pex_pex: PexPEX, pex: Pex) -> PexResolveInf process_result = await Get( ProcessResult, PexProcess( - pex=Pex( - digest=pex_pex.digest, name=pex_pex.exe, python=pex.python, pex_path=pex.pex_path - ), + pex=Pex(digest=pex_pex.digest, name=pex_pex.exe, python=pex.python), argv=[pex.name, "repository", "info", "-v"], input_digest=pex.digest, extra_env={"PEX_MODULE": "pex.tools"}, @@ -1188,28 +1104,5 @@ async def determine_pex_resolve_info(pex_pex: PexPEX, pex: Pex) -> PexResolveInf return parse_repository_info(process_result.stdout.decode()) -@dataclass(frozen=True) -class ResolvedDistributions: - """A 'repository' pex, containing the entire contents of the resolve for multiple libraries. - - Generally constructed from a lockfile. - """ - - pex: Pex - - -@rule -async def resolve(request: PexRequest, platform: Platform) -> ResolvedDistributions: - # Build the repository PEX. - request = dataclasses.replace( - request, additional_args=(*request.additional_args, "--include-tools") - ) - pex = await Get(Pex, PexRequest, request) - - # TODO: extract the graph. - - return ResolvedDistributions(pex) - - def rules(): return [*collect_rules(), *pex_cli.rules()] diff --git a/src/python/pants/backend/python/util_rules/pex_cli.py b/src/python/pants/backend/python/util_rules/pex_cli.py index 70290b121af..15723fa8b79 100644 --- a/src/python/pants/backend/python/util_rules/pex_cli.py +++ b/src/python/pants/backend/python/util_rules/pex_cli.py @@ -37,10 +37,9 @@ class PexBinary(TemplatedExternalTool): name = "pex" help = "The PEX (Python EXecutable) tool (https://github.com/pantsbuild/pex)." - default_version = "v2.1.47" + default_version = "v2.1.48" default_url_template = "https://github.com/pantsbuild/pex/releases/download/{version}/pex" - # N.B.: 2.1.46 contains a regression that breaks Pants usage; so we exclude it. - version_constraints = ">=2.1.42,!=2.1.46,<3.0" + version_constraints = ">=2.1.48,<3.0" @classproperty def default_known_versions(cls): @@ -49,8 +48,8 @@ def default_known_versions(cls): ( cls.default_version, plat, - "d28a3c4dac818709e91dc61272c389a5ba4ce3cb802b9b6d5ad3bc58acfd0b6f", - "3631561", + "4f86bc7e9e852fe59f9bc774d910b60c7f1df5a194ba0bdf491f70ba78a2e6bb", + "3640820", ) ) for plat in ["macos_arm64", "macos_x86_64", "linux_x86_64"] diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets.py b/src/python/pants/backend/python/util_rules/pex_from_targets.py index 139fda7ad65..a7894962219 100644 --- a/src/python/pants/backend/python/util_rules/pex_from_targets.py +++ b/src/python/pants/backend/python/util_rules/pex_from_targets.py @@ -20,10 +20,10 @@ from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.pex import ( Lockfile, + Pex, PexPlatforms, PexRequest, PexRequirements, - ResolvedDistributions, ) from pants.backend.python.util_rules.pex import rules as pex_rules from pants.backend.python.util_rules.python_sources import ( @@ -138,25 +138,17 @@ def for_requirements( *, internal_only: bool, hardcoded_interpreter_constraints: InterpreterConstraints | None = None, - zip_safe: bool = False, direct_deps_only: bool = False, ) -> PexFromTargetsRequest: """Create an instance that can be used to get a requirements pex. Useful to ensure that these requests are uniform (e.g., the using the same output filename), so that the underlying pexes are more likely to be reused instead of re-resolved. - - We default to zip_safe=False because there are various issues with running zipped pexes - directly, and it's best to only use those if you're sure it's the right thing to do. - Also, pytest must use zip_safe=False for performance reasons (see comment in - pytest_runner.py) and we get more re-use of pexes if other uses follow suit. - This default is a helpful nudge in that direction. """ return PexFromTargetsRequest( addresses=sorted(addresses), output_filename="requirements.pex", include_source_files=False, - additional_args=() if zip_safe else ("--not-zip-safe",), hardcoded_interpreter_constraints=hardcoded_interpreter_constraints, internal_only=internal_only, direct_deps_only=direct_deps_only, @@ -164,12 +156,12 @@ def for_requirements( @dataclass(frozen=True) -class _ConstraintsResolvedDistributions: - maybe_resolved_dists: ResolvedDistributions | None +class _ConstraintsRepositoryPex: + maybe_pex: Pex | None @dataclass(frozen=True) -class _ConstraintsResolvedDistributionsRequest: +class _ConstraintsRepositoryPexRequest: requirements: PexRequirements platforms: PexPlatforms interpreter_constraints: InterpreterConstraints @@ -225,11 +217,11 @@ async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonS description = request.description if requirements: - resolved_dists: ResolvedDistributions | None = None + repository_pex: Pex | None = None if python_setup.requirement_constraints: - constraints_resolved_dists = await Get( - _ConstraintsResolvedDistributions, - _ConstraintsResolvedDistributionsRequest( + maybe_constraints_repository_pex = await Get( + _ConstraintsRepositoryPex, + _ConstraintsRepositoryPexRequest( requirements, request.platforms, interpreter_constraints, @@ -237,7 +229,8 @@ async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonS request.additional_args, ), ) - resolved_dists = constraints_resolved_dists.maybe_resolved_dists + if maybe_constraints_repository_pex.maybe_pex: + repository_pex = maybe_constraints_repository_pex.maybe_pex elif ( python_setup.resolve_all_constraints and python_setup.resolve_all_constraints_was_set_explicitly() @@ -247,8 +240,8 @@ async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonS "`[python-setup].requirement_constraints` must also be set." ) elif python_setup.lockfile: - resolved_dists = await Get( - ResolvedDistributions, + repository_pex = await Get( + Pex, PexRequest( description=f"Resolving {python_setup.lockfile}", output_filename="lockfile.pex", @@ -267,7 +260,7 @@ async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonS additional_args=request.additional_args, ), ) - requirements = dataclasses.replace(requirements, resolved_dists=resolved_dists) + requirements = dataclasses.replace(requirements, repository_pex=repository_pex) return PexRequest( output_filename=request.output_filename, @@ -280,17 +273,18 @@ async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonS additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=description, + apply_requirement_constraints=True, ) @rule async def _setup_constraints_repository_pex( - request: _ConstraintsResolvedDistributionsRequest, python_setup: PythonSetup -) -> _ConstraintsResolvedDistributions: + request: _ConstraintsRepositoryPexRequest, python_setup: PythonSetup +) -> _ConstraintsRepositoryPex: # NB: it isn't safe to resolve against the whole constraints file if # platforms are in use. See https://github.com/pantsbuild/pants/issues/12222. if not python_setup.resolve_all_constraints or request.platforms: - return _ConstraintsResolvedDistributions(None) + return _ConstraintsRepositoryPex(None) constraints_path = python_setup.requirement_constraints assert constraints_path is not None @@ -336,7 +330,7 @@ async def _setup_constraints_repository_pex( f"entries for the following requirements: {', '.join(unconstrained_projects)}.\n\n" f"Ignoring `[python_setup].resolve_all_constraints` option." ) - return _ConstraintsResolvedDistributions(None) + return _ConstraintsRepositoryPex(None) # To get a full set of requirements we must add the URL requirements to the # constraints file, since the latter cannot contain URL requirements. @@ -347,19 +341,20 @@ async def _setup_constraints_repository_pex( # all these repository pexes will have identical pinned versions of everything, # this is not a correctness issue, only a performance one. all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)} - resolved_dists = await Get( - ResolvedDistributions, + repository_pex = await Get( + Pex, PexRequest( description=f"Resolving {constraints_path}", output_filename="repository.pex", internal_only=request.internal_only, - requirements=PexRequirements(all_constraints, apply_constraints=True), + requirements=PexRequirements(all_constraints), interpreter_constraints=request.interpreter_constraints, platforms=request.platforms, additional_args=request.additional_args, + apply_requirement_constraints=True, ), ) - return _ConstraintsResolvedDistributions(resolved_dists) + return _ConstraintsRepositoryPex(repository_pex) def rules(): diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py index e1baec7596f..82a65072ae1 100644 --- a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py +++ b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py @@ -31,7 +31,6 @@ def rule_runner() -> RuleRunner: rules=[ *pex_from_targets.rules(), QueryRule(PexRequest, (PexFromTargetsRequest,)), - QueryRule(Pex, (PexFromTargetsRequest,)), ], target_types=[PythonLibrary, PythonRequirementLibrary], ) @@ -213,13 +212,13 @@ def get_pex_request( additional_args=additional_args, ) assert pex_req1.requirements == PexRequirements( - ["foo-bar>=0.1.2", "bar==5.5.5", "baz", url_req], apply_constraints=True + ["foo-bar>=0.1.2", "bar==5.5.5", "baz", url_req] ) pex_req1_direct = get_pex_request( "constraints1.txt", resolve_all_constraints=False, direct_deps_only=True ) - assert pex_req1_direct.requirements == PexRequirements(["baz", url_req], apply_constraints=True) + assert pex_req1_direct.requirements == PexRequirements(["baz", url_req]) pex_req2 = get_pex_request( "constraints1.txt", @@ -229,11 +228,11 @@ def get_pex_request( pex_req2_reqs = pex_req2.requirements assert isinstance(pex_req2_reqs, PexRequirements) assert list(pex_req2_reqs.req_strings) == ["bar==5.5.5", "baz", "foo-bar>=0.1.2", url_req] - assert pex_req2_reqs.resolved_dists is not None - assert not info(rule_runner, pex_req2_reqs.resolved_dists.pex)["strip_pex_env"] - resolved_dists = pex_req2_reqs.resolved_dists + assert pex_req2_reqs.repository_pex is not None + assert not info(rule_runner, pex_req2_reqs.repository_pex)["strip_pex_env"] + repository_pex = pex_req2_reqs.repository_pex assert ["Foo._-BAR==1.0.0", "bar==5.5.5", "baz==2.2.2", "foorl", "qux==3.4.5"] == requirements( - rule_runner, resolved_dists.pex + rule_runner, repository_pex ) pex_req2_direct = get_pex_request( @@ -245,8 +244,8 @@ def get_pex_request( pex_req2_reqs = pex_req2_direct.requirements assert isinstance(pex_req2_reqs, PexRequirements) assert list(pex_req2_reqs.req_strings) == ["baz", url_req] - assert pex_req2_reqs.resolved_dists == resolved_dists - assert not info(rule_runner, pex_req2_reqs.resolved_dists.pex)["strip_pex_env"] + assert pex_req2_reqs.repository_pex == repository_pex + assert not info(rule_runner, pex_req2_reqs.repository_pex)["strip_pex_env"] pex_req3_direct = get_pex_request( "constraints1.txt", @@ -256,9 +255,9 @@ def get_pex_request( pex_req3_reqs = pex_req3_direct.requirements assert isinstance(pex_req3_reqs, PexRequirements) assert list(pex_req3_reqs.req_strings) == ["baz", url_req] - assert pex_req3_reqs.resolved_dists is not None - assert pex_req3_reqs.resolved_dists != resolved_dists - assert info(rule_runner, pex_req3_reqs.resolved_dists.pex)["strip_pex_env"] + assert pex_req3_reqs.repository_pex is not None + assert pex_req3_reqs.repository_pex != repository_pex + assert info(rule_runner, pex_req3_reqs.repository_pex)["strip_pex_env"] with pytest.raises(ExecutionError) as err: get_pex_request(None, resolve_all_constraints=True) @@ -300,54 +299,4 @@ def test_issue_12222(rule_runner: RuleRunner) -> None: ) result = rule_runner.request(PexRequest, [request]) - assert result.requirements == PexRequirements(["foo"], apply_constraints=True) - - -@pytest.mark.parametrize("internal_only", [True, False]) -def test_component_pexes(rule_runner: RuleRunner, internal_only: bool) -> None: - """An internal-only PexFromTargetsRequest with a lockfile produces component PEXes.""" - - rule_runner.write_files( - { - "constraints.txt": dedent( - """ - certifi==2021.5.30 - charset_normalizer==2.0.4 - idna==3.2 - requests==2.26.0 - urllib3==1.26.6 - """ - ), - "BUILD": dedent( - """ - python_requirement_library(name="requests",requirements=["requests"]) - python_library(name="lib",sources=[],dependencies=[":requests"]) - """ - ), - } - ) - request = PexFromTargetsRequest( - [Address("", target_name="lib")], - output_filename="demo.pex", - internal_only=internal_only, - ) - rule_runner.set_options( - [ - "--backend-packages=pants.backend.python", - "--python-setup-requirement-constraints=constraints.txt", - "--python-setup-resolve-all-constraints", - ], - env_inherit={"PATH", "PYENV_ROOT", "HOME"}, - ) - pex_info = info(rule_runner, rule_runner.request(Pex, [request])) - - if internal_only: - # Should have a pex-path containing the root requirement. - assert pex_info["requirements"] == [] - assert pex_info["distributions"] == {} - assert pex_info["pex_path"] == "__reqs/requests.pex" - else: - # Should have a root requirement, and five distributions. - assert pex_info["requirements"] == ["requests"] - assert len(pex_info["distributions"]) == 5 - assert pex_info["pex_path"] is None + assert result.requirements == PexRequirements(["foo"]) diff --git a/src/python/pants/backend/python/util_rules/pex_test.py b/src/python/pants/backend/python/util_rules/pex_test.py index 527f8766c7f..ce3aa294f0c 100644 --- a/src/python/pants/backend/python/util_rules/pex_test.py +++ b/src/python/pants/backend/python/util_rules/pex_test.py @@ -9,7 +9,8 @@ import textwrap import zipfile from dataclasses import dataclass -from typing import Dict, Iterable, Iterator, Mapping, Tuple, cast +from pathlib import PurePath +from typing import Any, Iterable, Iterator, Mapping, Tuple from unittest.mock import MagicMock import pytest @@ -30,7 +31,6 @@ PexRequest, PexRequirements, PexResolveInfo, - ResolvedDistributions, ToolCustomLockfile, ToolDefaultLockfile, VenvPex, @@ -75,12 +75,11 @@ def parse_requirements(requirements: Iterable[str]) -> Iterator[ExactRequirement @pytest.fixture def rule_runner() -> RuleRunner: - rule_runner = RuleRunner( + return RuleRunner( rules=[ *pex_rules(), QueryRule(Pex, (PexRequest,)), QueryRule(VenvPex, (PexRequest,)), - QueryRule(ResolvedDistributions, (PexRequest,)), QueryRule(Process, (PexProcess,)), QueryRule(Process, (VenvPexProcess,)), QueryRule(ProcessResult, (Process,)), @@ -89,11 +88,16 @@ def rule_runner() -> RuleRunner: QueryRule(PexPEX, ()), ], ) - rule_runner.set_options( - ["--backend-packages=pants.backend.python"], - env_inherit={"PATH", "PYENV_ROOT", "HOME"}, - ) - return rule_runner + + +@dataclass(frozen=True) +class PexData: + pex: Pex | VenvPex + is_zipapp: bool + sandbox_path: PurePath + local_path: PurePath + info: Mapping[str, Any] + files: Tuple[str, ...] def create_pex_and_get_all_data( @@ -110,7 +114,7 @@ def create_pex_and_get_all_data( additional_pex_args: Tuple[str, ...] = (), env: Mapping[str, str] | None = None, internal_only: bool = True, -) -> Dict: +) -> PexData: request = PexRequest( output_filename="test.pex", internal_only=internal_only, @@ -121,21 +125,25 @@ def create_pex_and_get_all_data( sources=sources, additional_inputs=additional_inputs, additional_args=additional_pex_args, + apply_requirement_constraints=True, ) rule_runner.set_options( ["--backend-packages=pants.backend.python", *additional_pants_args], env=env, env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) - pex = rule_runner.request(pex_type, [request]) - if isinstance(pex, Pex): + + pex: Pex | VenvPex + if pex_type == Pex: + pex = rule_runner.request(Pex, [request]) digest = pex.digest + sandbox_path = pex.name pex_pex = rule_runner.request(PexPEX, []) process = rule_runner.request( Process, [ PexProcess( - Pex(digest=pex_pex.digest, name=pex_pex.exe, python=pex.python, pex_path=()), + Pex(digest=pex_pex.digest, name=pex_pex.exe, python=pex.python), argv=["-m", "pex.tools", pex.name, "info"], input_digest=pex.digest, extra_env=dict(PEX_INTERPRETER="1"), @@ -143,8 +151,10 @@ def create_pex_and_get_all_data( ) ], ) - elif isinstance(pex, VenvPex): + else: + pex = rule_runner.request(VenvPex, [request]) digest = pex.digest + sandbox_path = pex.pex_filename process = rule_runner.request( Process, [ @@ -156,23 +166,31 @@ def create_pex_and_get_all_data( ), ], ) - else: - raise AssertionError(f"Expected a Pex or a VenvPex but got a {type(pex)}.") rule_runner.scheduler.write_digest(digest) - pex_path = os.path.join(rule_runner.build_root, "test.pex") + local_path = PurePath(rule_runner.build_root) / "test.pex" result = rule_runner.request(ProcessResult, [process]) pex_info_content = result.stdout.decode() - with zipfile.ZipFile(pex_path, "r") as zipfp: - pex_list = zipfp.namelist() + is_zipapp = zipfile.is_zipfile(local_path) + if is_zipapp: + with zipfile.ZipFile(local_path, "r") as zipfp: + files = tuple(zipfp.namelist()) + else: + files = tuple( + os.path.normpath(os.path.relpath(os.path.join(root, path), local_path)) + for root, dirs, files in os.walk(local_path) + for path in dirs + files + ) - return { - "pex": pex, - "local_path": pex_path, - "info": json.loads(pex_info_content), - "files": pex_list, - } + return PexData( + pex=pex, + is_zipapp=is_zipapp, + sandbox_path=PurePath(sandbox_path), + local_path=local_path, + info=json.loads(pex_info_content), + files=files, + ) def create_pex_and_get_pex_info( @@ -187,25 +205,26 @@ def create_pex_and_get_pex_info( additional_pants_args: Tuple[str, ...] = (), additional_pex_args: Tuple[str, ...] = (), internal_only: bool = True, -) -> Dict: - return cast( - Dict, - create_pex_and_get_all_data( - rule_runner, - pex_type=pex_type, - requirements=requirements, - main=main, - interpreter_constraints=interpreter_constraints, - platforms=platforms, - sources=sources, - additional_pants_args=additional_pants_args, - additional_pex_args=additional_pex_args, - internal_only=internal_only, - )["info"], - ) +) -> Mapping[str, Any]: + return create_pex_and_get_all_data( + rule_runner, + pex_type=pex_type, + requirements=requirements, + main=main, + interpreter_constraints=interpreter_constraints, + platforms=platforms, + sources=sources, + additional_pants_args=additional_pants_args, + additional_pex_args=additional_pex_args, + internal_only=internal_only, + ).info -def test_pex_execution(rule_runner: RuleRunner) -> None: +@pytest.mark.parametrize("pex_type", [Pex, VenvPex]) +@pytest.mark.parametrize("internal_only", [True, False]) +def test_pex_execution( + rule_runner: RuleRunner, pex_type: type[Pex | VenvPex], internal_only: bool +) -> None: sources = rule_runner.request( Digest, [ @@ -217,19 +236,29 @@ def test_pex_execution(rule_runner: RuleRunner) -> None: ), ], ) - pex_output = create_pex_and_get_all_data(rule_runner, main=EntryPoint("main"), sources=sources) + pex_data = create_pex_and_get_all_data( + rule_runner, + pex_type=pex_type, + internal_only=internal_only, + main=EntryPoint("main"), + sources=sources, + ) - pex_files = pex_output["files"] - assert "pex" not in pex_files - assert "main.py" in pex_files - assert "subdir/sub.py" in pex_files + assert "pex" not in pex_data.files + assert "main.py" in pex_data.files + assert "subdir/sub.py" in pex_data.files - # This should run the Pex using the same interpreter used to create it. We must set the `PATH` so that the shebang - # works. + # This should run the Pex using the same interpreter used to create it. We must set the `PATH` + # so that the shebang works. + pex_exe = ( + f"./{pex_data.sandbox_path}" + if pex_data.is_zipapp + else os.path.join(pex_data.sandbox_path, "__main__.py") + ) process = Process( - argv=("./test.pex",), + argv=(pex_exe,), env={"PATH": os.getenv("PATH", "")}, - input_digest=pex_output["pex"].digest, + input_digest=pex_data.pex.digest, description="Run the pex and make sure it works", ) result = rule_runner.request(ProcessResult, [process]) @@ -257,7 +286,7 @@ def test_pex_environment(rule_runner: RuleRunner, pex_type: type[Pex | VenvPex]) ), ], ) - pex_output = create_pex_and_get_all_data( + pex_data = create_pex_and_get_all_data( rule_runner, pex_type=pex_type, main=EntryPoint("main"), @@ -270,13 +299,12 @@ def test_pex_environment(rule_runner: RuleRunner, pex_type: type[Pex | VenvPex]) env={"LANG": "es_PY.UTF-8"}, ) - pex = pex_output["pex"] - pex_process_type = PexProcess if isinstance(pex, Pex) else VenvPexProcess + pex_process_type = PexProcess if isinstance(pex_data.pex, Pex) else VenvPexProcess process = rule_runner.request( Process, [ pex_process_type( - pex, + pex_data.pex, description="Run the pex and check its reported environment", ), ], @@ -312,7 +340,7 @@ def test_pex_working_directory(rule_runner: RuleRunner, pex_type: type[Pex | Ven ], ) - pex_output = create_pex_and_get_all_data( + pex_data = create_pex_and_get_all_data( rule_runner, pex_type=pex_type, main=EntryPoint("main"), @@ -320,8 +348,7 @@ def test_pex_working_directory(rule_runner: RuleRunner, pex_type: type[Pex | Ven interpreter_constraints=InterpreterConstraints(["CPython>=3.6"]), ) - pex = pex_output["pex"] - pex_process_type = PexProcess if isinstance(pex, Pex) else VenvPexProcess + pex_process_type = PexProcess if isinstance(pex_data.pex, Pex) else VenvPexProcess dirpath = "foo/bar/baz" runtime_files = rule_runner.request(Digest, [CreateDigest([Directory(path=dirpath)])]) @@ -334,7 +361,7 @@ def test_pex_working_directory(rule_runner: RuleRunner, pex_type: type[Pex | Ven Process, [ pex_process_type( - pex, + pex_data.pex, description="Run the pex and check its cwd", working_directory=working_dir, input_digest=runtime_files, @@ -347,7 +374,7 @@ def test_pex_working_directory(rule_runner: RuleRunner, pex_type: type[Pex | Ven # For VenvPexes, run the PEX twice while clearing the venv dir in between. This emulates # situations where a PEX creation hits the process cache, while venv seeding misses the PEX # cache. - if isinstance(pex, VenvPex): + if isinstance(pex_data.pex, VenvPex): # Request once to ensure that the directory is seeded, and then start a new session so that # the second run happens as well. _ = rule_runner.request(ProcessResult, [process]) @@ -360,7 +387,7 @@ def test_pex_working_directory(rule_runner: RuleRunner, pex_type: type[Pex | Ven named_caches_dir = ( rule_runner.options_bootstrapper.bootstrap_options.for_global_scope().named_caches_dir ) - venv_dir = os.path.join(named_caches_dir, "pex_root", pex.venv_rel_dir) + venv_dir = os.path.join(named_caches_dir, "pex_root", pex_data.pex.venv_rel_dir) assert os.path.isdir(venv_dir) safe_rmtree(venv_dir) @@ -396,7 +423,7 @@ def assert_direct_requirements(pex_info): # Unconstrained, we should always pick the top of the range (requests 2.23.0) since the top of # the range is a transitive closure over universal wheels. direct_pex_info = create_pex_and_get_pex_info( - rule_runner, requirements=PexRequirements(direct_deps, apply_constraints=False) + rule_runner, requirements=PexRequirements(direct_deps) ) assert_direct_requirements(direct_pex_info) assert "requests-2.23.0-py2.py3-none-any.whl" in set(direct_pex_info["distributions"].keys()) @@ -411,7 +438,7 @@ def assert_direct_requirements(pex_info): rule_runner.create_file("constraints.txt", "\n".join(constraints)) constrained_pex_info = create_pex_and_get_pex_info( rule_runner, - requirements=PexRequirements(direct_deps, apply_constraints=True), + requirements=PexRequirements(direct_deps), additional_pants_args=("--python-setup-requirement-constraints=constraints.txt",), ) assert_direct_requirements(constrained_pex_info) @@ -439,8 +466,8 @@ def test_interpreter_constraints(rule_runner: RuleRunner) -> None: def test_additional_args(rule_runner: RuleRunner) -> None: - pex_info = create_pex_and_get_pex_info(rule_runner, additional_pex_args=("--not-zip-safe",)) - assert pex_info["zip_safe"] is False + pex_info = create_pex_and_get_pex_info(rule_runner, additional_pex_args=("--no-strip-pex-env",)) + assert pex_info["strip_pex_env"] is False def test_platforms(rule_runner: RuleRunner) -> None: @@ -448,7 +475,7 @@ def test_platforms(rule_runner: RuleRunner) -> None: # actually used. platforms = PexPlatforms(["linux-x86_64-cp-27-cp27mu"]) constraints = InterpreterConstraints(["CPython>=2.7,<3", "CPython>=3.6"]) - pex_output = create_pex_and_get_all_data( + pex_data = create_pex_and_get_all_data( rule_runner, requirements=PexRequirements(["cryptography==2.9"]), platforms=platforms, @@ -456,31 +483,49 @@ def test_platforms(rule_runner: RuleRunner) -> None: internal_only=False, # Internal only PEXes do not support (foreign) platforms. ) assert any( - "cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl" in fp for fp in pex_output["files"] + "cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl" in fp for fp in pex_data.files ) - assert not any("cryptography-2.9-cp27-cp27m-" in fp for fp in pex_output["files"]) - assert not any("cryptography-2.9-cp35-abi3" in fp for fp in pex_output["files"]) + assert not any("cryptography-2.9-cp27-cp27m-" in fp for fp in pex_data.files) + assert not any("cryptography-2.9-cp35-abi3" in fp for fp in pex_data.files) # NB: Platforms override interpreter constraints. - assert pex_output["info"]["interpreter_constraints"] == [] + assert pex_data.info["interpreter_constraints"] == [] -def test_additional_inputs(rule_runner: RuleRunner) -> None: - # We use pex's --preamble-file option to set a custom preamble from a file. +@pytest.mark.parametrize("pex_type", [Pex, VenvPex]) +@pytest.mark.parametrize("internal_only", [True, False]) +def test_additional_inputs( + rule_runner: RuleRunner, pex_type: type[Pex | VenvPex], internal_only: bool +) -> None: + # We use Pex's --sources-directory option to add an extra source file to the PEX. # This verifies that the file was indeed provided as additional input to the pex call. - preamble_file = "custom_preamble.txt" - preamble = "#!CUSTOM PREAMBLE\n" + extra_src_dir = "extra_src" + data_file = os.path.join("data", "file") + data = "42" additional_inputs = rule_runner.request( - Digest, [CreateDigest([FileContent(path=preamble_file, content=preamble.encode())])] + Digest, + [ + CreateDigest( + [FileContent(path=os.path.join(extra_src_dir, data_file), content=data.encode())] + ) + ], ) - additional_pex_args = (f"--preamble-file={preamble_file}",) - pex_output = create_pex_and_get_all_data( - rule_runner, additional_inputs=additional_inputs, additional_pex_args=additional_pex_args + additional_pex_args = ("--sources-directory", extra_src_dir) + pex_data = create_pex_and_get_all_data( + rule_runner, + pex_type=pex_type, + internal_only=internal_only, + additional_inputs=additional_inputs, + additional_pex_args=additional_pex_args, ) - with zipfile.ZipFile(pex_output["local_path"], "r") as zipfp: - with zipfp.open("__main__.py", "r") as main: - main_content = main.read().decode() - assert main_content[: len(preamble)] == preamble + if pex_data.is_zipapp: + with zipfile.ZipFile(pex_data.local_path, "r") as zipfp: + with zipfp.open(data_file, "r") as datafp: + data_file_content = datafp.read() + else: + with open(pex_data.local_path / data_file, "rb") as datafp: + data_file_content = datafp.read() + assert data == data_file_content.decode() @pytest.mark.parametrize("pex_type", [Pex, VenvPex]) @@ -493,13 +538,13 @@ def test_venv_pex_resolve_info(rule_runner: RuleRunner, pex_type: type[Pex | Ven "urllib3==1.25.11", ] rule_runner.create_file("constraints.txt", "\n".join(constraints)) - venv_pex = create_pex_and_get_all_data( + pex = create_pex_and_get_all_data( rule_runner, pex_type=pex_type, - requirements=PexRequirements(["requests==2.23.0"], apply_constraints=True), + requirements=PexRequirements(["requests==2.23.0"]), additional_pants_args=("--python-setup-requirement-constraints=constraints.txt",), - )["pex"] - dists = rule_runner.request(PexResolveInfo, [venv_pex]) + ).pex + dists = rule_runner.request(PexResolveInfo, [pex]) assert dists[0] == PexDistributionInfo("certifi", Version("2020.12.5"), None, ()) assert dists[1] == PexDistributionInfo("chardet", Version("3.0.4"), None, ()) assert dists[2] == PexDistributionInfo( @@ -515,7 +560,6 @@ def test_build_pex_description() -> None: def assert_description( requirements: PexRequirements | Lockfile | LockfileContent, *, - pex_path_length: int = 0, description: str | None = None, expected: str, ) -> None: @@ -524,32 +568,25 @@ def assert_description( internal_only=True, requirements=requirements, description=description, - pex_path=(Pex(EMPTY_DIGEST, f"{i}.pex", None, ()) for i in range(0, pex_path_length)), ) assert _build_pex_description(request) == expected - resolved_dists = ResolvedDistributions( - Pex(digest=EMPTY_DIGEST, name="repo.pex", python=None, pex_path=()) - ) + repo_pex = Pex(EMPTY_DIGEST, "repo.pex", None) assert_description(PexRequirements(), description="Custom!", expected="Custom!") assert_description( - PexRequirements(resolved_dists=resolved_dists), description="Custom!", expected="Custom!" + PexRequirements(repository_pex=repo_pex), description="Custom!", expected="Custom!" ) assert_description(PexRequirements(), expected="Building new.pex") - assert_description( - PexRequirements(resolved_dists=resolved_dists), - pex_path_length=2, - expected="Composing 2 requirements to build new.pex from repo.pex", - ) + assert_description(PexRequirements(repository_pex=repo_pex), expected="Building new.pex") assert_description( PexRequirements(["req"]), expected="Building new.pex with 1 requirement: req" ) assert_description( - PexRequirements(["req"], resolved_dists=resolved_dists), - expected="Extracting req from repo.pex", + PexRequirements(["req"], repository_pex=repo_pex), + expected="Extracting 1 requirement to build new.pex from repo.pex: req", ) assert_description( @@ -557,8 +594,8 @@ def assert_description( expected="Building new.pex with 2 requirements: req1, req2", ) assert_description( - PexRequirements(["req1"], resolved_dists=resolved_dists), - expected="Extracting req1 from repo.pex", + PexRequirements(["req1", "req2"], repository_pex=repo_pex), + expected="Extracting 2 requirements to build new.pex from repo.pex: req1, req2", ) assert_description( @@ -566,14 +603,14 @@ def assert_description( file_content=FileContent("lock.txt", b""), lockfile_hex_digest=None, ), - expected="Resolving new.pex from lock.txt", + expected="Building new.pex from lock.txt", ) assert_description( Lockfile( file_path="lock.txt", file_path_description_of_origin="foo", lockfile_hex_digest=None ), - expected="Resolving new.pex from lock.txt", + expected="Building new.pex from lock.txt", ) diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index 0b0cd992d1b..c4a24c66dfe 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -20,7 +20,6 @@ pex_binary( dependencies=[ ':pants_loader', ], - execution_mode="unzip", strip_pex_env=False, ) diff --git a/src/python/pants/init/plugin_resolver.py b/src/python/pants/init/plugin_resolver.py index 26d004391fa..d7837eb2f66 100644 --- a/src/python/pants/init/plugin_resolver.py +++ b/src/python/pants/init/plugin_resolver.py @@ -51,9 +51,7 @@ async def resolve_plugins( `named_caches` directory), but consequently needs to disable the process cache: see the ProcessCacheScope reference in the body. """ - # The repository's constraints are not relevant here, because this resolve is mixed - # into the Pants' process' path, and never into user code. - requirements = PexRequirements(sorted(global_options.options.plugins), apply_constraints=False) + requirements = PexRequirements(sorted(global_options.options.plugins)) if not requirements: return ResolvedPluginDistributions() @@ -74,6 +72,9 @@ async def resolve_plugins( python=python, requirements=requirements, interpreter_constraints=request.interpreter_constraints, + # The repository's constraints are not relevant here, because this resolve is mixed + # into the Pants' process' path, and never into user code. + apply_requirement_constraints=False, description=f"Resolving plugins: {', '.join(requirements.req_strings)}", ), ) diff --git a/tests/python/pants_test/init/test_plugin_resolver.py b/tests/python/pants_test/init/test_plugin_resolver.py index de5dbc32045..3867cfa328c 100644 --- a/tests/python/pants_test/init/test_plugin_resolver.py +++ b/tests/python/pants_test/init/test_plugin_resolver.py @@ -20,7 +20,7 @@ from pants.engine.environment import CompleteEnvironment from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests, Snapshot from pants.engine.internals.scheduler import ExecutionError -from pants.engine.process import Process, ProcessResult +from pants.engine.process import ProcessResult from pants.init.options_initializer import create_bootstrap_scheduler from pants.init.plugin_resolver import PluginResolver from pants.option.options_bootstrapper import OptionsBootstrapper @@ -45,8 +45,7 @@ def rule_runner() -> RuleRunner: *external_tool.rules(), *archive.rules(), QueryRule(Pex, [PexRequest]), - QueryRule(Process, [PexProcess]), - QueryRule(ProcessResult, [Process]), + QueryRule(ProcessResult, [PexProcess]), ] ) rule_runner.set_options( @@ -94,11 +93,9 @@ def _run_setup_py( ) merged_digest = rule_runner.request(Digest, [MergeDigests([pex_obj.digest, source_digest])]) - # This should run the Pex using the same interpreter used to create it. We must set the `PATH` so that the shebang - # works. - process = Process( - argv=("./setup-py-runner.pex", "setup.py", *setup_py_args), - env={k: os.environ[k] for k in ["PATH", "HOME", "PYENV_ROOT"] if k in os.environ}, + process = PexProcess( + pex=pex_obj, + argv=("setup.py", *setup_py_args), input_digest=merged_digest, description="Run setup.py", output_directories=("dist/",),