diff --git a/package/pex-scie.lock b/package/pex-scie.lock index 62b50791d..193ba8ff5 100644 --- a/package/pex-scie.lock +++ b/package/pex-scie.lock @@ -12,25 +12,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0", - "url": "https://files.pythonhosted.org/packages/0b/37/f8da2fbd29690b3557cca414c1949f92162981920699cd62095a984983bf/psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl" + "hash": "6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688", + "url": "https://files.pythonhosted.org/packages/01/9e/8be43078a171381953cfee33c07c0d628594b5dbfc5157847b85022c2c1b/psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl" } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", - "version": "6.0.0" + "version": "6.1.0" } ], "platform_tag": [ - "cp312", - "cp312", + "cp313", + "cp313", "macosx_13_0_x86_64" ] }, @@ -40,26 +34,20 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd", - "url": "https://files.pythonhosted.org/packages/19/74/f59e7e0d392bc1070e9a70e2f9190d652487ac115bb16e2eff6b22ad1d24/psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e", + "url": "https://files.pythonhosted.org/packages/1d/cb/313e80644ea407f04f6602a9e23096540d9dc1878755f3952ea8d3d104be/psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl" } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", - "version": "6.0.0" + "version": "6.1.0" } ], "platform_tag": [ - "cp312", - "cp312", - "manylinux_2_31_x86_64" + "cp313", + "cp313", + "macosx_14_0_arm64" ] }, { @@ -68,26 +56,20 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0", - "url": "https://files.pythonhosted.org/packages/7c/06/63872a64c312a24fb9b4af123ee7007a306617da63ff13bcc1432386ead7/psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl" + "hash": "d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a", + "url": "https://files.pythonhosted.org/packages/27/c2/d034856ac47e3b3cdfa9720d0e113902e615f4190d5d1bdb8df4b2015fb2/psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", - "version": "6.0.0" + "version": "6.1.0" } ], "platform_tag": [ - "cp312", - "cp312", - "macosx_14_0_arm64" + "cp313", + "cp313", + "manylinux_2_31_aarch64" ] }, { @@ -96,26 +78,20 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132", - "url": "https://files.pythonhosted.org/packages/cd/5f/60038e277ff0a9cc8f0c9ea3d0c5eb6ee1d2470ea3f9389d776432888e47/psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b", + "url": "https://files.pythonhosted.org/packages/58/4d/8245e6f76a93c98aab285a43ea71ff1b171bcd90c9d238bf81f7021fb233/psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", - "version": "6.0.0" + "version": "6.1.0" } ], "platform_tag": [ - "cp312", - "cp312", - "manylinux_2_31_aarch64" + "cp313", + "cp313", + "manylinux_2_31_x86_64" ] } ], @@ -123,8 +99,8 @@ "only_wheels": [], "overridden": [], "path_mappings": {}, - "pex_version": "2.17.0", - "pip_version": "24.2", + "pex_version": "2.24.3", + "pip_version": "24.3.1", "prefer_older_binary": false, "requirements": [ "psutil>=5.3" @@ -134,5 +110,6 @@ "style": "strict", "target_systems": [], "transitive": true, - "use_pep517": null + "use_pep517": null, + "use_system_time": false } diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 318da3a6e..a9fd49966 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -526,6 +526,20 @@ def add_create_lock_options(cls, create_parser): ) ), ) + create_parser.add_argument( + "--elide-unused-requires-dist", + "--no-elide-unused-requires-dist", + dest="elide_unused_requires_dist", + type=bool, + default=False, + action=HandleBoolAction, + help=( + "When creating the lock, elide dependencies from the 'requires_dists' lists that " + "can never be active due to markers. This does not change the reachable content of " + "the lock, but it does cut down on lock file size. This currently only elides " + "extras deps that are never activated, but may trim more in the future." + ), + ) cls._add_lock_options(create_parser) cls._add_resolve_options(create_parser) cls.add_json_options(create_parser, entity="lock", include_switch=False) @@ -899,6 +913,7 @@ def _create(self): for interpreter_constraint in target_configuration.interpreter_constraints ), target_systems=tuple(self.options.target_systems), + elide_unused_requires_dist=self.options.elide_unused_requires_dist, ) elif self.options.target_systems: return Error( @@ -907,7 +922,10 @@ def _create(self): ) ) else: - lock_configuration = LockConfiguration(style=self.options.style) + lock_configuration = LockConfiguration( + style=self.options.style, + elide_unused_requires_dist=self.options.elide_unused_requires_dist, + ) targets = try_( self._resolve_targets( @@ -1547,6 +1565,7 @@ def _sync(self): for interpreter_constraint in target_configuration.interpreter_constraints ), target_systems=tuple(self.options.target_systems), + elide_unused_requires_dist=self.options.elide_unused_requires_dist, ) elif self.options.target_systems: return Error( @@ -1555,7 +1574,10 @@ def _sync(self): ) ) else: - lock_configuration = LockConfiguration(style=self.options.style) + lock_configuration = LockConfiguration( + style=self.options.style, + elide_unused_requires_dist=self.options.elide_unused_requires_dist, + ) lock_file_path = self.options.lock if os.path.exists(lock_file_path): @@ -1566,6 +1588,7 @@ def _sync(self): style=lock_configuration.style, requires_python=SortedTuple(lock_configuration.requires_python), target_systems=SortedTuple(lock_configuration.target_systems), + elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist, pip_version=pip_configuration.version, resolver_version=pip_configuration.resolver_version, allow_prereleases=pip_configuration.allow_prereleases, diff --git a/pex/resolve/lock_downloader.py b/pex/resolve/lock_downloader.py index aa83d01ac..a89885eb7 100644 --- a/pex/resolve/lock_downloader.py +++ b/pex/resolve/lock_downloader.py @@ -25,7 +25,6 @@ DownloadableArtifact, FileArtifact, LocalProjectArtifact, - LockConfiguration, VCSArtifact, ) from pex.resolve.lockfile.download_manager import DownloadedArtifact, DownloadManager @@ -233,11 +232,7 @@ def create( file_lock_style=file_lock_style, downloader=ArtifactDownloader( resolver=resolver, - lock_configuration=LockConfiguration( - style=lock.style, - requires_python=lock.requires_python, - target_systems=lock.target_systems, - ), + lock_configuration=lock.lock_configuration(), target=target, package_index_configuration=PackageIndexConfiguration.create( pip_version=pip_version, diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index 684a73f0c..a57edd272 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -87,6 +87,7 @@ class LockConfiguration(object): style = attr.ib() # type: LockStyle.Value requires_python = attr.ib(default=()) # type: Tuple[str, ...] target_systems = attr.ib(default=()) # type: Tuple[TargetSystem.Value, ...] + elide_unused_requires_dist = attr.ib(default=False) # type: bool @requires_python.validator @target_systems.validator diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 5d153121c..220092665 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -341,9 +341,11 @@ def lock(self, downloaded): package_index_configuration=self.package_index_configuration, max_parallel_jobs=self.max_parallel_jobs, ), - platform_tag=None - if self.lock_configuration.style == LockStyle.UNIVERSAL - else target.platform.tag, + platform_tag=( + None + if self.lock_configuration.style == LockStyle.UNIVERSAL + else target.platform.tag + ), ) for target, resolved_requirements in resolved_requirements_by_target.items() ) @@ -456,6 +458,7 @@ def create( excluded=dependency_configuration.excluded, overridden=dependency_configuration.all_overrides(), locked_resolves=locked_resolves, + elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist, ) if lock_configuration.style is LockStyle.UNIVERSAL and ( diff --git a/pex/resolve/lockfile/json_codec.py b/pex/resolve/lockfile/json_codec.py index b8283ce79..45bb7cfd6 100644 --- a/pex/resolve/lockfile/json_codec.py +++ b/pex/resolve/lockfile/json_codec.py @@ -209,6 +209,8 @@ def parse_version_specifier( for index, target_system in enumerate(get("target_systems", list, optional=True) or ()) ] + elide_unused_requires_dist = get("elide_unused_requires_dist", bool, optional=True) or False + only_wheels = [ parse_project_name(project_name, path=".only_wheels[{index}]".format(index=index)) for index, project_name in enumerate(get("only_wheels", list, optional=True) or ()) @@ -337,6 +339,7 @@ def assemble_tag( style=get_enum_value(LockStyle, "style"), requires_python=get("requires_python", list), target_systems=target_systems, + elide_unused_requires_dist=elide_unused_requires_dist, pip_version=get_enum_value( PipVersion, "pip_version", diff --git a/pex/resolve/lockfile/model.py b/pex/resolve/lockfile/model.py index bb309580e..36293fa34 100644 --- a/pex/resolve/lockfile/model.py +++ b/pex/resolve/lockfile/model.py @@ -11,7 +11,14 @@ from pex.pep_503 import ProjectName from pex.pip.version import PipVersion, PipVersionValue from pex.requirements import LocalProjectRequirement -from pex.resolve.locked_resolve import LocalProjectArtifact, LockedResolve, LockStyle, TargetSystem +from pex.resolve.locked_resolve import ( + LocalProjectArtifact, + LockConfiguration, + LockedResolve, + LockStyle, + TargetSystem, +) +from pex.resolve.lockfile import requires_dist from pex.resolve.resolved_requirement import Pin from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion from pex.sorted_tuple import SortedTuple @@ -47,6 +54,7 @@ def create( source=None, # type: Optional[str] pip_version=None, # type: Optional[PipVersionValue] resolver_version=None, # type: Optional[ResolverVersion.Value] + elide_unused_requires_dist=False, # type: bool ): # type: (...) -> Lockfile @@ -94,6 +102,7 @@ def extract_requirement(req): style=style, requires_python=SortedTuple(requires_python), target_systems=SortedTuple(target_systems), + elide_unused_requires_dist=elide_unused_requires_dist, pip_version=pip_ver, resolver_version=resolver_version or ResolverVersion.default(pip_ver), requirements=SortedTuple(resolve_requirements, key=str), @@ -110,7 +119,14 @@ def extract_requirement(req): transitive=transitive, excluded=SortedTuple(excluded), overridden=SortedTuple(overridden), - locked_resolves=SortedTuple(locked_resolves), + locked_resolves=SortedTuple( + ( + requires_dist.remove_unused_requires_dist(resolve_requirements, locked_resolve) + if elide_unused_requires_dist + else locked_resolve + ) + for locked_resolve in locked_resolves + ), local_project_requirement_mapping=requirement_by_local_project_directory, source=source, ) @@ -119,6 +135,7 @@ def extract_requirement(req): style = attr.ib() # type: LockStyle.Value requires_python = attr.ib() # type: SortedTuple[str] target_systems = attr.ib() # type: SortedTuple[TargetSystem.Value] + elide_unused_requires_dist = attr.ib() # type: bool pip_version = attr.ib() # type: PipVersionValue resolver_version = attr.ib() # type: ResolverVersion.Value requirements = attr.ib() # type: SortedTuple[Requirement] @@ -139,6 +156,15 @@ def extract_requirement(req): local_project_requirement_mapping = attr.ib(eq=False) # type: Mapping[str, Requirement] source = attr.ib(default=None, eq=False) # type: Optional[str] + def lock_configuration(self): + # type: () -> LockConfiguration + return LockConfiguration( + style=self.style, + requires_python=self.requires_python, + target_systems=self.target_systems, + elide_unused_requires_dist=self.elide_unused_requires_dist, + ) + def build_configuration(self): # type: () -> BuildConfiguration return BuildConfiguration.create( diff --git a/pex/resolve/lockfile/requires_dist.py b/pex/resolve/lockfile/requires_dist.py new file mode 100644 index 000000000..410f7177c --- /dev/null +++ b/pex/resolve/lockfile/requires_dist.py @@ -0,0 +1,162 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import operator +from collections import defaultdict, deque + +from pex.dist_metadata import Requirement +from pex.exceptions import production_assert +from pex.orderedset import OrderedSet +from pex.pep_503 import ProjectName +from pex.resolve.locked_resolve import LockedRequirement, LockedResolve +from pex.sorted_tuple import SortedTuple +from pex.third_party.packaging.markers import Marker, Variable +from pex.typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from typing import Callable, DefaultDict, Dict, Iterable, List, Optional, Tuple, Union + + import attr # vendor:skip + + EvalExtra = Callable[[ProjectName], bool] +else: + from pex.third_party import attr + + +_OPERATORS = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +class _Op(object): + def __init__(self, lhs): + self.lhs = lhs # type: EvalExtra + self.rhs = None # type: Optional[EvalExtra] + + +class _And(_Op): + def __call__(self, extra): + # type: (ProjectName) -> bool + production_assert(self.rhs is not None) + return self.lhs(extra) and cast("EvalExtra", self.rhs)(extra) + + +class _Or(_Op): + def __call__(self, extra): + # type: (ProjectName) -> bool + production_assert(self.rhs is not None) + return self.lhs(extra) or cast("EvalExtra", self.rhs)(extra) + + +def _parse_extra_item( + stack, # type: List[EvalExtra] + item, # type: Union[str, List, Tuple] + marker, # type: Marker +): + # type: (...) -> None + + if item == "and": + stack.append(_And(stack.pop())) + elif item == "or": + stack.append(_Or(stack.pop())) + elif isinstance(item, list): + for element in item: + _parse_extra_item(stack, element, marker) + elif isinstance(item, tuple): + lhs, op, rhs = item + if isinstance(lhs, Variable) and "extra" == str(lhs): + check = lambda extra: _OPERATORS[str(op)](extra, ProjectName(str(rhs))) + elif isinstance(rhs, Variable) and "extra" == str(rhs): + check = lambda extra: _OPERATORS[str(op)](extra, ProjectName(str(lhs))) + else: + # Any other condition could potentially be true. + check = lambda _: True + if stack: + production_assert(isinstance(stack[-1], _Op)) + cast(_Op, stack[-1]).rhs = check + else: + stack.append(check) + else: + raise ValueError("Marker is invalid: {marker}".format(marker=marker)) + + +def _parse_extra_check(marker): + # type: (Marker) -> EvalExtra + checks = [] # type: List[EvalExtra] + for item in marker._markers: + _parse_extra_item(checks, item, marker) + production_assert(len(checks) == 1) + return checks[0] + + +_EXTRA_CHECKS = {} # type: Dict[str, EvalExtra] + + +def _parse_marker_for_extra_check(marker): + # type: (Marker) -> EvalExtra + maker_str = str(marker) + eval_extra = _EXTRA_CHECKS.get(maker_str) + if not eval_extra: + eval_extra = _parse_extra_check(marker) + _EXTRA_CHECKS[maker_str] = eval_extra + return eval_extra + + +def _evaluate_for_extras( + marker, # type: Optional[Marker] + extras, # type: Iterable[str] +): + # type: (...) -> bool + if not marker: + return True + eval_extra = _parse_marker_for_extra_check(marker) + return any(eval_extra(ProjectName(extra)) for extra in (extras or [""])) + + +def remove_unused_requires_dist( + resolve_requirements, # type: Iterable[Requirement] + locked_resolve, # type: LockedResolve +): + # type: (...) -> LockedResolve + + locked_req_by_project_name = { + locked_req.pin.project_name: locked_req for locked_req in locked_resolve.locked_requirements + } + requires_dist_by_locked_req = defaultdict( + OrderedSet + ) # type: DefaultDict[LockedRequirement, OrderedSet[Requirement]] + seen = set() + requirements = deque(resolve_requirements) + while requirements: + requirement = requirements.popleft() + if requirement in seen: + continue + + seen.add(requirement) + locked_req = locked_req_by_project_name[requirement.project_name] + for dep in locked_req.requires_dists: + if _evaluate_for_extras(dep.marker, requirement.extras): + requires_dist_by_locked_req[locked_req].add(dep) + requirements.append(dep) + + return attr.evolve( + locked_resolve, + locked_requirements=SortedTuple( + attr.evolve( + locked_requirement, + requires_dists=SortedTuple( + requires_dist_by_locked_req[locked_requirement], key=str + ), + ) + for locked_requirement in locked_resolve.locked_requirements + ), + ) diff --git a/pex/resolve/lockfile/updater.py b/pex/resolve/lockfile/updater.py index d66224352..11b88fec2 100644 --- a/pex/resolve/lockfile/updater.py +++ b/pex/resolve/lockfile/updater.py @@ -661,11 +661,6 @@ def create( ): # type: (...) -> LockUpdater - lock_configuration = LockConfiguration( - style=lock_file.style, - requires_python=lock_file.requires_python, - target_systems=lock_file.target_systems, - ) pip_configuration = PipConfiguration( version=lock_file.pip_version, resolver_version=lock_file.resolver_version, @@ -680,7 +675,7 @@ def create( ) return cls( lock_file=lock_file, - lock_configuration=lock_configuration, + lock_configuration=lock_file.lock_configuration(), pip_configuration=pip_configuration, dependency_configuration=dependency_configuration, ) diff --git a/scripts/gen-scie-platform.py b/scripts/gen-scie-platform.py index 70b00a771..cec3866bd 100644 --- a/scripts/gen-scie-platform.py +++ b/scripts/gen-scie-platform.py @@ -195,6 +195,7 @@ def create_lock( ), "--pip-version", "latest", + "--elide-unused-requires-dist", "--indent", "2", "--lock", @@ -285,6 +286,7 @@ def main(out: IO[str]) -> str | int | None: parser.add_argument("--all", action="store_true") parser.add_argument("-f", "--force", action="store_true") parser.add_argument("--lock-file", type=Path, default=PACKAGE_DIR / "pex-scie.lock") + parser.add_argument("-L", "--only-sync-lock", action="store_true") parser.add_argument("-v", "--verbose", action="store_true") try: options = parser.parse_args() @@ -298,39 +300,40 @@ def main(out: IO[str]) -> str | int | None: logging.basicConfig(level=logging.INFO if options.verbose else logging.WARNING) generated_files: list[Path] = [] - if options.all: - try: - generated_files.extend( - ensure_all_complete_platforms( - dest_dir=options.dest_dir, scie_config=scie_config, force=options.force + if not options.only_sync_lock: + if options.all: + try: + generated_files.extend( + ensure_all_complete_platforms( + dest_dir=options.dest_dir, scie_config=scie_config, force=options.force + ) ) - ) - except ( - GitHubError, - github.GithubException, - github.BadAttributeException, - httpx.HTTPError, - ) as e: - return str(e) - - try: - create_lock( - lock_file=options.lock_file, - complete_platforms=generated_files, - scie_config=scie_config, - ) - except subprocess.CalledProcessError as e: - return str(e) - generated_files.append(options.lock_file) - else: - complete_platform_file = options.dest_dir / f"{plat}.json" - try: - create_complete_platform( - complete_platform_file=complete_platform_file, scie_config=scie_config - ) - except subprocess.CalledProcessError as e: - return str(e) - generated_files.append(complete_platform_file) + except ( + GitHubError, + github.GithubException, + github.BadAttributeException, + httpx.HTTPError, + ) as e: + return str(e) + else: + complete_platform_file = options.dest_dir / f"{plat}.json" + try: + create_complete_platform( + complete_platform_file=complete_platform_file, scie_config=scie_config + ) + except subprocess.CalledProcessError as e: + return str(e) + generated_files.append(complete_platform_file) + + try: + create_lock( + lock_file=options.lock_file, + complete_platforms=tuple(options.dest_dir.glob("*.json")), + scie_config=scie_config, + ) + except subprocess.CalledProcessError as e: + return str(e) + generated_files.append(options.lock_file) for file in generated_files: print(str(file), file=out) diff --git a/tests/integration/cli/commands/test_export.py b/tests/integration/cli/commands/test_export.py index 3d616f639..0a5d4423b 100644 --- a/tests/integration/cli/commands/test_export.py +++ b/tests/integration/cli/commands/test_export.py @@ -45,6 +45,7 @@ style=LockStyle.UNIVERSAL, requires_python=SortedTuple(), target_systems=SortedTuple(), + elide_unused_requires_dist=False, pip_version=PipVersion.DEFAULT, resolver_version=ResolverVersion.PIP_2020, requirements=SortedTuple([Requirement.parse("ansicolors")]), diff --git a/tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py b/tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py new file mode 100644 index 000000000..fe463780a --- /dev/null +++ b/tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py @@ -0,0 +1,81 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from typing import Dict + +from pex.pep_503 import ProjectName +from pex.resolve.locked_resolve import LockedRequirement +from pex.resolve.lockfile import json_codec +from pex.resolve.lockfile.model import Lockfile +from testing.cli import run_pex3 +from testing.pytest.tmp import Tempdir + + +def index_locked_reqs(lockfile): + # type: (Lockfile) -> Dict[ProjectName, LockedRequirement] + return { + locked_req.pin.project_name: locked_req + for locked_resolve in lockfile.locked_resolves + for locked_req in locked_resolve.locked_requirements + } + + +def test_lock_elide_unused_requires_dist(tmpdir): + # type: (Tempdir) -> None + + lock = tmpdir.join("lock.json") + run_pex3( + "lock", + "create", + "requests==2.31.0", + "--style", + "universal", + "--interpreter-constraint", + ">=3.7,<3.14", + "--indent", + "2", + "-o", + lock, + ).assert_success() + lockfile = json_codec.load(lock) + + elided_lock = tmpdir.join("lock.json") + run_pex3( + "lock", + "create", + "requests==2.31.0", + "--style", + "universal", + "--interpreter-constraint", + ">=3.7,<3.14", + "--elide-unused-requires-dist", + "--indent", + "2", + "-o", + elided_lock, + ).assert_success() + elided_lockfile = json_codec.load(elided_lock) + + assert lockfile != elided_lockfile + + locked_reqs = index_locked_reqs(lockfile) + requests = locked_reqs[ProjectName("requests")] + + elided_locked_reqs = index_locked_reqs(elided_lockfile) + elided_requests = elided_locked_reqs[ProjectName("requests")] + + assert requests != elided_requests + + assert requests.pin == elided_requests.pin + assert list(requests.iter_artifacts()) == list(elided_requests.iter_artifacts()) + assert requests.requires_python == elided_requests.requires_python + + assert requests.requires_dists != elided_requests.requires_dists + assert len(elided_requests.requires_dists) < len(requests.requires_dists) + elided_deps = set(requests.requires_dists) - set(elided_requests.requires_dists) + assert len(elided_deps) > 0 + assert not any( + elided_dep.project_name in elided_locked_reqs for elided_dep in elided_deps + ), "No dependencies that require extra activation should have been locked." diff --git a/tests/resolve/lockfile/test_requires_dist.py b/tests/resolve/lockfile/test_requires_dist.py new file mode 100644 index 000000000..05bd88a48 --- /dev/null +++ b/tests/resolve/lockfile/test_requires_dist.py @@ -0,0 +1,170 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from pex.dist_metadata import Requirement +from pex.pep_440 import Version +from pex.pep_503 import ProjectName +from pex.resolve.locked_resolve import Artifact, LockedRequirement, LockedResolve +from pex.resolve.lockfile import requires_dist +from pex.resolve.resolved_requirement import Fingerprint, Pin +from pex.sorted_tuple import SortedTuple + +req = Requirement.parse + + +def locked_req( + project_name, # type: str + version, # type: str + *requirements # type: str +): + # type: (...) -> LockedRequirement + return LockedRequirement.create( + pin=Pin(project_name=ProjectName(project_name), version=Version(version)), + artifact=Artifact.from_url( + "https://artifact.store/{project_name}-{version}-py2.py3-none-any.whl".format( + project_name=project_name, version=version + ), + fingerprint=Fingerprint(algorithm="md5", hash="abcd0123"), + ), + requires_dists=map(req, requirements), + ) + + +def locked_resolve(*locked_requirements): + # type: (*LockedRequirement) -> LockedResolve + return LockedResolve(locked_requirements=SortedTuple(locked_requirements)) + + +def test_remove_unused_requires_dist_noop(): + # type: () -> None + + locked_resolve_with_no_extras = locked_resolve( + locked_req("foo", "1.0", "bar", "baz"), + locked_req("bar", "1.0"), + locked_req("baz", "1.0"), + ) + assert locked_resolve_with_no_extras == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo")], locked_resolve=locked_resolve_with_no_extras + ) + + +def test_remove_unused_requires_dist_simple(): + # type: () -> None + + assert locked_resolve( + locked_req("foo", "1.0", "bar", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo")], + locked_resolve=locked_resolve( + locked_req("foo", "1.0", "bar", "baz; extra == 'tests'", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ) + + +def test_remove_unused_requires_dist_mixed_extras(): + # type: () -> None + + assert locked_resolve( + locked_req("foo", "1.0", "bar; extra == 'extra1'", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo[extra1]")], + locked_resolve=locked_resolve( + locked_req("foo", "1.0", "bar; extra == 'extra1'", "baz; extra == 'tests'", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ) + + +def test_remove_unused_requires_dist_mixed_markers(): + # type: () -> None + + assert locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "baz; extra == 'tests' or python_version > '3.11'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("baz", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo[extra1]")], + locked_resolve=locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "baz; extra == 'tests' or python_version > '3.11'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("baz", "1.0"), + locked_req("spam", "1.0"), + ), + ), ( + "The python_version marker clause might evaluate to true, which should be enough to retain " + "the baz dep even though the 'tests' extra is never activated." + ) + + assert locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo[extra1]")], + locked_resolve=locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "baz; extra == 'tests' and python_version > '3.11'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ), "The 'tests' extra is never active; so the baz dep should never be reached." + + +def test_remove_unused_requires_dist_complex_markers(): + # type: () -> None + + assert locked_resolve( + locked_req( + "foo", + "1.0", + "bar; python_version < '3' and (extra == 'docs' or python_version >= '3')", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo")], + locked_resolve=locked_resolve( + locked_req( + "foo", + "1.0", + "bar; python_version < '3' and (extra == 'docs' or python_version >= '3')", + "baz; python_version == '3.11.*' and (extra == 'admin' or extra == 'docs')", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + )