From f45256d3409a7529ca3ed8df8241db0432ba933d Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 24 Jan 2022 08:25:19 -0800 Subject: [PATCH] Add Requires-Dist and Requires-Python to locks. (#1585) These bits of metadata are needed to consume the lock without needing to resolve through Pip. Work towards #1583. --- pex/cli/commands/lockfile/json_codec.py | 46 +++- pex/cli/commands/lockfile/updater.py | 9 +- pex/dist_metadata.py | 30 ++- pex/pep_440.py | 28 +++ pex/pip.py | 34 ++- pex/resolve/locked_resolve.py | 44 ++-- pex/resolve/testing.py | 11 - pex/resolver.py | 214 +++++++++++++----- .../cli/commands/lockfile/test_json_codec.py | 61 +++-- tests/cli/commands/lockfile/test_lockfile.py | 24 +- tests/integration/cli/commands/test_lock.py | 69 +++--- 11 files changed, 378 insertions(+), 192 deletions(-) create mode 100644 pex/pep_440.py diff --git a/pex/cli/commands/lockfile/json_codec.py b/pex/cli/commands/lockfile/json_codec.py index bea8df6e6..04d163ce8 100644 --- a/pex/cli/commands/lockfile/json_codec.py +++ b/pex/cli/commands/lockfile/json_codec.py @@ -9,6 +9,7 @@ from pex.cli.commands.lockfile import ParseError from pex.cli.commands.lockfile.lockfile import Lockfile from pex.enum import Enum +from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.resolve.locked_resolve import ( Artifact, @@ -17,10 +18,10 @@ LockedResolve, LockStyle, Pin, - Version, ) from pex.resolve.resolver_configuration import ResolverVersion from pex.third_party.packaging import tags +from pex.third_party.packaging.specifiers import InvalidSpecifier, SpecifierSet from pex.third_party.pkg_resources import Requirement, RequirementParseError from pex.typing import TYPE_CHECKING, cast @@ -131,6 +132,18 @@ def parse_requirement( "The requirement string at '{path}' is invalid: {err}".format(path=path, err=e) ) + def parse_version_specifier( + raw_version_specifier, # type: str + path, # type: str + ): + # type: (...) -> SpecifierSet + try: + return SpecifierSet(raw_version_specifier) + except InvalidSpecifier as e: + raise ParseError( + "The version specifier at '{path}' is invalid: {err}".format(path=path, err=e) + ) + requirements = [ parse_requirement(req, path=".requirements[{index}]".format(index=index)) for index, req in enumerate(get("requirements", list)) @@ -178,11 +191,6 @@ def assemble_tag( ): req_path = "{lock_path}[{req_index}]".format(lock_path=lock_path, req_index=req_index) - requirement = parse_requirement( - raw_requirement=get("requirement", data=req, path=req_path), - path='{path}["requirement"]'.format(path=req_path), - ) - artifacts = [] for i, artifact in enumerate(get("artifacts", list, data=req, path=req_path)): ap = '{path}["artifacts"][{index}]'.format(path=req_path, index=i) @@ -202,15 +210,31 @@ def assemble_tag( path=req_path, source=source ) ) + + requires_python = None + version_specifier = get("requires_python", data=req, path=req_path, optional=True) + if version_specifier: + requires_python = parse_version_specifier( + version_specifier, path='{path}["requires_python"]'.format(path=req_path) + ) + locked_reqs.append( LockedRequirement.create( pin=Pin( project_name=ProjectName(get("project_name", data=req, path=req_path)), version=Version(get("version", data=req, path=req_path)), ), - requirement=requirement, + requires_python=requires_python, + requires_dists=[ + parse_requirement( + requires_dist, + path='{path}["requires_dists"][{index}]'.format(path=req_path, index=i), + ) + for i, requires_dist in enumerate( + get("requires_dists", list, data=req, path=req_path) + ) + ], artifact=artifacts[0], - via=tuple(get("via", list, data=req, path=req_path)), additional_artifacts=artifacts[1:], ) ) @@ -288,8 +312,10 @@ def as_json_data(lockfile): { "project_name": str(req.pin.project_name), "version": str(req.pin.version), - "requirement": str(req.requirement), - "via": req.via, + "requires_dists": [str(dependency) for dependency in req.requires_dists], + "requires_python": str(req.requires_python) + if req.requires_python + else None, "artifacts": [ { "url": artifact.url, diff --git a/pex/cli/commands/lockfile/updater.py b/pex/cli/commands/lockfile/updater.py index 70d62206c..9480492be 100644 --- a/pex/cli/commands/lockfile/updater.py +++ b/pex/cli/commands/lockfile/updater.py @@ -13,8 +13,9 @@ from pex.common import pluralize from pex.distribution_target import DistributionTarget, DistributionTargets from pex.network_configuration import NetworkConfiguration +from pex.pep_440 import Version from pex.pep_503 import ProjectName -from pex.resolve.locked_resolve import LockConfiguration, LockedRequirement, LockedResolve, Version +from pex.resolve.locked_resolve import LockConfiguration, LockedRequirement, LockedResolve from pex.resolve.requirement_configuration import RequirementConfiguration from pex.resolve.resolver_configuration import PipConfiguration, ReposConfiguration from pex.sorted_tuple import SortedTuple @@ -186,12 +187,6 @@ def update_resolve( elif project_name in self.update_constraints_by_project_name: updates[project_name] = None - updated_requirements_by_project_name[project_name] = attr.evolve( - updated_requirement, - requirement=locked_requirement.requirement, - via=locked_requirement.via, - ) - return ResolveUpdate( updated_resolve=attr.evolve( locked_resolve, diff --git a/pex/dist_metadata.py b/pex/dist_metadata.py index b0bc0eb1d..229609f99 100644 --- a/pex/dist_metadata.py +++ b/pex/dist_metadata.py @@ -17,13 +17,15 @@ from pex import pex_warnings from pex.common import open_zip, pluralize from pex.compatibility import to_unicode +from pex.pep_440 import Version +from pex.pep_503 import ProjectName from pex.third_party.packaging.specifiers import SpecifierSet from pex.third_party.pkg_resources import DistInfoDistribution, Distribution, Requirement from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: import attr # vendor:skip - from typing import Dict, Iterable, Iterator, List, Optional, Union + from typing import Dict, Iterable, Iterator, List, Optional, Tuple, Union DistributionLike = Union[Distribution, str] else: @@ -329,3 +331,29 @@ def requires_dists(dist): ), ) ) + + +@attr.s(frozen=True) +class DistMetadata(object): + @classmethod + def for_dist(cls, dist): + # type: (DistributionLike) -> DistMetadata + + project_name_and_ver = project_name_and_version(dist) + if not project_name_and_ver: + raise MetadataError( + "Failed to determine project name and version for distribution {dist}.".format( + dist=dist + ) + ) + return cls( + project_name=ProjectName(project_name_and_ver.project_name), + version=Version(project_name_and_ver.version), + requires_dists=tuple(requires_dists(dist)), + requires_python=requires_python(dist), + ) + + project_name = attr.ib() # type: ProjectName + version = attr.ib() # type: Version + requires_dists = attr.ib() # type: Tuple[Requirement, ...] + requires_python = attr.ib() # type: Optional[SpecifierSet] diff --git a/pex/pep_440.py b/pex/pep_440.py new file mode 100644 index 000000000..9f5d61df2 --- /dev/null +++ b/pex/pep_440.py @@ -0,0 +1,28 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from pex.third_party.packaging import utils as packaging_utils +from pex.typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + import attr # vendor:skip +else: + from pex.third_party import attr + + +def _canonicalize_version(version): + # type: (str) -> str + return cast(str, packaging_utils.canonicalize_version(version)) + + +@attr.s(frozen=True) +class Version(object): + """A PEP-440 normalized version: https://www.python.org/dev/peps/pep-0440/#normalization""" + + version = attr.ib(converter=_canonicalize_version) # type: str + + def __str__(self): + # type: () -> str + return self.version diff --git a/pex/pip.py b/pex/pip.py index beb9ca0c5..a25adfd64 100644 --- a/pex/pip.py +++ b/pex/pip.py @@ -23,7 +23,7 @@ from pex import dist_metadata, third_party from pex.common import atomic_directory, is_python_script, safe_mkdtemp from pex.compatibility import MODE_READ_UNIVERSAL_NEWLINES, get_stdout_bytes_buffer, urlparse -from pex.dist_metadata import ProjectNameAndVersion +from pex.dist_metadata import DistMetadata, ProjectNameAndVersion from pex.distribution_target import DistributionTarget from pex.fetcher import URLFetcher from pex.interpreter import PythonInterpreter @@ -327,6 +327,7 @@ class ResolvedRequirement(object): def lock_all( cls, resolved_requirements, # type: Iterable[ResolvedRequirement] + dist_metadatas, # type: Iterable[DistMetadata] url_fetcher, # type: URLFetcher ): # type: (...) -> Iterator[LockedRequirement] @@ -355,16 +356,32 @@ def resolve_fingerprint(partial_artifact): or fingerprint_by_url[partial_artifact.url], ) + dist_metadata_by_pin = { + Pin(dist_info.project_name, dist_info.version): dist_info + for dist_info in dist_metadatas + } for resolved_requirement in resolved_requirements: + distribution_metadata = dist_metadata_by_pin.get(resolved_requirement.pin) + if distribution_metadata is None: + raise ValueError( + "No distribution metadata found for {project}.\n" + "Given distribution metadata for:\n" + "{projects}".format( + project=resolved_requirement.pin.as_requirement(), + projects="\n".join( + sorted(str(pin.as_requirement()) for pin in dist_metadata_by_pin) + ), + ) + ) yield LockedRequirement.create( pin=resolved_requirement.pin, artifact=resolve_fingerprint(resolved_requirement.artifact), - requirement=resolved_requirement.requirement, + requires_dists=distribution_metadata.requires_dists, + requires_python=distribution_metadata.requires_python, additional_artifacts=( resolve_fingerprint(artifact) for artifact in resolved_requirement.additional_artifacts ), - via=resolved_requirement.via, ) pin = attr.ib() # type: Pin @@ -506,8 +523,11 @@ def analysis_completed(self): # type: () -> None self._analysis_completed = True - def lock(self): - # type: () -> LockedResolve + def lock( + self, + dist_metadatas, # type: Iterable[DistMetadata] + ): + # type: (...) -> LockedResolve if not self._analysis_completed: raise self.StateError( "Lock retrieval was attempted before Pip log analysis was complete." @@ -516,7 +536,9 @@ def lock(self): self._locked_resolve = LockedResolve.from_target( target=self._target, locked_requirements=tuple( - ResolvedRequirement.lock_all(self._resolved_requirements, self._url_fetcher) + ResolvedRequirement.lock_all( + self._resolved_requirements, dist_metadatas, self._url_fetcher + ) ), ) return self._locked_resolve diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index ed3207355..7d7dfef9e 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -8,17 +8,18 @@ from pex.dist_metadata import ProjectNameAndVersion from pex.distribution_target import DistributionTarget from pex.enum import Enum +from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.sorted_tuple import SortedTuple from pex.third_party.packaging import tags -from pex.third_party.packaging import utils as packaging_utils +from pex.third_party.packaging.specifiers import SpecifierSet from pex.third_party.pkg_resources import Requirement -from pex.typing import TYPE_CHECKING, cast +from pex.typing import TYPE_CHECKING from pex.util import CacheHelper if TYPE_CHECKING: import attr # vendor:skip - from typing import BinaryIO, IO, Iterable, Iterator, Tuple + from typing import BinaryIO, IO, Iterable, Iterator, Optional else: from pex.third_party import attr @@ -59,20 +60,6 @@ class Artifact(object): fingerprint = attr.ib() # type: Fingerprint -def _canonicalize_version(version): - # type: (str) -> str - return cast(str, packaging_utils.canonicalize_version(version)) - - -@attr.s(frozen=True) -class Version(object): - version = attr.ib(converter=_canonicalize_version) # type: str - - def __str__(self): - # type: () -> str - return self.version - - @attr.s(frozen=True) class Pin(object): @classmethod @@ -100,24 +87,24 @@ def create( cls, pin, # type: Pin artifact, # type: Artifact - requirement, # type: Requirement + requires_dists=(), # type: Iterable[Requirement] + requires_python=None, # type: Optional[SpecifierSet] additional_artifacts=(), # type: Iterable[Artifact] - via=(), # type: Iterable[str] ): # type: (...) -> LockedRequirement return cls( pin=pin, artifact=artifact, - requirement=requirement, + requires_dists=SortedTuple(requires_dists, key=lambda req: str(req)), + requires_python=requires_python, additional_artifacts=SortedTuple(additional_artifacts), - via=tuple(via), ) pin = attr.ib() # type: Pin artifact = attr.ib() # type: Artifact - requirement = attr.ib(order=str) # type: Requirement - additional_artifacts = attr.ib(default=()) # type: SortedTuple[Artifact] - via = attr.ib(default=()) # type: Tuple[str, ...] + requires_dists = attr.ib(default=SortedTuple()) # type: SortedTuple[Requirement] + requires_python = attr.ib(default=None) # type: Optional[SpecifierSet] + additional_artifacts = attr.ib(default=SortedTuple()) # type: SortedTuple[Artifact] def iter_artifacts(self): # type: () -> Iterator[Artifact] @@ -160,25 +147,20 @@ def emit_artifact( ): # type: (...) -> None stream.write( - " --hash:{algorithm}={hash} # {url}{line_continuation}\n".format( + " --hash={algorithm}:{hash} {line_continuation}\n".format( algorithm=artifact.fingerprint.algorithm, hash=artifact.fingerprint.hash, - url=artifact.url, line_continuation=" \\" if line_continuation else "", ) ) for locked_requirement in self.locked_requirements: stream.write( - "{project_name}=={version} # {requirement}".format( + "{project_name}=={version} \\\n".format( project_name=locked_requirement.pin.project_name, version=locked_requirement.pin.version, - requirement=locked_requirement.requirement, ) ) - if locked_requirement.via: - stream.write(" via -> {}".format(" via -> ".join(locked_requirement.via))) - stream.write(" \\\n") emit_artifact( locked_requirement.artifact, line_continuation=bool(locked_requirement.additional_artifacts), diff --git a/pex/resolve/testing.py b/pex/resolve/testing.py index 961091355..3bbcbd175 100644 --- a/pex/resolve/testing.py +++ b/pex/resolve/testing.py @@ -3,10 +3,8 @@ from __future__ import absolute_import -from pex.pep_503 import ProjectName from pex.resolve.locked_resolve import Artifact, LockedRequirement, LockedResolve from pex.sorted_tuple import SortedTuple -from pex.third_party.pkg_resources import Requirement from pex.typing import TYPE_CHECKING if TYPE_CHECKING: @@ -29,23 +27,14 @@ def normalize_locked_requirement( skip_urls=False, # type: bool ): # type: (...) -> LockedRequirement - - # We always normalize the following: - # 1. If an input requirement is not pinned, its locked equivalent always will be; so just check - # matching project names. - # 2. Creating a lock using a lock file as input will differ from a creating a lock using - # requirement strings in its via descriptions for each requirement; so don't compare vias at - # all. return attr.evolve( locked_req, artifact=normalize_artifact(locked_req.artifact, skip_urls=skip_urls), - requirement=Requirement.parse(str(ProjectName(locked_req.requirement.project_name))), additional_artifacts=() if skip_additional_artifacts else SortedTuple( normalize_artifact(a, skip_urls=skip_urls) for a in locked_req.additional_artifacts ), - via=(), ) diff --git a/pex/resolver.py b/pex/resolver.py index d8dec4c99..ff098e847 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -5,11 +5,13 @@ from __future__ import absolute_import import functools +import itertools import os import zipfile from collections import OrderedDict, defaultdict -from pex.common import AtomicDirectory, atomic_directory, safe_mkdtemp +from pex.common import AtomicDirectory, atomic_directory, pluralize, safe_mkdtemp +from pex.dist_metadata import DistMetadata from pex.distribution_target import DistributionTarget, DistributionTargets from pex.environment import FingerprintedDistribution from pex.jobs import Raise, SpawnedJob, execute_parallel @@ -30,7 +32,17 @@ if TYPE_CHECKING: import attr # vendor:skip - from typing import DefaultDict, Iterable, Iterator, List, Optional, Sequence, Tuple + from typing import ( + DefaultDict, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Sequence, + Tuple, + ) from pex.requirements import ParsedRequirement else: @@ -74,7 +86,7 @@ def download_distributions(self, dest=None, max_parallel_jobs=None): return [] dest = dest or safe_mkdtemp() - spawn_download = functools.partial(self._spawn_download, dest) + spawn_download = functools.partial(self._spawn_download, dest, max_parallel_jobs) with TRACER.timed("Resolving for:\n {}".format("\n ".join(map(str, self.targets)))): return list( execute_parallel( @@ -88,6 +100,7 @@ def download_distributions(self, dest=None, max_parallel_jobs=None): def _spawn_download( self, resolved_dists_dir, # type: str + max_parallel_jobs, # type: Optional[int] target, # type: DistributionTarget ): # type: (...) -> SpawnedJob[DownloadResult] @@ -120,13 +133,57 @@ def _spawn_download( build_isolation=self.build_isolation, locker=locker, ) - return SpawnedJob.and_then( - job=download_job, - result_func=lambda: DownloadResult( - target, download_dir, locked_resolve=locker.lock() if locker else None - ), + + wheel_builder = WheelBuilder( + package_index_configuration=self.package_index_configuration, + cache=self.cache, + prefer_older_binary=self.prefer_older_binary, + use_pep517=self.use_pep517, + build_isolation=self.build_isolation, ) + def result_func(): + return DownloadResult(target, download_dir) + + if locker: + result_func = functools.partial( + self._finalize_lock, + locker=locker, + download_result=result_func(), + wheel_builder=wheel_builder, + max_parallel_jobs=max_parallel_jobs, + ) + + return SpawnedJob.and_then(job=download_job, result_func=result_func) + + @staticmethod + def _finalize_lock( + locker, # type: Locker + download_result, # type: DownloadResult + wheel_builder, # type: WheelBuilder + max_parallel_jobs, # type: Optional[int] + ): + # type: (...) -> DownloadResult + + build_requests = tuple(download_result.build_requests()) + with TRACER.timed( + "Building {count} source {distributions} to gather metadata for lock.".format( + count=len(build_requests), distributions=pluralize(build_requests, "distribution") + ) + ): + build_results = wheel_builder.build_wheels( + build_requests=build_requests, + max_parallel_jobs=max_parallel_jobs, + ) + dist_metadatas = tuple( + DistMetadata.for_dist(install_request.wheel_path) + for install_request in itertools.chain( + tuple(download_result.install_requests()), + build_results.values(), + ) + ) + return attr.evolve(download_result, locked_resolve=locker.lock(dist_metadatas)) + @attr.s(frozen=True) class DownloadResult(object): @@ -436,40 +493,32 @@ def _iter_installed_distributions( ) -class BuildAndInstallRequest(object): +class WheelBuilder(object): def __init__( self, - build_requests, # type: Iterable[BuildRequest] - install_requests, # type: Iterable[InstallRequest] - direct_requirements=None, # type: Optional[Iterable[ParsedRequirement]] package_index_configuration=None, # type: Optional[PackageIndexConfiguration] cache=None, # type: Optional[str] - compile=False, # type: bool prefer_older_binary=False, # type: bool use_pep517=None, # type: Optional[bool] build_isolation=True, # type: bool verify_wheels=True, # type: bool ): # type: (...) -> None - self._build_requests = tuple(build_requests) - self._install_requests = tuple(install_requests) - self._direct_requirements = tuple(direct_requirements or ()) self._package_index_configuration = package_index_configuration self._cache = cache - self._compile = compile self._prefer_older_binary = prefer_older_binary self._use_pep517 = use_pep517 self._build_isolation = build_isolation self._verify_wheels = verify_wheels + @staticmethod def _categorize_build_requests( - self, build_requests, # type: Iterable[BuildRequest] dist_root, # type: str ): - # type: (...) -> Tuple[Iterable[BuildRequest], Iterable[InstallRequest]] + # type: (...) -> Tuple[Iterable[BuildRequest], Dict[str, InstallRequest]] unsatisfied_build_requests = [] - install_requests = [] # type: List[InstallRequest] + build_results = {} # type: Dict[str, InstallRequest] for build_request in build_requests: build_result = build_request.result(dist_root) if not build_result.is_built: @@ -483,8 +532,8 @@ def _categorize_build_requests( build_request.source_path, build_result.dist_dir ) ) - install_requests.append(build_result.finalize_build()) - return unsatisfied_build_requests, install_requests + build_results[build_request.source_path] = build_result.finalize_build() + return unsatisfied_build_requests, build_results def _spawn_wheel_build( self, @@ -506,8 +555,72 @@ def _spawn_wheel_build( ) return SpawnedJob.wait(job=build_job, result=build_result) - def _categorize_install_requests( + def build_wheels( + self, + build_requests, # type: Iterable[BuildRequest] + workspace=None, # type: Optional[str] + max_parallel_jobs=None, # type: Optional[int] + ): + # type: (...) -> Mapping[str, InstallRequest] + + if not build_requests: + # Nothing to build or install. + return {} + + cache = self._cache or workspace or safe_mkdtemp() + + built_wheels_dir = os.path.join(cache, "built_wheels") + spawn_wheel_build = functools.partial(self._spawn_wheel_build, built_wheels_dir) + + with TRACER.timed( + "Building distributions for:" "\n {}".format("\n ".join(map(str, build_requests))) + ): + build_requests, build_results = self._categorize_build_requests( + build_requests=build_requests, dist_root=built_wheels_dir + ) + + for build_result in execute_parallel( + inputs=build_requests, + spawn_func=spawn_wheel_build, + error_handler=Raise(Untranslatable), + max_jobs=max_parallel_jobs, + ): + build_results[build_result.request.source_path] = build_result.finalize_build() + + return build_results + + +class BuildAndInstallRequest(object): + def __init__( self, + build_requests, # type: Iterable[BuildRequest] + install_requests, # type: Iterable[InstallRequest] + direct_requirements=None, # type: Optional[Iterable[ParsedRequirement]] + package_index_configuration=None, # type: Optional[PackageIndexConfiguration] + cache=None, # type: Optional[str] + compile=False, # type: bool + prefer_older_binary=False, # type: bool + use_pep517=None, # type: Optional[bool] + build_isolation=True, # type: bool + verify_wheels=True, # type: bool + ): + # type: (...) -> None + self._build_requests = tuple(build_requests) + self._install_requests = tuple(install_requests) + self._direct_requirements = tuple(direct_requirements or ()) + self._cache = cache + self._compile = compile + self._wheel_builder = WheelBuilder( + package_index_configuration=package_index_configuration, + cache=self._cache, + prefer_older_binary=prefer_older_binary, + use_pep517=use_pep517, + build_isolation=build_isolation, + verify_wheels=verify_wheels, + ) + + @staticmethod + def _categorize_install_requests( install_requests, # type: Iterable[InstallRequest] installed_wheels_dir, # type: str ): @@ -563,9 +676,6 @@ def install_distributions( cache = self._cache or workspace or safe_mkdtemp() - built_wheels_dir = os.path.join(cache, "built_wheels") - spawn_wheel_build = functools.partial(self._spawn_wheel_build, built_wheels_dir) - installed_wheels_dir = os.path.join(cache, PexInfo.INSTALL_CACHE) spawn_install = functools.partial(self._spawn_install, installed_wheels_dir) @@ -573,24 +683,12 @@ def install_distributions( installations = [] # type: List[InstalledDistribution] # 1. Build local projects and sdists. - if self._build_requests: - with TRACER.timed( - "Building distributions for:" - "\n {}".format("\n ".join(map(str, self._build_requests))) - ): - - build_requests, install_requests = self._categorize_build_requests( - build_requests=self._build_requests, dist_root=built_wheels_dir - ) - to_install.extend(install_requests) - - for build_result in execute_parallel( - inputs=build_requests, - spawn_func=spawn_wheel_build, - error_handler=Raise(Untranslatable), - max_jobs=max_parallel_jobs, - ): - to_install.append(build_result.finalize_build()) + build_results = self._wheel_builder.build_wheels( + build_requests=self._build_requests, + workspace=workspace, + max_parallel_jobs=max_parallel_jobs, + ) + to_install.extend(build_results.values()) # 2. All requirements are now in wheel form: calculate any missing direct requirement # project names from the wheel names. @@ -598,9 +696,6 @@ def install_distributions( "Calculating project names for direct requirements:" "\n {}".format("\n ".join(map(str, self._direct_requirements))) ): - build_requests_by_path = { - build_request.source_path: build_request for build_request in self._build_requests - } def iter_direct_requirements(): # type: () -> Iterator[Requirement] @@ -609,23 +704,24 @@ def iter_direct_requirements(): yield requirement.requirement continue - build_request = build_requests_by_path.get(requirement.path) - if build_request is None: + install_req = build_results.get(requirement.path) + if install_req is None: raise AssertionError( "Failed to compute a project name for {requirement}. No corresponding " - "build request was found from amongst:\n{build_requests}".format( + "wheel was found from amongst:\n{install_requests}".format( requirement=requirement, - build_requests="\n".join( + install_requests="\n".join( sorted( - "{path} -> {build_request}".format( - path=path, build_request=build_request + "{path} -> {wheel_path} {fingerprint}".format( + path=path, + wheel_path=build_result.wheel_path, + fingerprint=build_result.fingerprint, ) - for path, build_request in build_requests_by_path.items() + for path, build_result in build_results.items() ) ), ) ) - install_req = build_request.result(built_wheels_dir).finalize_build() yield requirement.as_requirement(dist=install_req.wheel_path) direct_requirements_by_project_name = defaultdict( @@ -644,10 +740,9 @@ def iter_direct_requirements(): OrderedDict() ) # type: OrderedDict[str, List[InstallRequest]] for install_request in to_install: - install_requests = install_requests_by_wheel_file.setdefault( - install_request.wheel_file, [] + install_requests_by_wheel_file.setdefault(install_request.wheel_file, []).append( + install_request ) - install_requests.append(install_request) representative_install_requests = [ requests[0] for requests in install_requests_by_wheel_file.values() @@ -692,7 +787,8 @@ def add_installation(install_result): ) return installed_distributions - def _check_install(self, installed_distributions): + @staticmethod + def _check_install(installed_distributions): # type: (Iterable[InstalledDistribution]) -> None installed_distribution_by_project_name = OrderedDict( (ProjectName(resolved_distribution.distribution), resolved_distribution) diff --git a/tests/cli/commands/lockfile/test_json_codec.py b/tests/cli/commands/lockfile/test_json_codec.py index cc16d079f..3152e55b6 100644 --- a/tests/cli/commands/lockfile/test_json_codec.py +++ b/tests/cli/commands/lockfile/test_json_codec.py @@ -12,6 +12,7 @@ import pex.cli.commands.lockfile from pex.cli.commands.lockfile import Lockfile, json_codec from pex.compatibility import PY2 +from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.resolve.locked_resolve import ( Artifact, @@ -20,7 +21,6 @@ LockedResolve, LockStyle, Pin, - Version, ) from pex.resolve.resolver_configuration import ResolverVersion from pex.third_party.packaging import tags @@ -66,9 +66,7 @@ def test_roundtrip(tmpdir): url="https://example.org/colors-1.1.8-cp36-cp36m-macosx_10_6_x86_64.whl", fingerprint=Fingerprint(algorithm="blake256", hash="cafebabe"), ), - requirement=Requirement.parse("ansicolors"), additional_artifacts=(), - via=(), ), LockedRequirement.create( pin=Pin(project_name=ProjectName("requests"), version=Version("2.0.0")), @@ -76,14 +74,12 @@ def test_roundtrip(tmpdir): url="https://example.org/requests-2.0.0-py2.py3-none-any.whl", fingerprint=Fingerprint(algorithm="sha256", hash="456"), ), - requirement=Requirement.parse("requests>=2; sys_platform == 'darwin'"), additional_artifacts=( Artifact( url="file://find-links/requests-2.0.0.tar.gz", fingerprint=Fingerprint(algorithm="sha512", hash="123"), ), ), - via=("direct", "from", "a", "test"), ), ], ), @@ -96,9 +92,7 @@ def test_roundtrip(tmpdir): url="https://example.org/colors-1.1.8-cp37-cp37m-manylinux1_x86_64.whl", fingerprint=Fingerprint(algorithm="md5", hash="hackme"), ), - requirement=Requirement.parse("ansicolors"), additional_artifacts=(), - via=(), ), ], ), @@ -135,9 +129,9 @@ def test_roundtrip(tmpdir): } ], "project_name": "ansicolors", - "requirement": "ansicolors", - "version": "1.1.8", - "via": [] + "requires_dists": [], + "requires_python": null, + "version": "1.1.8" } ], "platform_tag": [ @@ -342,15 +336,40 @@ def test_load_invalid_requirement(patch_tool): patch_tool, dedent( """\ - @@ -23,3 +23,3 @@ + @@ -23,3 +23,6 @@ "project_name": "ansicolors", - - "requirement": "ansicolors", - + "requirement": "@invalid requirement", - "version": "1.1.8", + - "requires_dists": [], + + "requires_dists": [ + + "valid_requirement", + + "@invalid_requirement" + + ], + "requires_python": null, """ ), match=re.escape( - "The requirement string at '.locked_resolves[0][0][\"requirement\"]' is invalid: " + "The requirement string at '.locked_resolves[0][0][\"requires_dists\"][1]' is invalid:" + ), + ) + + +def test_load_invalid_requires_python(patch_tool): + # type: (PatchTool) -> None + + assert_parse_error( + patch_tool, + dedent( + """\ + --- lock.orig.json 2022-01-23 16:25:23.099399463 -0800 + +++ lock.json 2022-01-23 16:39:35.547488554 -0800 + @@ -24,3 +24,3 @@ + "requires_dists": [], + - "requires_python": null, + + "requires_python": "@invalid specifier", + "version": "1.1.8" + """ + ), + match=re.escape( + "The version specifier at '.locked_resolves[0][0][\"requires_python\"]' is invalid:" ), ) @@ -498,9 +517,9 @@ def test_load_invalid_no_locked_requirements(patch_tool): - } - ], - "project_name": "ansicolors", - - "requirement": "ansicolors", - - "version": "1.1.8", - - "via": [] + - "requires_dists": [], + - "requires_python": null, + - "version": "1.1.8" - } ], """ @@ -537,9 +556,9 @@ def test_load_invalid_no_locked_resolves(patch_tool): - } - ], - "project_name": "ansicolors", - - "requirement": "ansicolors", - - "version": "1.1.8", - - "via": [] + - "requires_dists": [], + - "requires_python": null, + - "version": "1.1.8" - } - ], - "platform_tag": [ diff --git a/tests/cli/commands/lockfile/test_lockfile.py b/tests/cli/commands/lockfile/test_lockfile.py index 2187e1f35..a6b710f40 100644 --- a/tests/cli/commands/lockfile/test_lockfile.py +++ b/tests/cli/commands/lockfile/test_lockfile.py @@ -46,9 +46,9 @@ } ], "project_name": "ansicolors", - "requirement": "ansicolors", - "version": "1.1.8", - "via": [] + "requires_dists": [], + "requires_python": null, + "version": "1.1.8" } ], "platform_tag": [ @@ -139,9 +139,9 @@ def test_select_universal_compatible_targets( } ], "project_name": "p537", - "requirement": "p537", - "version": "1.0.4", - "via": [] + "requires_dists": [], + "requires_python": null, + "version": "1.0.4" } ], "platform_tag": [ @@ -161,9 +161,9 @@ def test_select_universal_compatible_targets( } ], "project_name": "p537", - "requirement": "p537", - "version": "1.0.4", - "via": [] + "requires_dists": [], + "requires_python": null, + "version": "1.0.4" } ], "platform_tag": [ @@ -230,9 +230,9 @@ def test_select_compatible_targets( } ], "project_name": "p537", - "requirement": "p537==1.0.4", - "version": "1.0.4", - "via": [] + "requires_dists": [], + "requires_python": null, + "version": "1.0.4" } ], "platform_tag": [ diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py index d985ef1d8..d64e54faf 100644 --- a/tests/integration/cli/commands/test_lock.py +++ b/tests/integration/cli/commands/test_lock.py @@ -12,8 +12,9 @@ from pex.cli.commands.lockfile import Lockfile from pex.distribution_target import DistributionTarget from pex.interpreter import PythonInterpreter +from pex.pep_440 import Version from pex.pep_503 import ProjectName -from pex.resolve.locked_resolve import Artifact, Fingerprint, LockedRequirement, Pin, Version +from pex.resolve.locked_resolve import Artifact, Fingerprint, LockedRequirement, Pin from pex.resolve.resolver_configuration import ResolverVersion from pex.resolve.testing import normalize_locked_resolve from pex.sorted_tuple import SortedTuple @@ -154,11 +155,9 @@ def test_create_vcs_unsupported(): } ], "project_name": "certifi", - "requirement": "certifi>=2017.4.17", - "version": "2021.5.30", - "via": [ - "requests" - ] + "requires_dists": [], + "requires_python": null, + "version": "2021.5.30" }, { "artifacts": [ @@ -169,11 +168,11 @@ def test_create_vcs_unsupported(): } ], "project_name": "charset-normalizer", - "requirement": "charset-normalizer~=2.0.0", - "version": "2.0.6", - "via": [ - "requests" - ] + "requires_dists": [ + "unicodedata2; extra == \\"unicode_backport\\"" + ], + "requires_python": ">=3.5.0", + "version": "2.0.6" }, { "artifacts": [ @@ -184,11 +183,9 @@ def test_create_vcs_unsupported(): } ], "project_name": "idna", - "requirement": "idna<4,>=2.5", - "version": "3.2", - "via": [ - "requests" - ] + "requires_dists": [], + "requires_python": ">=3.5", + "version": "3.2" }, { "artifacts": [ @@ -199,9 +196,19 @@ def test_create_vcs_unsupported(): } ], "project_name": "requests", - "requirement": "requests", - "version": "2.26", - "via": [] + "requires_dists": [ + "PySocks!=1.5.7,>=1.5.6; extra == \\"socks\\"", + "certifi>=2017.4.17", + "chardet<5,>=3.0.2; extra == \\"use_chardet_on_py3\\"", + "chardet<5,>=3.0.2; python_version < \\"3\\"", + "charset-normalizer~=2.0.0; python_version >= \\"3\\"", + "idna<3,>=2.5; python_version < \\"3\\"", + "idna<4,>=2.5; python_version >= \\"3\\"", + "urllib3<1.27,>=1.21.1", + "win-inet-pton; (sys_platform == \\"win32\\" and python_version == \\"2.7\\") and extra == \\"socks\\"" + ], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", + "version": "2.26" }, { "artifacts": [ @@ -212,11 +219,9 @@ def test_create_vcs_unsupported(): } ], "project_name": "urllib3", - "requirement": "urllib3<1.27,>=1.21.1", - "version": "1.25.11", - "via": [ - "requests" - ] + "requires_dists": [], + "requires_python": null, + "version": "1.25.11" } ], "platform_tag": [ @@ -399,7 +404,6 @@ def test_update_targeted_closure_shrink(lock_file_path): hash="2ef65639cb9600443f85451df487818c31f993ab288f313d29cc9db4f3cbe6ed", ), ), - requirement=Requirement.parse("requests"), ) ] == list(locked_resolve.locked_requirements) @@ -470,9 +474,9 @@ def test_update_targeted_impossible( } ], "project_name": "p537", - "requirement": "p537", - "version": "1.0.4", - "via": [] + "requires_dists": [], + "requires_python": null, + "version": "1.0.4" } ], "platform_tag": [ @@ -492,9 +496,9 @@ def test_update_targeted_impossible( } ], "project_name": "p537", - "requirement": "p537", - "version": "1.0.4", - "via": [] + "requires_dists": [], + "requires_python": null, + "version": "1.0.4" } ], "platform_tag": [ @@ -601,7 +605,6 @@ def test_excludes_pep517_build_requirements_issue_1565(tmpdir): hash="7664530bb992e3847b61e3aab1580b4df9ed00c5898e80194a9933bc9c80950a", ), ), - requirement=Requirement.parse("ansicolors==1.0.2"), ), LockedRequirement.create( pin=Pin( @@ -619,7 +622,6 @@ def test_excludes_pep517_build_requirements_issue_1565(tmpdir): hash="7dadadb63e13de019463f13d83e0e0567a963cad99a568d0f0001ac1104d8210", ), ), - requirement=Requirement.parse("find==2020.12.3"), ), LockedRequirement.create( pin=Pin( @@ -637,7 +639,6 @@ def test_excludes_pep517_build_requirements_issue_1565(tmpdir): hash="2594b11d6624fff4bf5147b6bdd510ada54a7b5b4e3f2b15ac2a6d3cf99e0bf8", ), ), - requirement=Requirement.parse("cowsay==4.0"), ), ] )