Add Requires-Dist and Requires-Python to locks. (#1585)

These bits of metadata are needed to consume the lock without needing to
resolve through Pip.

Work towards #1583.

pex/cli/commands/lockfile/json_codec.py

@@ -9,6 +9,7 @@
 from pex.cli.commands.lockfile import ParseError
 from pex.cli.commands.lockfile.lockfile import Lockfile
 from pex.enum import Enum
+from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.resolve.locked_resolve import (
     Artifact,
@@ -17,10 +18,10 @@
     LockedResolve,
     LockStyle,
     Pin,
-    Version,
 )
 from pex.resolve.resolver_configuration import ResolverVersion
 from pex.third_party.packaging import tags
+from pex.third_party.packaging.specifiers import InvalidSpecifier, SpecifierSet
 from pex.third_party.pkg_resources import Requirement, RequirementParseError
 from pex.typing import TYPE_CHECKING, cast
 
@@ -131,6 +132,18 @@ def parse_requirement(
                 "The requirement string at '{path}' is invalid: {err}".format(path=path, err=e)
             )
 
+    def parse_version_specifier(
+        raw_version_specifier,  # type: str
+        path,  # type: str
+    ):
+        # type: (...) -> SpecifierSet
+        try:
+            return SpecifierSet(raw_version_specifier)
+        except InvalidSpecifier as e:
+            raise ParseError(
+                "The version specifier at '{path}' is invalid: {err}".format(path=path, err=e)
+            )
+
     requirements = [
         parse_requirement(req, path=".requirements[{index}]".format(index=index))
         for index, req in enumerate(get("requirements", list))
@@ -178,11 +191,6 @@ def assemble_tag(
         ):
             req_path = "{lock_path}[{req_index}]".format(lock_path=lock_path, req_index=req_index)
 
-            requirement = parse_requirement(
-                raw_requirement=get("requirement", data=req, path=req_path),
-                path='{path}["requirement"]'.format(path=req_path),
-            )
-
             artifacts = []
             for i, artifact in enumerate(get("artifacts", list, data=req, path=req_path)):
                 ap = '{path}["artifacts"][{index}]'.format(path=req_path, index=i)
@@ -202,15 +210,31 @@ def assemble_tag(
                         path=req_path, source=source
                     )
                 )
+
+            requires_python = None
+            version_specifier = get("requires_python", data=req, path=req_path, optional=True)
+            if version_specifier:
+                requires_python = parse_version_specifier(
+                    version_specifier, path='{path}["requires_python"]'.format(path=req_path)
+                )
+
             locked_reqs.append(
                 LockedRequirement.create(
                     pin=Pin(
                         project_name=ProjectName(get("project_name", data=req, path=req_path)),
                         version=Version(get("version", data=req, path=req_path)),
                     ),
-                    requirement=requirement,
+                    requires_python=requires_python,
+                    requires_dists=[
+                        parse_requirement(
+                            requires_dist,
+                            path='{path}["requires_dists"][{index}]'.format(path=req_path, index=i),
+                        )
+                        for i, requires_dist in enumerate(
+                            get("requires_dists", list, data=req, path=req_path)
+                        )
+                    ],
                     artifact=artifacts[0],
-                    via=tuple(get("via", list, data=req, path=req_path)),
                     additional_artifacts=artifacts[1:],
                 )
             )
@@ -288,8 +312,10 @@ def as_json_data(lockfile):
                     {
                         "project_name": str(req.pin.project_name),
                         "version": str(req.pin.version),
-                        "requirement": str(req.requirement),
-                        "via": req.via,
+                        "requires_dists": [str(dependency) for dependency in req.requires_dists],
+                        "requires_python": str(req.requires_python)
+                        if req.requires_python
+                        else None,
                         "artifacts": [
                             {
                                 "url": artifact.url,

pex/cli/commands/lockfile/updater.py

@@ -13,8 +13,9 @@
 from pex.common import pluralize
 from pex.distribution_target import DistributionTarget, DistributionTargets
 from pex.network_configuration import NetworkConfiguration
+from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
-from pex.resolve.locked_resolve import LockConfiguration, LockedRequirement, LockedResolve, Version
+from pex.resolve.locked_resolve import LockConfiguration, LockedRequirement, LockedResolve
 from pex.resolve.requirement_configuration import RequirementConfiguration
 from pex.resolve.resolver_configuration import PipConfiguration, ReposConfiguration
 from pex.sorted_tuple import SortedTuple
@@ -186,12 +187,6 @@ def update_resolve(
             elif project_name in self.update_constraints_by_project_name:
                 updates[project_name] = None
 
-            updated_requirements_by_project_name[project_name] = attr.evolve(
-                updated_requirement,
-                requirement=locked_requirement.requirement,
-                via=locked_requirement.via,
-            )
-
         return ResolveUpdate(
             updated_resolve=attr.evolve(
                 locked_resolve,

pex/dist_metadata.py

@@ -17,13 +17,15 @@
 from pex import pex_warnings
 from pex.common import open_zip, pluralize
 from pex.compatibility import to_unicode
+from pex.pep_440 import Version
+from pex.pep_503 import ProjectName
 from pex.third_party.packaging.specifiers import SpecifierSet
 from pex.third_party.pkg_resources import DistInfoDistribution, Distribution, Requirement
 from pex.typing import TYPE_CHECKING, cast
 
 if TYPE_CHECKING:
     import attr  # vendor:skip
-    from typing import Dict, Iterable, Iterator, List, Optional, Union
+    from typing import Dict, Iterable, Iterator, List, Optional, Tuple, Union
 
     DistributionLike = Union[Distribution, str]
 else:
@@ -329,3 +331,29 @@ def requires_dists(dist):
                 ),
             )
         )
+
+
+@attr.s(frozen=True)
+class DistMetadata(object):
+    @classmethod
+    def for_dist(cls, dist):
+        # type: (DistributionLike) -> DistMetadata
+
+        project_name_and_ver = project_name_and_version(dist)
+        if not project_name_and_ver:
+            raise MetadataError(
+                "Failed to determine project name and version for distribution {dist}.".format(
+                    dist=dist
+                )
+            )
+        return cls(
+            project_name=ProjectName(project_name_and_ver.project_name),
+            version=Version(project_name_and_ver.version),
+            requires_dists=tuple(requires_dists(dist)),
+            requires_python=requires_python(dist),
+        )
+
+    project_name = attr.ib()  # type: ProjectName
+    version = attr.ib()  # type: Version
+    requires_dists = attr.ib()  # type: Tuple[Requirement, ...]
+    requires_python = attr.ib()  # type: Optional[SpecifierSet]

pex/pep_440.py

@@ -0,0 +1,28 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+from pex.third_party.packaging import utils as packaging_utils
+from pex.typing import TYPE_CHECKING, cast
+
+if TYPE_CHECKING:
+    import attr  # vendor:skip
+else:
+    from pex.third_party import attr
+
+
+def _canonicalize_version(version):
+    # type: (str) -> str
+    return cast(str, packaging_utils.canonicalize_version(version))
+
+
+@attr.s(frozen=True)
+class Version(object):
+    """A PEP-440 normalized version: https://www.python.org/dev/peps/pep-0440/#normalization"""
+
+    version = attr.ib(converter=_canonicalize_version)  # type: str
+
+    def __str__(self):
+        # type: () -> str
+        return self.version

pex/pip.py

@@ -23,7 +23,7 @@
 from pex import dist_metadata, third_party
 from pex.common import atomic_directory, is_python_script, safe_mkdtemp
 from pex.compatibility import MODE_READ_UNIVERSAL_NEWLINES, get_stdout_bytes_buffer, urlparse
-from pex.dist_metadata import ProjectNameAndVersion
+from pex.dist_metadata import DistMetadata, ProjectNameAndVersion
 from pex.distribution_target import DistributionTarget
 from pex.fetcher import URLFetcher
 from pex.interpreter import PythonInterpreter
@@ -327,6 +327,7 @@ class ResolvedRequirement(object):
     def lock_all(
         cls,
         resolved_requirements,  # type: Iterable[ResolvedRequirement]
+        dist_metadatas,  # type: Iterable[DistMetadata]
         url_fetcher,  # type: URLFetcher
     ):
         # type: (...) -> Iterator[LockedRequirement]
@@ -355,16 +356,32 @@ def resolve_fingerprint(partial_artifact):
                 or fingerprint_by_url[partial_artifact.url],
             )
 
+        dist_metadata_by_pin = {
+            Pin(dist_info.project_name, dist_info.version): dist_info
+            for dist_info in dist_metadatas
+        }
         for resolved_requirement in resolved_requirements:
+            distribution_metadata = dist_metadata_by_pin.get(resolved_requirement.pin)
+            if distribution_metadata is None:
+                raise ValueError(
+                    "No distribution metadata found for {project}.\n"
+                    "Given distribution metadata for:\n"
+                    "{projects}".format(
+                        project=resolved_requirement.pin.as_requirement(),
+                        projects="\n".join(
+                            sorted(str(pin.as_requirement()) for pin in dist_metadata_by_pin)
+                        ),
+                    )
+                )
             yield LockedRequirement.create(
                 pin=resolved_requirement.pin,
                 artifact=resolve_fingerprint(resolved_requirement.artifact),
-                requirement=resolved_requirement.requirement,
+                requires_dists=distribution_metadata.requires_dists,
+                requires_python=distribution_metadata.requires_python,
                 additional_artifacts=(
                     resolve_fingerprint(artifact)
                     for artifact in resolved_requirement.additional_artifacts
                 ),
-                via=resolved_requirement.via,
             )
 
     pin = attr.ib()  # type: Pin
@@ -506,8 +523,11 @@ def analysis_completed(self):
         # type: () -> None
         self._analysis_completed = True
 
-    def lock(self):
-        # type: () -> LockedResolve
+    def lock(
+        self,
+        dist_metadatas,  # type: Iterable[DistMetadata]
+    ):
+        # type: (...) -> LockedResolve
         if not self._analysis_completed:
             raise self.StateError(
                 "Lock retrieval was attempted before Pip log analysis was complete."
@@ -516,7 +536,9 @@ def lock(self):
             self._locked_resolve = LockedResolve.from_target(
                 target=self._target,
                 locked_requirements=tuple(
-                    ResolvedRequirement.lock_all(self._resolved_requirements, self._url_fetcher)
+                    ResolvedRequirement.lock_all(
+                        self._resolved_requirements, dist_metadatas, self._url_fetcher
+                    )
                 ),
             )
         return self._locked_resolve

pex/resolve/locked_resolve.py

@@ -8,17 +8,18 @@
 from pex.dist_metadata import ProjectNameAndVersion
 from pex.distribution_target import DistributionTarget
 from pex.enum import Enum
+from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.sorted_tuple import SortedTuple
 from pex.third_party.packaging import tags
-from pex.third_party.packaging import utils as packaging_utils
+from pex.third_party.packaging.specifiers import SpecifierSet
 from pex.third_party.pkg_resources import Requirement
-from pex.typing import TYPE_CHECKING, cast
+from pex.typing import TYPE_CHECKING
 from pex.util import CacheHelper
 
 if TYPE_CHECKING:
     import attr  # vendor:skip
-    from typing import BinaryIO, IO, Iterable, Iterator, Tuple
+    from typing import BinaryIO, IO, Iterable, Iterator, Optional
 else:
     from pex.third_party import attr
 
@@ -59,20 +60,6 @@ class Artifact(object):
     fingerprint = attr.ib()  # type: Fingerprint
 
 
-def _canonicalize_version(version):
-    # type: (str) -> str
-    return cast(str, packaging_utils.canonicalize_version(version))
-
-
-@attr.s(frozen=True)
-class Version(object):
-    version = attr.ib(converter=_canonicalize_version)  # type: str
-
-    def __str__(self):
-        # type: () -> str
-        return self.version
-
-
 @attr.s(frozen=True)
 class Pin(object):
     @classmethod
@@ -100,24 +87,24 @@ def create(
         cls,
         pin,  # type: Pin
         artifact,  # type: Artifact
-        requirement,  # type: Requirement
+        requires_dists=(),  # type: Iterable[Requirement]
+        requires_python=None,  # type: Optional[SpecifierSet]
         additional_artifacts=(),  # type: Iterable[Artifact]
-        via=(),  # type: Iterable[str]
     ):
         # type: (...) -> LockedRequirement
         return cls(
             pin=pin,
             artifact=artifact,
-            requirement=requirement,
+            requires_dists=SortedTuple(requires_dists, key=lambda req: str(req)),
+            requires_python=requires_python,
             additional_artifacts=SortedTuple(additional_artifacts),
-            via=tuple(via),
         )
 
     pin = attr.ib()  # type: Pin
     artifact = attr.ib()  # type: Artifact
-    requirement = attr.ib(order=str)  # type: Requirement
-    additional_artifacts = attr.ib(default=())  # type: SortedTuple[Artifact]
-    via = attr.ib(default=())  # type: Tuple[str, ...]
+    requires_dists = attr.ib(default=SortedTuple())  # type: SortedTuple[Requirement]
+    requires_python = attr.ib(default=None)  # type: Optional[SpecifierSet]
+    additional_artifacts = attr.ib(default=SortedTuple())  # type: SortedTuple[Artifact]
 
     def iter_artifacts(self):
         # type: () -> Iterator[Artifact]
@@ -160,25 +147,20 @@ def emit_artifact(
         ):
             # type: (...) -> None
             stream.write(
-                "    --hash:{algorithm}={hash} # {url}{line_continuation}\n".format(
+                "    --hash={algorithm}:{hash} {line_continuation}\n".format(
                     algorithm=artifact.fingerprint.algorithm,
                     hash=artifact.fingerprint.hash,
-                    url=artifact.url,
                     line_continuation=" \\" if line_continuation else "",
                 )
             )
 
         for locked_requirement in self.locked_requirements:
             stream.write(
-                "{project_name}=={version} # {requirement}".format(
+                "{project_name}=={version} \\\n".format(
                     project_name=locked_requirement.pin.project_name,
                     version=locked_requirement.pin.version,
-                    requirement=locked_requirement.requirement,
                 )
             )
-            if locked_requirement.via:
-                stream.write(" via -> {}".format(" via -> ".join(locked_requirement.via)))
-            stream.write(" \\\n")
             emit_artifact(
                 locked_requirement.artifact,
                 line_continuation=bool(locked_requirement.additional_artifacts),