feat: New lock strategy: inherit_metadata (#2421)

docs/docs/reference/configuration.md

@@ -63,6 +63,7 @@ The following configuration items can be retrieved and modified by [`pdm config`
 | `strategy.save`                   | Specify how to save versions when a package is added                      | `minimum`(can be: `exact`, `wildcard`, `minimum`, `compatible`)       | Yes                  |                           |
 | `strategy.update`                 | The default strategy for updating packages                                | `reuse`(can be : `eager`)                                             | Yes                  |                           |
 | `strategy.resolve_max_rounds`     | Specify the max rounds of resolution process                              | 10000                                                                 | Yes                  | `PDM_RESOLVE_MAX_ROUNDS`  |
+| `strategy.inherit_metadata`       | Inherit the groups and markers from parents for each package              | Yes                                                                   | Yes                  |                           |
 | `venv.location`                   | Parent directory for virtualenvs                                          | `<default data location on OS>/venvs`                                 | No                   |                           |
 | `venv.backend`                    | Default backend to create virtualenv                                      | `virtualenv`                                                          | Yes                  | `PDM_VENV_BACKEND`        |
 | `venv.prompt`                     | Formatted string to be displayed in the prompt when virtualenv is active  | `{project_name}-{python_version}`                                     | Yes                  | `PDM_VENV_PROMPT`         |

docs/docs/usage/dependency.md

@@ -385,6 +385,16 @@ For example, if you specified `flask>=2.0` in the `pyproject.toml`, `flask` will
     Version constraints in package dependencies are not future-proof. If you resolve the dependencies to the minimal versions, there will likely be backwards-compatibility issues.
     For example, `flask==2.0.0` requires `werkzeug>=2.0`, but in fact, it can not work with `Werkzeug 3.0.0`, which is released 2 years after it.
 
+### Inherit the metadata from parents
+
+_New in version 2.11.0_
+
+By default, `pdm lock` records package metadata as-is, and when installing, PDM will traverse from the top requirements till the leaf node of the dependency tree,
+evaluate any marker it encounters with the current environment, and discard the package if the marker is not satisfied. This requires an extra "resolution"
+step when installing.
+
+By specifying the strategy by `--strategy inherit_metadata`, PDM will however inherit and merge the environment markers from the ancestors of a given package and encode them in the lockfile. This will make the installation much faster. You can also turn it on in the config by `pdm config strategy.inherit_metadata true`.
+
 ## Show what packages are installed
 
 Similar to `pip list`, you can list all packages installed in the packages directory:
@@ -450,7 +460,7 @@ pdm list flask-* requests-*
 
 ??? warning "Be careful with the shell expansion"
     In most shells, the wildcard `*` will be expanded if there are matching files under the current directory.
-    To avoid getting unexpected results, you can quote the patterns: `pdm list 'flask-*' 'requests-*'`.
+    To avoid getting unexpected results, you can wrap the patterns with single quotes: `pdm list 'flask-*' 'requests-*'`.
 
 In `--tree` mode, only the subtree of the matched packages will be displayed. This can be used to achieve the same purpose as `pnpm why`, which is to show why a specific package is required.
 

src/pdm/cli/actions.py

@@ -28,9 +28,10 @@
 from pdm.environments import BareEnvironment
 from pdm.exceptions import PdmException, PdmUsageError, ProjectError
 from pdm.models.candidates import Candidate
+from pdm.models.repositories import LockedRepository
 from pdm.models.requirements import Requirement, parse_requirement
 from pdm.project import Project
-from pdm.project.lockfile import FLAG_CROSS_PLATFORM, FLAG_DIRECT_MINIMAL_VERSIONS
+from pdm.project.lockfile import FLAG_CROSS_PLATFORM, FLAG_DIRECT_MINIMAL_VERSIONS, FLAG_INHERIT_METADATA
 from pdm.resolver import resolve
 from pdm.termui import logger
 from pdm.utils import deprecation_warning
@@ -50,6 +51,8 @@ def do_lock(
     """Performs the locking process and update lockfile."""
     hooks = hooks or HookManager(project)
     check_project_file(project)
+    if project.config["strategy.inherit_metadata"]:
+        project.lockfile.default_strategies.append(FLAG_INHERIT_METADATA)
     lock_strategy = project.lockfile.apply_strategy_change(strategy_change or [])
     if refresh:
         locked_repo = project.locked_repository
@@ -102,6 +105,7 @@ def do_lock(
                     requirements,
                     project.environment.python_requires,
                     resolve_max_rounds,
+                    inherit_metadata=FLAG_INHERIT_METADATA in lock_strategy,
                 )
                 spin.update("Fetching hashes for resolved packages...")
                 fetch_hashes(provider.repository, mapping)
@@ -130,7 +134,10 @@ def do_lock(
 
 
 def resolve_candidates_from_lockfile(
-    project: Project, requirements: Iterable[Requirement], cross_platform: bool = False
+    project: Project,
+    requirements: Iterable[Requirement],
+    cross_platform: bool = False,
+    groups: Collection[str] | None = None,
 ) -> dict[str, Candidate]:
     ui = project.core.ui
     resolve_max_rounds = int(project.config["strategy.resolve_max_rounds"])
@@ -143,14 +150,18 @@ def resolve_candidates_from_lockfile(
             provider = project.get_provider(for_install=True)
             if cross_platform:
                 provider.repository.ignore_compatibility = True
+            if FLAG_INHERIT_METADATA in project.lockfile.strategy and groups is not None:
+                return {
+                    c.identify(): c for c in cast(LockedRepository, provider.repository).evaluate_candidates(groups)
+                }
             resolver: Resolver = project.core.resolver_class(provider, reporter)
             try:
                 mapping, *_ = resolve(
                     resolver,
                     reqs,
                     project.environment.python_requires,
                     resolve_max_rounds,
-                    record_markers=cross_platform,
+                    inherit_metadata=cross_platform,
                 )
             except ResolutionImpossible as e:
                 logger.exception("Broken lockfile")
@@ -208,7 +219,7 @@ def do_sync(
         selection.validate()
         for group in selection:
             requirements.extend(project.get_dependencies(group).values())
-    candidates = resolve_candidates_from_lockfile(project, requirements)
+    candidates = resolve_candidates_from_lockfile(project, requirements, groups=list(selection))
     if tracked_names and dry_run:
         candidates = {name: c for name, c in candidates.items() if name in tracked_names}
     synchronizer = project.core.synchronizer_class(

src/pdm/cli/commands/export.py

@@ -61,7 +61,9 @@ def handle(self, project: Project, options: argparse.Namespace) -> None:
             if not project.lockfile.exists():
                 raise PdmUsageError("No lockfile found, please run `pdm lock` first.")
 
-            candidates = resolve_candidates_from_lockfile(project, requirements.values(), cross_platform=True)
+            candidates = resolve_candidates_from_lockfile(
+                project, requirements.values(), groups=set(selection), cross_platform=True
+            )
             # Remove candidates with [extras] because the bare candidates are already
             # included
             packages = (candidate for candidate in candidates.values() if not candidate.req.extras)

src/pdm/cli/commands/list.py

@@ -160,7 +160,9 @@ def handle(self, project: Project, options: argparse.Namespace) -> None:
             r for g in selected_groups if g != SUBDEP_GROUP_LABEL for r in project.get_dependencies(g).values()
         ]
         if options.resolve:
-            candidates = actions.resolve_candidates_from_lockfile(project, requirements)
+            candidates = actions.resolve_candidates_from_lockfile(
+                project, requirements, groups=selected_groups - {SUBDEP_GROUP_LABEL}
+            )
             packages: Mapping[str, im.Distribution] = {
                 k: c.prepare(project.environment).metadata for k, c in candidates.items()
             }

src/pdm/cli/options.py

@@ -372,8 +372,8 @@ def ignore_python_option(project: Project, namespace: argparse.Namespace, values
     dest="strategy_change",
     metavar="STRATEGY",
     action=split_lists(","),
-    help="Specify lock strategy (cross_platform, static_urls, direct_minimal_versions). Add 'no_' prefix to disable."
-    " Can be supplied multiple times or split by comma.",
+    help="Specify lock strategy (cross_platform, static_urls, direct_minimal_versions, inherit_metadata). "
+    "Add 'no_' prefix to disable. Can be supplied multiple times or split by comma.",
 )
 lock_strategy_group.add_argument(
     "--no-cross-platform",

src/pdm/cli/utils.py

@@ -37,7 +37,7 @@
     strip_extras,
 )
 from pdm.models.specifiers import PySpecSet, get_specifier
-from pdm.project.lockfile import FLAG_CROSS_PLATFORM, FLAG_STATIC_URLS
+from pdm.project.lockfile import FLAG_CROSS_PLATFORM, FLAG_INHERIT_METADATA, FLAG_STATIC_URLS
 from pdm.utils import (
     comparable_version,
     is_path_relative_to,
@@ -524,6 +524,10 @@ def format_lockfile(
         base = tomlkit.table()
         base.update(v.as_lockfile_entry(project.root))
         base.add("summary", v.summary or "")
+        if FLAG_INHERIT_METADATA in strategy:
+            base.add("groups", v.req.groups)
+            if v.req.marker is not None:
+                base.add("marker", str(v.req.marker))
         deps: list[str] = []
         for r in fetched_dependencies[v.dep_key]:
             # Try to convert to relative paths to make it portable

src/pdm/installers/synchronizers.py

@@ -15,7 +15,7 @@
 from pdm.environments import BaseEnvironment
 from pdm.exceptions import InstallationError
 from pdm.installers.manager import InstallManager
-from pdm.models.candidates import Candidate, make_candidate
+from pdm.models.candidates import Candidate
 from pdm.models.reporter import BaseReporter, RichProgressReporter
 from pdm.models.requirements import FileRequirement, Requirement, parse_requirement, strip_extras
 from pdm.utils import is_editable, normalize_name
@@ -159,7 +159,7 @@ def candidates(self) -> dict[str, Candidate]:
                 candidate = candidates[key]
                 # Create a new candidate with editable=False
                 req = dataclasses.replace(candidate.req, editable=False)
-                candidates[key] = make_candidate(req, candidate.name, candidate.version, candidate.link)
+                candidates[key] = candidate.copy_with(req)
         return candidates
 
     def should_install_editables(self) -> bool:

src/pdm/models/candidates.py

@@ -5,7 +5,7 @@
 import os
 import re
 import warnings
-from functools import cached_property, lru_cache
+from functools import cached_property
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import TYPE_CHECKING, Any, cast, no_type_check
@@ -685,14 +685,3 @@ def _get_wheel_dir(self) -> str:
             return wheel_cache.get_ephemeral_path_for_link(
                 self.candidate.link, self.environment.target_python
             ).as_posix()
-
-
-@lru_cache(maxsize=None)
-def make_candidate(
-    req: Requirement,
-    name: str | None = None,
-    version: str | None = None,
-    link: Link | None = None,
-) -> Candidate:
-    """Construct a candidate and cache it in memory"""
-    return Candidate(req, name, version, link)

src/pdm/models/repositories.py

@@ -7,11 +7,11 @@
 import sys
 import warnings
 from functools import wraps
-from typing import TYPE_CHECKING, Generator, TypeVar, cast
+from typing import TYPE_CHECKING, Collection, Generator, TypeVar, cast
 
 from pdm import termui
 from pdm.exceptions import CandidateInfoNotFound, CandidateNotFound, PackageWarning, PdmException
-from pdm.models.candidates import Candidate, make_candidate
+from pdm.models.candidates import Candidate
 from pdm.models.requirements import (
     Requirement,
     filter_requirements_with_extras,
@@ -148,7 +148,7 @@ def make_this_candidate(self, requirement: Requirement) -> Candidate:
 
         project = self.environment.project
         link = Link.from_path(project.root)
-        candidate = make_candidate(requirement, project.name, link=link)
+        candidate = Candidate(requirement, project.name, link=link)
         candidate.prepare(self.environment).metadata
         return candidate
 
@@ -514,7 +514,7 @@ def _read_lockfile(self, lockfile: Mapping[str, Any]) -> None:
                 req = Requirement.from_req_dict(package_name, req_dict)
                 if req.is_file_or_url and req.path and not req.url:  # type: ignore[attr-defined]
                     req.url = path_to_url(posixpath.join(root, req.path))  # type: ignore[attr-defined]
-                can = make_candidate(req, name=package_name, version=version)
+                can = Candidate(req, name=package_name, version=version)
                 can.hashes = package.get("files", [])
                 if not static_urls and any("url" in f for f in can.hashes):
                     raise PdmException(
@@ -605,3 +605,15 @@ def find_candidates(
 
     def get_hashes(self, candidate: Candidate) -> list[FileHash]:
         return candidate.hashes
+
+    def evaluate_candidates(self, groups: Collection[str]) -> Iterable[Candidate]:
+        for can in self.packages.values():
+            if not any(g in can.req.groups for g in groups):
+                continue
+            if (
+                not self.ignore_compatibility
+                and can.req.marker is not None
+                and not can.req.marker.evaluate(self.environment.marker_environment)
+            ):
+                continue
+            yield can

src/pdm/models/requirements.py

@@ -82,6 +82,7 @@ class Requirement:
     specifier: SpecifierSet | None = None
     editable: bool = False
     prerelease: bool = False
+    groups: list[str] = dataclasses.field(default_factory=list)
 
     def __post_init__(self) -> None:
         self.requires_python = self.marker.split_pyspec()[1] if self.marker else PySpecSet()

src/pdm/project/config.py

@@ -140,6 +140,9 @@ class Config(MutableMapping[str, str]):
             env_var="PDM_RESOLVE_MAX_ROUNDS",
             coerce=int,
         ),
+        "strategy.inherit_metadata": ConfigItem(
+            "Inherit the groups and markers from parents for each package", False, coerce=ensure_boolean
+        ),
         "install.parallel": ConfigItem(
             "Whether to perform installation and uninstallation in parallel",
             True,

src/pdm/project/core.py

@@ -326,6 +326,7 @@ def get_dependencies(self, group: str | None = None) -> dict[str, Requirement]:
                     req = parse_requirement(line[3:].strip(), True)
                 else:
                     req = parse_requirement(line)
+                req.groups = [group]
                 # make editable packages behind normal ones to override correctly.
                 result[req.identify()] = req
         return result
@@ -497,12 +498,12 @@ def write_lockfile(self, toml_data: dict, show_message: bool = True, write: bool
     def make_self_candidate(self, editable: bool = True) -> Candidate:
         from unearth import Link
 
-        from pdm.models.candidates import make_candidate
+        from pdm.models.candidates import Candidate
 
         req = parse_requirement(path_to_url(self.root.as_posix()), editable)
         assert self.name
         req.name = self.name
-        can = make_candidate(req, name=self.name, link=Link.from_path(self.root))
+        can = Candidate(req, name=self.name, link=Link.from_path(self.root))
         can.prepare(self.environment).metadata
         return can
 

src/pdm/project/lockfile.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import enum
+from functools import cached_property
 from typing import Any, Iterable, Mapping
 
 import tomlkit
@@ -17,7 +18,10 @@
 FLAG_STATIC_URLS = "static_urls"
 FLAG_CROSS_PLATFORM = "cross_platform"
 FLAG_DIRECT_MINIMAL_VERSIONS = "direct_minimal_versions"
-SUPPORTED_FLAGS = frozenset((FLAG_STATIC_URLS, FLAG_CROSS_PLATFORM, FLAG_DIRECT_MINIMAL_VERSIONS))
+FLAG_INHERIT_METADATA = "inherit_metadata"
+SUPPORTED_FLAGS = frozenset(
+    (FLAG_STATIC_URLS, FLAG_CROSS_PLATFORM, FLAG_DIRECT_MINIMAL_VERSIONS, FLAG_INHERIT_METADATA)
+)
 
 
 class Compatibility(enum.IntEnum):
@@ -28,7 +32,11 @@ class Compatibility(enum.IntEnum):
 
 
 class Lockfile(TOMLBase):
-    spec_version = Version("4.4")
+    spec_version = Version("4.4.1")
+
+    @cached_property
+    def default_strategies(self) -> list[str]:
+        return [FLAG_CROSS_PLATFORM]
 
     @property
     def hash(self) -> str:
@@ -45,7 +53,7 @@ def groups(self) -> list[str] | None:
     @property
     def strategy(self) -> set[str]:
         metadata = self._data.get("metadata", {})
-        result: set[str] = set(metadata.get("strategy", [FLAG_CROSS_PLATFORM]))
+        result: set[str] = set(metadata.get("strategy", self.default_strategies))
         if not metadata.get(FLAG_CROSS_PLATFORM, True):
             result.discard(FLAG_CROSS_PLATFORM)
         if metadata.get(FLAG_STATIC_URLS, False):

src/pdm/resolver/core.py

@@ -6,7 +6,7 @@
 from pdm.models.candidates import Candidate
 from pdm.models.repositories import BaseRepository
 from pdm.models.requirements import strip_extras
-from pdm.resolver.graph import merge_markers
+from pdm.resolver.graph import merge_markers, populate_groups
 from pdm.resolver.providers import BaseProvider
 from pdm.resolver.python import PythonRequirement
 from pdm.utils import normalize_name
@@ -24,7 +24,7 @@ def resolve(
     requires_python: PySpecSet,
     max_rounds: int = 10000,
     keep_self: bool = False,
-    record_markers: bool = False,
+    inherit_metadata: bool = False,
 ) -> tuple[dict[str, Candidate], dict[tuple[str, str | None], list[Requirement]]]:
     """Core function to perform the actual resolve process.
     Return a tuple containing 2 items:
@@ -50,8 +50,9 @@ def resolve(
     local_name = (
         normalize_name(repository.environment.project.name) if repository.environment.project.is_library else None
     )
-    if record_markers:
+    if inherit_metadata:
         all_markers = merge_markers(result)
+        populate_groups(result)
     else:
         all_markers = {}
     for key, candidate in list(mapping.items()):

src/pdm/resolver/graph.py

@@ -109,3 +109,27 @@ def _build_marker(
         # Use 'or' to connect metasets inherited from different parents.
         marker = marker | merged if marker is not None else merged
     return marker if marker is not None else get_marker("")
+
+
+def populate_groups(result: Result[Requirement, Candidate, str]) -> None:
+    """Find where the candidates come from by traversing
+    the dependency tree back to the top.
+    """
+
+    resolved: dict[str, set[str]] = {}
+
+    def get_candidate_groups(key: str) -> set[str]:
+        if key in resolved:
+            return resolved[key]
+        res = resolved[key] = set()
+        crit = result.criteria[key]
+        for req, parent in crit.information:
+            if parent is None:
+                res.update(req.groups)
+            else:
+                pkey = _identify_parent(parent)
+                res.update(get_candidate_groups(pkey))
+        return res
+
+    for k, can in result.mapping.items():
+        can.req.groups = sorted(get_candidate_groups(k))