From 956a206dfaecc503332d625f6a55698c0af8b711 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 16:55:28 +0800 Subject: [PATCH 01/13] Extract environment related code from Candidate --- pdm/cli/actions.py | 2 +- pdm/cli/commands/show.py | 3 +- pdm/cli/utils.py | 6 +- pdm/formats/requirements.py | 2 +- pdm/installers/manager.py | 3 +- pdm/models/candidates.py | 365 +++++++++++++++++--------------- pdm/models/repositories.py | 14 +- pdm/project/core.py | 4 +- pdm/resolver/providers.py | 8 +- pdm/resolver/python.py | 8 +- pdm/resolver/reporters.py | 2 +- tests/cli/test_add.py | 5 +- tests/conftest.py | 1 - tests/models/test_candidates.py | 136 ++++++------ tests/test_installer.py | 12 +- tests/test_integration.py | 1 + 16 files changed, 296 insertions(+), 276 deletions(-) diff --git a/pdm/cli/actions.py b/pdm/cli/actions.py index bce5f97ff8..4b58c42ade 100644 --- a/pdm/cli/actions.py +++ b/pdm/cli/actions.py @@ -95,7 +95,7 @@ def do_lock( ui.echo(format_resolution_impossible(err), err=True) raise ResolutionImpossible("Unable to find a resolution") from None else: - data = format_lockfile(mapping, dependencies) + data = format_lockfile(project, mapping, dependencies) spin.succeed(f"{termui.Emoji.LOCK} Lock successful") signals.post_lock.send(project, resolution=mapping, dry_run=dry_run) diff --git a/pdm/cli/commands/show.py b/pdm/cli/commands/show.py index 559648ea22..f709a51938 100644 --- a/pdm/cli/commands/show.py +++ b/pdm/cli/commands/show.py @@ -49,8 +49,7 @@ def handle(self, project: Project, options: argparse.Namespace) -> None: ) return latest_stable = next(filter(filter_stable, matches), None) - - metadata = latest.metadata + metadata = latest.prepare(project.environment).metadata else: if not project.meta.name: raise PdmUsageError("This project is not a package") diff --git a/pdm/cli/utils.py b/pdm/cli/utils.py index 8d2fed443d..78f6929480 100644 --- a/pdm/cli/utils.py +++ b/pdm/cli/utils.py @@ -429,7 +429,9 @@ def format_dependency_graph( def format_lockfile( - mapping: dict[str, Candidate], fetched_dependencies: dict[str, list[Requirement]] + project: Project, + mapping: dict[str, Candidate], + fetched_dependencies: dict[str, list[Requirement]], ) -> dict: """Format lock file from a dict of resolved candidates, a mapping of dependencies and a collection of package summaries. @@ -438,7 +440,7 @@ def format_lockfile( file_hashes = tomlkit.table() for k, v in sorted(mapping.items()): base = tomlkit.table() - base.update(v.as_lockfile_entry()) # type: ignore + base.update(v.as_lockfile_entry(project.root)) # type: ignore base.add("summary", v.summary or "") deps = make_array(sorted(r.as_line() for r in fetched_dependencies[k]), True) if len(deps) > 0: diff --git a/pdm/formats/requirements.py b/pdm/formats/requirements.py index 1e4959ed9d..7a215bae94 100644 --- a/pdm/formats/requirements.py +++ b/pdm/formats/requirements.py @@ -53,7 +53,7 @@ def ireq_as_line(ireq: InstallRequirement, environment: Environment) -> str: else: if not ireq.req: req = parse_requirement("dummy @" + ireq.link.url) # type: ignore - req.name = Candidate(req, environment).metadata.metadata["Name"] + req.name = Candidate(req).prepare(environment).metadata.metadata["Name"] ireq.req = req # type: ignore line = _requirement_to_str_lowercase_name(cast(PRequirement, ireq.req)) diff --git a/pdm/installers/manager.py b/pdm/installers/manager.py index 6179cb7d52..b7afd0abec 100644 --- a/pdm/installers/manager.py +++ b/pdm/installers/manager.py @@ -35,7 +35,8 @@ def install(self, candidate: Candidate) -> None: installer = install_wheel_with_cache else: installer = install_wheel - installer(candidate.build(), self.environment, candidate.direct_url()) + prepared = candidate.prepare(self.environment) + installer(prepared.build(), self.environment, prepared.direct_url()) def get_paths_to_remove(self, dist: Distribution) -> BaseRemovePaths: """Get the path collection to be removed from the disk""" diff --git a/pdm/models/candidates.py b/pdm/models/candidates.py index a6929e1923..4c0a19b1ef 100644 --- a/pdm/models/candidates.py +++ b/pdm/models/candidates.py @@ -82,7 +82,6 @@ class Candidate: def __init__( self, req: Requirement, - environment: Environment, name: str | None = None, version: str | None = None, link: pip_shims.Link | None = None, @@ -95,34 +94,140 @@ def __init__( :param link: the file link of the candidate. """ self.req = req - self.environment = environment self.name = name or self.req.project_name self.version = version or self.req.version - if link is None and self.req: - link = self.ireq.link self.link = link self.summary = "" - self.source_dir: str | None = None self.hashes: dict[str, str] | None = None - self._requires_python: str | None = None - self.wheel: str | None = None - self._metadata_dir: str | None = None + self._requires_python: str | None = None + self._prepared: PreparedCandidate | None = None def __hash__(self) -> int: return hash((self.name, self.version)) + def identify(self) -> str: + return self.req.identify() + + @property + def prepared(self) -> PreparedCandidate | None: + return self._prepared + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Candidate): + return False + if self.req.is_named: + return self.name == other.name and self.version == other.version + return self.name == other.name and self.link == other.link + + def get_revision(self) -> str: + if not self.req.is_vcs: + raise AttributeError("Non-VCS candidate doesn't have revision attribute") + if self.req.revision: # type: ignore + return self.req.revision # type: ignore + assert self._prepared + return self._prepared.revision + + def __repr__(self) -> str: + source = getattr(self.link, "comes_from", "unknown") + return f"" + + @classmethod + def from_installation_candidate( + cls, candidate: pip_shims.InstallationCandidate, req: Requirement + ) -> Candidate: + """Build a candidate from pip's InstallationCandidate.""" + return cls( + req, + name=candidate.name, + version=str(candidate.version), + link=candidate.link, + ) + + @property + def requires_python(self) -> str: + """The Python version constraint of the candidate.""" + if self._requires_python is not None: + return self._requires_python + if self.link: + requires_python = self.link.requires_python + if requires_python and requires_python.isdigit(): + requires_python = f">={requires_python},<{int(requires_python) + 1}" + self._requires_python = requires_python + return self._requires_python or "" + + @requires_python.setter + def requires_python(self, value: str) -> None: + self._requires_python = value + + @no_type_check + def as_lockfile_entry(self, project_root: Path) -> dict[str, Any]: + """Build a lockfile entry dictionary for the candidate.""" + root_path = project_root.as_posix() + result = { + "name": normalize_name(self.name), + "version": str(self.version), + "extras": sorted(self.req.extras or ()), + "requires_python": str(self.requires_python), + "editable": self.req.editable, + } + if self.req.is_vcs: + result.update( + { + self.req.vcs: self.req.repo, + "ref": self.req.ref, + } + ) + if not self.req.editable: + result.update(revision=self.get_revision()) + elif not self.req.is_named: + if self.req.is_file_or_url and self.req.is_local_dir: + result.update(path=path_replace(root_path, ".", self.req.str_path)) + else: + result.update( + url=path_replace( + root_path.lstrip("/"), "${PROJECT_ROOT}", self.req.url + ) + ) + return {k: v for k, v in result.items() if v} + + def format(self) -> str: + """Format for output.""" + return ( + f"{termui.green(self.name, bold=True)} " + f"{termui.yellow(str(self.version))}" + ) + + def prepare(self, environment: Environment) -> PreparedCandidate: + """Prepare the candidate for installation.""" + if self._prepared is None: + self._prepared = PreparedCandidate(self, environment) + return self._prepared + + +class PreparedCandidate: + """A candidate that has been prepared for installation. + The metadata and built wheel are available. + """ + + def __init__(self, candidate: Candidate, environment: Environment) -> None: + self.candidate = candidate + self.environment = environment + self.wheel: str | None = None + self.req = candidate.req + + self._metadata_dir: str | None = None + self._metadata = self.prepare_metadata() + @cached_property def ireq(self) -> pip_shims.InstallRequirement: - rv = self.req.as_ireq() + rv, project = self.req.as_ireq(), self.environment.project if rv.link: - rv.link = pip_shims.Link( + rv.original_link = rv.link = pip_shims.Link( expand_env_vars_in_auth( rv.link.url.replace( "${PROJECT_ROOT}", - self.environment.project.root.as_posix().lstrip( # type: ignore - "/" - ), + project.root.as_posix().lstrip("/"), # type: ignore ) ) ) @@ -130,24 +235,12 @@ def ireq(self) -> pip_shims.InstallRequirement: rv.source_dir = os.path.normpath(os.path.abspath(rv.link.file_path)) if rv.local_file_path: rv.local_file_path = rv.link.file_path + elif self.candidate.link: + rv.link = rv.original_link = self.candidate.link return rv - def identify(self) -> str: - return self.req.identify() - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Candidate): - return False - if self.req.is_named: - return self.name == other.name and self.version == other.version - return self.name == other.name and self.link == other.link - @cached_property def revision(self) -> str: - if not self.req.is_vcs: - raise AttributeError("Non-VCS candidate doesn't have revision attribute") - if self.req.revision: # type: ignore - return self.req.revision # type: ignore if not (self.ireq.source_dir and os.path.exists(self.ireq.source_dir)): # It happens because the cached wheel is hit and the source code isn't # pulled to local. In this case the link url must contain the full commit @@ -220,122 +313,110 @@ def direct_url(self) -> dict[str, Any] | None: else: return None - def prepare(self, allow_all: bool = False) -> None: + def build(self) -> str: + """Call PEP 517 build hook to build the candidate into a wheel""" + if self.wheel and self._wheel_compatible(self.wheel): + return self.wheel + self.obtain(allow_all=False) + cached = self._get_cached_wheel() + if cached: + self.wheel = cached.file_path + return self.wheel # type: ignore + assert self.ireq.source_dir, "Source directory isn't ready yet" + builder_cls = EditableBuilder if self.req.editable else WheelBuilder + builder = builder_cls(self.ireq.unpacked_source_directory, self.environment) + build_dir = self._get_wheel_dir() + if not os.path.exists(build_dir): + os.makedirs(build_dir) + self.wheel = builder.build(build_dir, metadata_directory=self._metadata_dir) + return self.wheel + + def obtain(self, allow_all: bool = False) -> None: """Fetch the link of the candidate and unpack to local if necessary. :param allow_all: If true, don't validate the wheel tag nor hashes """ - if ( - self.source_dir - or self.wheel - and self._wheel_compatible(self.wheel, allow_all) - ): - return ireq = self.ireq - if not allow_all and self.hashes: - ireq.hash_options = convert_hashes(self.hashes) + if ireq.is_wheel: + if self.wheel and self._wheel_compatible(self.wheel, allow_all): + return + elif ireq.source_dir: + return + + if not allow_all and self.candidate.hashes: + ireq.hash_options = convert_hashes(self.candidate.hashes) with self.environment.get_finder(ignore_requires_python=True) as finder: if ( - not self.link - or self.link.is_wheel - and not self._wheel_compatible(self.link.filename, allow_all) + not ireq.link + or ireq.link.is_wheel + and not self._wheel_compatible(ireq.link.filename, allow_all) ): - self.link = ireq.link = None + ireq.link = None with allow_all_wheels(allow_all): - self.link = populate_link(finder, ireq, False) - if not self.link: + ireq.link = populate_link(finder, ireq, False) + if not ireq.link: raise CandidateNotFound("No candidate is found for %s", self) + if not ireq.original_link: + ireq.original_link = ireq.link if allow_all and not self.req.editable: cached = self._get_cached_wheel() if cached: self.wheel = cached.file_path return downloader = pip_shims.Downloader(finder.session, "off") # type: ignore - self._populate_source_dir(ireq) - if not self.link.is_existing_dir(): + self._populate_source_dir() + if not ireq.link.is_existing_dir(): assert ireq.source_dir downloaded = pip_shims.unpack_url( - self.link, + ireq.link, ireq.source_dir, downloader, hashes=ireq.hashes(False), ) - if self.link.is_wheel: + if ireq.link.is_wheel: assert downloaded self.wheel = downloaded.path return - self.source_dir = ireq.unpacked_source_directory - @cached_property - def metadata(self) -> Distribution: - """Get the metadata of the candidate. + def prepare_metadata(self) -> Distribution: + """Prepare the metadata for the candidate. Will call the prepare_metadata_* hooks behind the scene """ - self.prepare(True) + self.obtain(allow_all=True) metadir_parent = create_tracked_tempdir(prefix="pdm-meta-") + result: Distribution if self.wheel: self._metadata_dir = _get_wheel_metadata_from_wheel( self.wheel, metadir_parent ) - result: Distribution = PathDistribution(Path(self._metadata_dir)) + result = PathDistribution(Path(self._metadata_dir)) else: - assert self.source_dir + source_dir = self.ireq.unpacked_source_directory builder = EditableBuilder if self.req.editable else WheelBuilder try: self._metadata_dir = builder( - self.source_dir, self.environment + source_dir, self.environment ).prepare_metadata(metadir_parent) except BuildError: termui.logger.warn( "Failed to build package, try parsing project files." ) - result = parse_metadata_from_source(self.source_dir) + result = parse_metadata_from_source(source_dir) else: result = PathDistribution(Path(self._metadata_dir)) - if not self.name: - self.name = str(result.metadata["Name"]) # type: ignore - self.req.name = self.name - if not self.version: - self.version = result.version # type: ignore + if not self.candidate.name: + self.req.name = self.candidate.name = cast(str, result.metadata["Name"]) + if not self.candidate.version: + self.candidate.version = result.version + if not self.candidate.requires_python: + self.candidate.requires_python = cast( + str, result.metadata.get("Requires-Python", "") + ) return result - def build(self) -> str: - """Call PEP 517 build hook to build the candidate into a wheel""" - self.prepare() - if self.wheel: - return self.wheel - cached = self._get_cached_wheel() - if cached: - self.wheel = cached.file_path - return self.wheel # type: ignore - assert self.source_dir, "Source directory isn't ready yet" - builder_cls = EditableBuilder if self.req.editable else WheelBuilder - builder = builder_cls(self.source_dir, self.environment) - build_dir = self._get_wheel_dir() - if not os.path.exists(build_dir): - os.makedirs(build_dir) - self.wheel = builder.build(build_dir, metadata_directory=self._metadata_dir) - return self.wheel - - def __repr__(self) -> str: - source = getattr(self.link, "comes_from", "unknown") - return f"" - - @classmethod - def from_installation_candidate( - cls, - candidate: pip_shims.InstallationCandidate, - req: Requirement, - environment: Environment, - ) -> Candidate: - """Build a candidate from pip's InstallationCandidate.""" - return cls( - req, - environment, - name=candidate.name, - version=str(candidate.version), - link=candidate.link, - ) + @property + def metadata(self) -> Distribution: + return self._metadata def get_dependencies_from_metadata(self) -> list[str]: """Get the dependencies of a candidate from metadata.""" @@ -344,79 +425,24 @@ def get_dependencies_from_metadata(self) -> list[str]: self.req.project_name, self.metadata.requires or [], extras # type: ignore ) - @property - def requires_python(self) -> str: - """The Python version constraint of the candidate.""" - if self._requires_python is not None: - return self._requires_python - assert self.link - requires_python = self.link.requires_python - if requires_python and requires_python.isdigit(): - requires_python = f">={requires_python},<{int(requires_python) + 1}" - self._requires_python = requires_python - return requires_python or "" - - @requires_python.setter - def requires_python(self, value: str) -> None: - self._requires_python = value - - @no_type_check - def as_lockfile_entry(self) -> dict[str, Any]: - """Build a lockfile entry dictionary for the candidate.""" - result = { - "name": normalize_name(self.name), - "version": str(self.version), - "extras": sorted(self.req.extras or ()), - "requires_python": str(self.requires_python), - "editable": self.req.editable, - } - project_root = self.environment.project.root.as_posix() - if self.req.is_vcs: - result.update( - { - self.req.vcs: self.req.repo, - "ref": self.req.ref, - } - ) - if not self.req.editable: - result.update(revision=self.revision) - elif not self.req.is_named: - if self.req.is_file_or_url and self.req.is_local_dir: - result.update(path=path_replace(project_root, ".", self.req.str_path)) - else: - result.update( - url=path_replace( - project_root.lstrip("/"), "${PROJECT_ROOT}", self.req.url - ) - ) - return {k: v for k, v in result.items() if v} - - def format(self) -> str: - """Format for output.""" - return ( - f"{termui.green(self.name, bold=True)} " - f"{termui.yellow(str(self.version))}" - ) - def should_cache(self) -> bool: """Determine whether to cache the dependencies and built wheel.""" + link, source_dir = self.ireq.original_link, self.ireq.source_dir if self.req.is_vcs and not self.req.editable: - if not self.ireq.source_dir: + if not source_dir: # If the candidate isn't prepared, we can't cache it return False vcs = pip_shims.VcsSupport() - assert self.link - vcs_backend = vcs.get_backend_for_scheme(self.link.scheme) + assert link + vcs_backend = vcs.get_backend_for_scheme(link.scheme) return bool( vcs_backend - and vcs_backend.is_immutable_rev_checkout( - self.link.url, self.ireq.source_dir - ) + and vcs_backend.is_immutable_rev_checkout(link.url, source_dir) ) elif self.req.is_named: return True - elif self.link and not self.link.is_existing_dir(): - base, _ = self.link.splitext() + elif link and not link.is_existing_dir(): + base, _ = link.splitext() # Cache if the link contains egg-info like 'foo-1.0' return _egg_info_re.search(base) is not None return False @@ -424,21 +450,20 @@ def should_cache(self) -> bool: def _get_cached_wheel(self) -> pip_shims.Link | None: wheel_cache = self.environment.project.make_wheel_cache() supported_tags = pip_shims.get_supported(self.environment.interpreter.for_tag()) - assert self.link + assert self.ireq.original_link cache_entry = wheel_cache.get_cache_entry( - self.link, - self.req.project_name, # type: ignore - supported_tags, + self.ireq.original_link, cast(str, self.req.project_name), supported_tags ) if cache_entry is not None: termui.logger.debug("Using cached wheel link: %s", cache_entry.link) return cache_entry.link return None - def _populate_source_dir(self, ireq: pip_shims.InstallRequirement) -> None: - assert self.link - if self.link.is_existing_dir(): - ireq.source_dir = self.link.file_path + def _populate_source_dir(self) -> None: + ireq = self.ireq + assert ireq.original_link + if ireq.original_link.is_existing_dir(): + ireq.source_dir = ireq.original_link.file_path elif self.req.editable: if self.environment.packages_path: src_dir = self.environment.packages_path / "src" @@ -456,7 +481,7 @@ def _populate_source_dir(self, ireq: pip_shims.InstallRequirement) -> None: elif not ireq.source_dir: ireq.source_dir = create_tracked_tempdir(prefix="pdm-build-") - def _wheel_compatible(self, wheel_file: str, allow_all: bool) -> bool: + def _wheel_compatible(self, wheel_file: str, allow_all: bool = False) -> bool: if allow_all: return True supported_tags = pip_shims.get_supported(self.environment.interpreter.for_tag()) @@ -465,9 +490,9 @@ def _wheel_compatible(self, wheel_file: str, allow_all: bool) -> bool: ) def _get_wheel_dir(self) -> str: - assert self.link + assert self.ireq.original_link if self.should_cache(): wheel_cache = self.environment.project.make_wheel_cache() - return wheel_cache.get_path_for_link(self.link) + return wheel_cache.get_path_for_link(self.ireq.original_link) else: return create_tracked_tempdir(prefix="pdm-wheel-") diff --git a/pdm/models/repositories.py b/pdm/models/repositories.py index 87a4670053..b0ec717b26 100644 --- a/pdm/models/repositories.py +++ b/pdm/models/repositories.py @@ -33,7 +33,8 @@ def cache_result( @wraps(func) def wrapper(self: T, candidate: Candidate) -> CandidateInfo: result = func(self, candidate) - if candidate.should_cache(): + prepared = candidate.prepared + if prepared and prepared.should_cache(): self._candidate_info_cache.set(candidate, result) return result @@ -176,8 +177,6 @@ def print_candidates( return applicable_cans def _get_dependencies_from_cache(self, candidate: Candidate) -> CandidateInfo: - if not candidate.should_cache(): - raise CandidateInfoNotFound(candidate) try: result = self._candidate_info_cache.get(candidate) except CorruptedCacheError: @@ -189,9 +188,10 @@ def _get_dependencies_from_cache(self, candidate: Candidate) -> CandidateInfo: @cache_result def _get_dependencies_from_metadata(self, candidate: Candidate) -> CandidateInfo: - deps = candidate.get_dependencies_from_metadata() + prepared = candidate.prepare(self.environment) + deps = prepared.get_dependencies_from_metadata() requires_python = candidate.requires_python - summary = candidate.metadata.metadata["Summary"] + summary = prepared.metadata.metadata["Summary"] return deps, requires_python, summary def get_hashes(self, candidate: Candidate) -> dict[str, str] | None: @@ -290,7 +290,7 @@ def _find_candidates(self, requirement: Requirement) -> Iterable[Candidate]: sources = self.get_filtered_sources(requirement) with self.environment.get_finder(sources, True) as finder, allow_all_wheels(): cans = [ - Candidate.from_installation_candidate(c, requirement, self.environment) + Candidate.from_installation_candidate(c, requirement) for c in finder.find_all_candidates(requirement.project_name) ] if not cans: @@ -373,7 +373,7 @@ def _read_lockfile(self, lockfile: Mapping[str, Any]) -> None: if k not in ("dependencies", "requires_python", "summary") } req = Requirement.from_req_dict(package_name, req_dict) - can = Candidate(req, self.environment, name=package_name, version=version) + can = Candidate(req, name=package_name, version=version) can_id = self._identify_candidate(can) self.packages[can_id] = can candidate_info: CandidateInfo = ( diff --git a/pdm/project/core.py b/pdm/project/core.py index b54078e080..37a595beb7 100644 --- a/pdm/project/core.py +++ b/pdm/project/core.py @@ -428,9 +428,7 @@ def write_lockfile( def make_self_candidate(self, editable: bool = True) -> Candidate: req = parse_requirement(pip_shims.path_to_url(self.root.as_posix()), editable) req.name = self.meta.name - return Candidate( - req, self.environment, name=self.meta.name, version=self.meta.version - ) + return Candidate(req, name=self.meta.name, version=self.meta.version) def get_content_hash(self, algo: str = "md5") -> str: # Only calculate sources and dependencies groups. Otherwise lock file is diff --git a/pdm/resolver/providers.py b/pdm/resolver/providers.py index b7801e6f60..ce09e4df5f 100644 --- a/pdm/resolver/providers.py +++ b/pdm/resolver/providers.py @@ -121,8 +121,8 @@ def get_override_candidates(self, identifier: str) -> Iterable[Candidate]: def _find_candidates(self, requirement: Requirement) -> Iterable[Candidate]: if not requirement.is_named: - can = Candidate(requirement, self.repository.environment) - can.metadata + can = Candidate(requirement) + can.prepare(self.repository.environment) return [can] else: return self.repository.find_candidates( @@ -137,9 +137,7 @@ def find_matches( ) -> Iterable[Candidate]: incompat = list(incompatibilities[identifier]) if identifier == "python": - candidates = find_python_matches( - identifier, requirements, self.repository.environment - ) + candidates = find_python_matches(identifier, requirements) return [c for c in candidates if c not in incompat] elif identifier in self.overrides: return self.get_override_candidates(identifier) diff --git a/pdm/resolver/python.py b/pdm/resolver/python.py index 0ecce30fd5..938d24366e 100644 --- a/pdm/resolver/python.py +++ b/pdm/resolver/python.py @@ -6,7 +6,6 @@ from pdm import termui from pdm.models.candidates import Candidate -from pdm.models.environment import Environment from pdm.models.requirements import NamedRequirement, Requirement from pdm.models.specifiers import PySpecSet @@ -24,14 +23,13 @@ class PythonRequirement(NamedRequirement): def from_pyspec_set(cls, spec: PySpecSet) -> "PythonRequirement": return cls(name="python", specifier=spec) - def as_candidate(self, environment: Environment) -> PythonCandidate: - return PythonCandidate(self, environment) + def as_candidate(self) -> PythonCandidate: + return PythonCandidate(self) def find_python_matches( identifier: str, requirements: Mapping[str, Iterator[Requirement]], - environment: Environment, ) -> Iterable[Candidate]: """All requires-python except for the first one(must come from the project) must be superset of the first one. @@ -40,7 +38,7 @@ def find_python_matches( project_req = next(python_reqs) python_specs = cast(Iterator[PySpecSet], (req.specifier for req in python_reqs)) if all(spec.is_superset(project_req.specifier or "") for spec in python_specs): - return [project_req.as_candidate(environment)] + return [project_req.as_candidate()] else: # There is a conflict, no match is found. return [] diff --git a/pdm/resolver/reporters.py b/pdm/resolver/reporters.py index 96db8539d7..03a2845fd9 100644 --- a/pdm/resolver/reporters.py +++ b/pdm/resolver/reporters.py @@ -58,7 +58,7 @@ def ending(self, state: State) -> None: if not can.req.is_named: can_info = can.req.url if can.req.is_vcs: - can_info = f"{can_info}@{can.revision}" + can_info = f"{can_info}@{can.get_revision()}" else: can_info = can.version logger.info(f" {k.rjust(column_width)} {can_info}") diff --git a/tests/cli/test_add.py b/tests/cli/test_add.py index 1e1f3ba8b7..ace8ec943f 100644 --- a/tests/cli/test_add.py +++ b/tests/cli/test_add.py @@ -72,7 +72,10 @@ def test_add_editable_package(project, working_set, is_dev): assert "demo" in group[0] assert "-e git+https://github.com/test-root/demo.git#egg=demo" in group[1] locked_candidates = project.locked_repository.all_candidates - assert locked_candidates["demo"].revision == "1234567890abcdef" + assert ( + locked_candidates["demo"].prepare(project.environment).revision + == "1234567890abcdef" + ) assert locked_candidates["idna"].version == "2.7" assert "idna" in working_set diff --git a/tests/conftest.py b/tests/conftest.py index 21761c4b25..b7fca25889 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -141,7 +141,6 @@ def _find_candidates(self, requirement: Requirement) -> Iterable[Candidate]: for version, candidate in self._pypi_data.get(requirement.key, {}).items(): c = Candidate( requirement, - self.environment, name=requirement.project_name, version=version, ) diff --git a/tests/models/test_candidates.py b/tests/models/test_candidates.py index d2059c9c5c..4dea5a1ff6 100644 --- a/tests/models/test_candidates.py +++ b/tests/models/test_candidates.py @@ -14,8 +14,8 @@ def test_parse_local_directory_metadata(project, is_editable): requirement_line = f"{(FIXTURES / 'projects/demo').as_posix()}" req = parse_requirement(requirement_line, is_editable) - candidate = Candidate(req, project.environment) - assert candidate.get_dependencies_from_metadata() == [ + candidate = Candidate(req) + assert candidate.prepare(project.environment).get_dependencies_from_metadata() == [ "idna", 'chardet; os_name == "nt"', ] @@ -27,14 +27,14 @@ def test_parse_local_directory_metadata(project, is_editable): def test_parse_vcs_metadata(project, is_editable): requirement_line = "git+https://github.com/test-root/demo.git@master#egg=demo" req = parse_requirement(requirement_line, is_editable) - candidate = Candidate(req, project.environment) - assert candidate.get_dependencies_from_metadata() == [ + candidate = Candidate(req) + assert candidate.prepare(project.environment).get_dependencies_from_metadata() == [ "idna", 'chardet; os_name == "nt"', ] assert candidate.name == "demo" assert candidate.version == "0.0.1" - lockfile = candidate.as_lockfile_entry() + lockfile = candidate.as_lockfile_entry(project.root) assert lockfile["ref"] == "master" if is_editable: assert "revision" not in lockfile @@ -52,8 +52,8 @@ def test_parse_vcs_metadata(project, is_editable): ) def test_parse_artifact_metadata(requirement_line, project): req = parse_requirement(requirement_line) - candidate = Candidate(req, project.environment) - assert candidate.get_dependencies_from_metadata() == [ + candidate = Candidate(req) + assert candidate.prepare(project.environment).get_dependencies_from_metadata() == [ "idna", 'chardet; os_name == "nt"', ] @@ -67,9 +67,10 @@ def test_parse_metadata_with_extras(project): f"demo[tests,security] @ file://" f"{(FIXTURES / 'artifacts/demo-0.0.1-py2.py3-none-any.whl').as_posix()}" ) - candidate = Candidate(req, project.environment) - assert candidate.ireq.is_wheel - assert sorted(candidate.get_dependencies_from_metadata()) == [ + candidate = Candidate(req) + prepared = candidate.prepare(project.environment) + assert prepared.ireq.is_wheel + assert sorted(prepared.get_dependencies_from_metadata()) == [ "pytest", 'requests; python_version >= "3.6"', ] @@ -80,9 +81,10 @@ def test_parse_remote_link_metadata(project): req = parse_requirement( "http://fixtures.test/artifacts/demo-0.0.1-py2.py3-none-any.whl" ) - candidate = Candidate(req, project.environment) - assert candidate.ireq.is_wheel - assert candidate.get_dependencies_from_metadata() == [ + candidate = Candidate(req) + prepared = candidate.prepare(project.environment) + assert prepared.ireq.is_wheel + assert prepared.get_dependencies_from_metadata() == [ "idna", 'chardet; os_name == "nt"', ] @@ -95,9 +97,10 @@ def test_extras_warning(project, recwarn): req = parse_requirement( "demo[foo] @ http://fixtures.test/artifacts/demo-0.0.1-py2.py3-none-any.whl" ) - candidate = Candidate(req, project.environment) - assert candidate.ireq.is_wheel - assert candidate.get_dependencies_from_metadata() == [] + candidate = Candidate(req) + prepared = candidate.prepare(project.environment) + assert prepared.ireq.is_wheel + assert prepared.get_dependencies_from_metadata() == [] warning = recwarn.pop(ExtrasWarning) assert str(warning.message) == "Extras not found for demo: [foo]" assert candidate.name == "demo" @@ -109,8 +112,8 @@ def test_parse_abnormal_specifiers(project): req = parse_requirement( "http://fixtures.test/artifacts/celery-4.4.2-py2.py3-none-any.whl" ) - candidate = Candidate(req, project.environment) - assert candidate.get_dependencies_from_metadata() + candidate = Candidate(req) + assert candidate.prepare(project.environment).get_dependencies_from_metadata() @pytest.mark.usefixtures("local_finder") @@ -130,12 +133,12 @@ def test_expand_project_root_in_url(req_str, core): req = parse_requirement(req_str[3:], True) else: req = parse_requirement(req_str) - candidate = Candidate(req, project.environment) - assert candidate.get_dependencies_from_metadata() == [ + candidate = Candidate(req) + assert candidate.prepare(project.environment).get_dependencies_from_metadata() == [ "idna", 'chardet; os_name == "nt"', ] - lockfile_entry = candidate.as_lockfile_entry() + lockfile_entry = candidate.as_lockfile_entry(project.root) if "path" in lockfile_entry: assert lockfile_entry["path"].startswith("./") else: @@ -145,8 +148,10 @@ def test_expand_project_root_in_url(req_str, core): @pytest.mark.usefixtures("local_finder") def test_parse_project_file_on_build_error(project): req = parse_requirement(f"{(FIXTURES / 'projects/demo-failure').as_posix()}") - candidate = Candidate(req, project.environment) - assert sorted(candidate.get_dependencies_from_metadata()) == [ + candidate = Candidate(req) + assert sorted( + candidate.prepare(project.environment).get_dependencies_from_metadata() + ) == [ 'chardet; os_name == "nt"', "idna", ] @@ -158,8 +163,8 @@ def test_parse_project_file_on_build_error(project): def test_parse_project_file_on_build_error_with_extras(project): req = parse_requirement(f"{(FIXTURES / 'projects/demo-failure').as_posix()}") req.extras = ("security", "tests") - candidate = Candidate(req, project.environment) - deps = candidate.get_dependencies_from_metadata() + candidate = Candidate(req) + deps = candidate.prepare(project.environment).get_dependencies_from_metadata() assert 'requests; python_version >= "3.6"' in deps assert "pytest" in deps assert candidate.name == "demo" @@ -169,8 +174,8 @@ def test_parse_project_file_on_build_error_with_extras(project): @pytest.mark.usefixtures("local_finder") def test_parse_project_file_on_build_error_no_dep(project): req = parse_requirement(f"{(FIXTURES / 'projects/demo-failure-no-dep').as_posix()}") - candidate = Candidate(req, project.environment) - assert candidate.get_dependencies_from_metadata() == [] + candidate = Candidate(req) + assert candidate.prepare(project.environment).get_dependencies_from_metadata() == [] assert candidate.name == "demo" assert candidate.version == "0.0.1" @@ -180,9 +185,11 @@ def test_parse_poetry_project_metadata(project, is_editable): req = parse_requirement( f"{(FIXTURES / 'projects/poetry-demo').as_posix()}", is_editable ) - candidate = Candidate(req, project.environment) + candidate = Candidate(req) requests_dep = "requests<3.0,>=2.6" - assert candidate.get_dependencies_from_metadata() == [requests_dep] + assert candidate.prepare(project.environment).get_dependencies_from_metadata() == [ + requests_dep + ] assert candidate.name == "poetry-demo" assert candidate.version == "0.1.0" @@ -192,8 +199,8 @@ def test_parse_flit_project_metadata(project, is_editable): req = parse_requirement( f"{(FIXTURES / 'projects/flit-demo').as_posix()}", is_editable ) - candidate = Candidate(req, project.environment) - deps = candidate.get_dependencies_from_metadata() + candidate = Candidate(req) + deps = candidate.prepare(project.environment).get_dependencies_from_metadata() requests_dep = "requests>=2.6" assert requests_dep in deps assert 'configparser; python_version == "2.7"' in deps @@ -208,8 +215,10 @@ def test_vcs_candidate_in_subdirectory(project, is_editable): "@master#egg=package-a&subdirectory=package-a" ) req = parse_requirement(line, is_editable) - candidate = Candidate(req, project.environment) - assert candidate.get_dependencies_from_metadata() == ["flask"] + candidate = Candidate(req) + assert candidate.prepare(project.environment).get_dependencies_from_metadata() == [ + "flask" + ] assert candidate.version == "0.1.0" line = ( @@ -217,11 +226,14 @@ def test_vcs_candidate_in_subdirectory(project, is_editable): "@master#egg=package-b&subdirectory=package-b" ) req = parse_requirement(line, is_editable) - candidate = Candidate(req, project.environment) + candidate = Candidate(req) expected_deps = ["django"] if is_editable: expected_deps.append("editables") - assert candidate.get_dependencies_from_metadata() == expected_deps + assert ( + candidate.prepare(project.environment).get_dependencies_from_metadata() + == expected_deps + ) assert candidate.version == "0.1.0" @@ -235,76 +247,70 @@ def test_sdist_candidate_with_wheel_cache(project, mocker): Path(cache_path).mkdir(parents=True) shutil.copy2(built_path, cache_path) req = parse_requirement(file_link.url) - candidate = Candidate(req, project.environment) downloader = mocker.patch("pdm.models.pip_shims.unpack_url") - candidate.prepare(True) + prepared = Candidate(req).prepare(project.environment) downloader.assert_not_called() - assert Path(candidate.wheel) == Path(cache_path) / Path(built_path).name + assert Path(prepared.wheel) == Path(cache_path) / Path(built_path).name - candidate.wheel = None + prepared.wheel = None builder = mocker.patch("pdm.builders.WheelBuilder.build") - candidate.build() + wheel = prepared.build() builder.assert_not_called() - assert Path(candidate.wheel) == Path(cache_path) / Path(built_path).name + assert Path(wheel) == Path(cache_path) / Path(built_path).name @pytest.mark.usefixtures("vcs", "local_finder") def test_cache_vcs_immutable_revision(project): req = parse_requirement("git+https://github.com/test-root/demo.git@master#egg=demo") - candidate = Candidate(req, project.environment) - wheel = candidate.build() + candidate = Candidate(req) + wheel = candidate.prepare(project.environment).build() with pytest.raises(ValueError): Path(wheel).relative_to(project.cache_dir) - assert candidate.revision == "1234567890abcdef" + assert candidate.get_revision() == "1234567890abcdef" req = parse_requirement( "git+https://github.com/test-root/demo.git@1234567890abcdef#egg=demo" ) - candidate = Candidate(req, project.environment) - wheel = candidate.build() + candidate = Candidate(req) + wheel = candidate.prepare(project.environment).build() assert Path(wheel).relative_to(project.cache_dir) - assert candidate.revision == "1234567890abcdef" + assert candidate.get_revision() == "1234567890abcdef" # test the revision can be got correctly after cached - candidate = Candidate(req, project.environment) - wheel = candidate.prepare(True) - assert not candidate.source_dir - assert candidate.revision == "1234567890abcdef" + prepared = Candidate(req).prepare(project.environment) + assert not prepared.ireq.source_dir + assert prepared.revision == "1234567890abcdef" @pytest.mark.usefixtures("local_finder") def test_cache_egg_info_sdist(project): req = parse_requirement("demo @ http://fixtures.test/artifacts/demo-0.0.1.tar.gz") - candidate = Candidate(req, project.environment) - wheel = candidate.build() + candidate = Candidate(req) + wheel = candidate.prepare(project.environment).build() assert Path(wheel).relative_to(project.cache_dir) def test_invalidate_incompatible_wheel_link(project, index): req = parse_requirement("demo") - candidate = Candidate(req, project.environment, name="demo", version="0.0.1") - candidate.prepare(True) + prepared = Candidate(req, name="demo", version="0.0.1").prepare(project.environment) assert ( - Path(candidate.wheel).name - == candidate.link.filename + Path(prepared.wheel).name + == prepared.ireq.link.filename == "demo-0.0.1-cp36-cp36m-win_amd64.whl" ) - candidate.prepare() + prepared.obtain(False) assert ( - Path(candidate.wheel).name - == candidate.link.filename + Path(prepared.wheel).name + == prepared.ireq.link.filename == "demo-0.0.1-py2.py3-none-any.whl" ) def test_legacy_pep345_tag_link(project, index): req = parse_requirement("pep345-legacy") - candidate = Candidate(req, project.environment) - try: - candidate.prepare() - except Exception: - pass + repo = project.get_repository() + candidate = next(iter(repo.find_candidates(req))) assert candidate.requires_python == ">=3,<4" diff --git a/tests/test_installer.py b/tests/test_installer.py index c041482356..dfc26a2efb 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -17,7 +17,6 @@ def test_install_wheel_with_inconsistent_dist_info(project): req = parse_requirement("pyfunctional") candidate = Candidate( req, - project.environment, link=Link("http://fixtures.test/artifacts/PyFunctional-1.4.3-py3-none-any.whl"), ) installer = InstallManager(project.environment) @@ -29,7 +28,6 @@ def test_install_with_file_existing(project): req = parse_requirement("demo") candidate = Candidate( req, - project.environment, link=Link("http://fixtures.test/artifacts/demo-0.0.1-py2.py3-none-any.whl"), ) (project.environment.packages_path / "lib/demo.py").touch() @@ -41,7 +39,6 @@ def test_uninstall_commit_rollback(project): req = parse_requirement("demo") candidate = Candidate( req, - project.environment, link=Link("http://fixtures.test/artifacts/demo-0.0.1-py2.py3-none-any.whl"), ) installer = InstallManager(project.environment) @@ -63,7 +60,6 @@ def test_rollback_after_commit(project, caplog): req = parse_requirement("demo") candidate = Candidate( req, - project.environment, link=Link("http://fixtures.test/artifacts/demo-0.0.1-py2.py3-none-any.whl"), ) installer = InstallManager(project.environment) @@ -92,7 +88,6 @@ def test_uninstall_with_console_scripts(project, use_install_cache): req = parse_requirement("celery") candidate = Candidate( req, - project.environment, link=Link("http://fixtures.test/artifacts/celery-4.4.2-py2.py3-none-any.whl"), ) installer = InstallManager(project.environment, use_install_cache=use_install_cache) @@ -111,7 +106,6 @@ def test_install_wheel_with_cache(project, invoke): req = parse_requirement("future-fstrings") candidate = Candidate( req, - project.environment, link=Link( "http://fixtures.test/artifacts/future_fstrings-1.2.0-py2.py3-none-any.whl" ), @@ -149,10 +143,7 @@ def test_url_requirement_is_not_cached(project): "future-fstrings @ http://fixtures.test/artifacts/" "future_fstrings-1.2.0-py2.py3-none-any.whl" ) - candidate = Candidate( - req, - project.environment, - ) + candidate = Candidate(req) installer = InstallManager(project.environment, use_install_cache=True) installer.install(candidate) cache_path = project.cache("packages") / "future_fstrings-1.2.0-py2.py3-none-any" @@ -169,7 +160,6 @@ def test_install_wheel_with_data_scripts(project, use_install_cache): req = parse_requirement("jmespath") candidate = Candidate( req, - project.environment, link=Link( "http://fixtures.test/artifacts/jmespath-0.10.0-py2.py3-none-any.whl" ), diff --git a/tests/test_integration.py b/tests/test_integration.py index 2ded4f2ff5..49759d0df5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -27,6 +27,7 @@ def test_basic_integration(python_version, core, tmp_path, invoke): def test_actual_list_freeze(project, local_finder, invoke): + invoke(["config", "-l", "install.parallel", "false"], obj=project, strict=True) invoke(["add", "first"], obj=project, strict=True) r = invoke(["list", "--freeze"], obj=project) assert "first==2.0.2" in r.output From 134be0fd0ddec0ead5b69e78e993486e0307bd66 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 17:19:50 +0800 Subject: [PATCH 02/13] change the default editable backend --- pdm/installers/synchronizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdm/installers/synchronizers.py b/pdm/installers/synchronizers.py index 5c6396be07..1ebe3431bf 100644 --- a/pdm/installers/synchronizers.py +++ b/pdm/installers/synchronizers.py @@ -102,7 +102,7 @@ def __init__( candidate.req.editable = None # type: ignore elif ( self.install_self - and getattr(self.environment.project.meta, "editable_backend", "editables") + and getattr(self.environment.project.meta, "editable_backend", "path") == "editables" and "editables" not in candidates ): From b683396487a0219070dec3c4b398236b8a6b0a1c Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 17:37:00 +0800 Subject: [PATCH 03/13] Fix tests --- tests/cli/test_install.py | 4 ++-- tests/test_integration.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cli/test_install.py b/tests/cli/test_install.py index 9853448886..84c01d88c0 100644 --- a/tests/cli/test_install.py +++ b/tests/cli/test_install.py @@ -72,7 +72,7 @@ def test_sync_only_different(project, working_set, capsys): working_set.add_distribution(Distribution("idna", "2.7")) actions.do_add(project, packages=["requests"]) out, _ = capsys.readouterr() - assert "4 to add" in out, out + assert "3 to add" in out, out assert "1 to update" in out assert "foo" in working_set assert "test-project" in working_set @@ -84,7 +84,7 @@ def test_sync_in_sequential_mode(project, working_set, capsys): project.project_config["install.parallel"] = False actions.do_add(project, packages=["requests"]) out, _ = capsys.readouterr() - assert "6 to add" in out + assert "5 to add" in out assert "test-project" in working_set assert working_set["chardet"].version == "3.0.4" diff --git a/tests/test_integration.py b/tests/test_integration.py index 49759d0df5..6897ce0520 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -14,7 +14,7 @@ def test_basic_integration(python_version, core, tmp_path, invoke): invoke(["init"], input="\ny\n\n\n\n\n\n>=2.7\n", obj=project, strict=True) invoke(["use", "-f", python_version], obj=project, strict=True) project._environment = None - invoke(["add", "django"] + additional_args, obj=project, strict=True) + invoke(["add", "django", "-v"] + additional_args, obj=project, strict=True) with cd(project.root): invoke(["run", "python", "foo.py"], obj=project, strict=True) if python_version != "2.7": From 0a57a6964b559faa4127a11eaf2210e2c3a014a9 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 17:37:53 +0800 Subject: [PATCH 04/13] add news --- news/920.refactor.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/920.refactor.md diff --git a/news/920.refactor.md b/news/920.refactor.md new file mode 100644 index 0000000000..ef18efc23c --- /dev/null +++ b/news/920.refactor.md @@ -0,0 +1 @@ +Extract the environment related code from `Candidate` into a new class `PreparedCandidate`. From f541403ebd2456b5de9c7d4bbd7f860c2914ab43 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 18:03:47 +0800 Subject: [PATCH 05/13] fix compatibility check --- pdm/models/candidates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pdm/models/candidates.py b/pdm/models/candidates.py index 4c0a19b1ef..01d93d1cd9 100644 --- a/pdm/models/candidates.py +++ b/pdm/models/candidates.py @@ -337,8 +337,8 @@ def obtain(self, allow_all: bool = False) -> None: :param allow_all: If true, don't validate the wheel tag nor hashes """ ireq = self.ireq - if ireq.is_wheel: - if self.wheel and self._wheel_compatible(self.wheel, allow_all): + if self.wheel: + if self._wheel_compatible(self.wheel, allow_all): return elif ireq.source_dir: return From 05bdabe5c9b38e7dbb401ccaf884c1141b82ccd0 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 18:16:23 +0800 Subject: [PATCH 06/13] tell me why --- tests/models/test_candidates.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/models/test_candidates.py b/tests/models/test_candidates.py index 4dea5a1ff6..6f9a15814d 100644 --- a/tests/models/test_candidates.py +++ b/tests/models/test_candidates.py @@ -299,7 +299,14 @@ def test_invalidate_incompatible_wheel_link(project, index): == "demo-0.0.1-cp36-cp36m-win_amd64.whl" ) + print( + "Before obtain:", prepared.wheel, prepared._is_wheel_compatible(prepared.wheel) + ) prepared.obtain(False) + print( + "After obtain:", prepared.wheel, prepared._is_wheel_compatible(prepared.wheel) + ) + assert ( Path(prepared.wheel).name == prepared.ireq.link.filename From 034cf6d05abb53e8a46f09ddc91de2f231e22e3d Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 18:22:06 +0800 Subject: [PATCH 07/13] Control the concurrency --- .github/workflows/ci.yml | 4 ++++ tests/models/test_candidates.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d9fad02ed..d2e105c808 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,10 @@ on: - "*.md" - Dockerfile +concurrency: + group: ${{ github.event.number || github.run_id }} + cancel-in-progress: true + jobs: Testing: runs-on: ${{ matrix.os }} diff --git a/tests/models/test_candidates.py b/tests/models/test_candidates.py index 6f9a15814d..c40781eadf 100644 --- a/tests/models/test_candidates.py +++ b/tests/models/test_candidates.py @@ -300,11 +300,17 @@ def test_invalidate_incompatible_wheel_link(project, index): ) print( - "Before obtain:", prepared.wheel, prepared._is_wheel_compatible(prepared.wheel) + "Before obtain:", + prepared.wheel, + prepared.ireq.link.filename, + prepared._wheel_compatible(prepared.wheel), ) prepared.obtain(False) print( - "After obtain:", prepared.wheel, prepared._is_wheel_compatible(prepared.wheel) + "After obtain:", + prepared.wheel, + prepared.ireq.link.filename, + prepared._wheel_compatible(prepared.wheel), ) assert ( From aae63ea01ee912cd6d7f898711a6318af3b58fb5 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 18:36:08 +0800 Subject: [PATCH 08/13] why? --- tests/models/test_candidates.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/models/test_candidates.py b/tests/models/test_candidates.py index c40781eadf..8c04951cfe 100644 --- a/tests/models/test_candidates.py +++ b/tests/models/test_candidates.py @@ -4,6 +4,7 @@ import pytest from pdm.exceptions import ExtrasWarning +from pdm.models import pip_shims from pdm.models.candidates import Candidate from pdm.models.pip_shims import Link, path_to_url from pdm.models.requirements import parse_requirement @@ -303,14 +304,16 @@ def test_invalidate_incompatible_wheel_link(project, index): "Before obtain:", prepared.wheel, prepared.ireq.link.filename, - prepared._wheel_compatible(prepared.wheel), + prepared._wheel_compatible(prepared.wheel, False), + pip_shims.PipWheel.supported, ) prepared.obtain(False) print( "After obtain:", prepared.wheel, prepared.ireq.link.filename, - prepared._wheel_compatible(prepared.wheel), + prepared._wheel_compatible(prepared.wheel, False), + pip_shims.PipWheel.supported, ) assert ( From bcff99e929cea596e52fdb487e51c4bd322fc74a Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 18:52:05 +0800 Subject: [PATCH 09/13] monkey patch leaks --- pdm/utils.py | 2 ++ tests/models/test_candidates.py | 16 ---------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/pdm/utils.py b/pdm/utils.py index e3265b795b..6721fe68b1 100644 --- a/pdm/utils.py +++ b/pdm/utils.py @@ -206,7 +206,9 @@ def _find_most_preferred_tag( PipWheel.support_index_min = _wheel_support_index_min if has_find_most_preferred_tag: PipWheel.find_most_preferred_tag = _find_most_preferred_tag + print("Monkey patching wheel", _wheel_supported) yield + print("Restoring wheel monkey patching", original_wheel_supported) PipWheel.supported = original_wheel_supported PipWheel.support_index_min = original_support_index_min if has_find_most_preferred_tag: diff --git a/tests/models/test_candidates.py b/tests/models/test_candidates.py index 8c04951cfe..4dea5a1ff6 100644 --- a/tests/models/test_candidates.py +++ b/tests/models/test_candidates.py @@ -4,7 +4,6 @@ import pytest from pdm.exceptions import ExtrasWarning -from pdm.models import pip_shims from pdm.models.candidates import Candidate from pdm.models.pip_shims import Link, path_to_url from pdm.models.requirements import parse_requirement @@ -300,22 +299,7 @@ def test_invalidate_incompatible_wheel_link(project, index): == "demo-0.0.1-cp36-cp36m-win_amd64.whl" ) - print( - "Before obtain:", - prepared.wheel, - prepared.ireq.link.filename, - prepared._wheel_compatible(prepared.wheel, False), - pip_shims.PipWheel.supported, - ) prepared.obtain(False) - print( - "After obtain:", - prepared.wheel, - prepared.ireq.link.filename, - prepared._wheel_compatible(prepared.wheel, False), - pip_shims.PipWheel.supported, - ) - assert ( Path(prepared.wheel).name == prepared.ireq.link.filename From a75dafa62fe011376825469827ee27fda20c3455 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 19:06:17 +0800 Subject: [PATCH 10/13] try to fix monkeypatch leak --- pdm/utils.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/pdm/utils.py b/pdm/utils.py index 6721fe68b1..6357426687 100644 --- a/pdm/utils.py +++ b/pdm/utils.py @@ -162,6 +162,18 @@ def join_list_with(items: list[Any], sep: Any) -> list[Any]: return new_items[:-1] +original_wheel_supported = PipWheel.supported +original_support_index_min = PipWheel.support_index_min +_has_find_most_preferred_tag = ( + getattr(PipWheel, "find_most_preferred_tag", None) is not None +) + +if _has_find_most_preferred_tag: + original_find: Any = PipWheel.find_most_preferred_tag +else: + original_find = None + + @no_type_check @contextmanager def allow_all_wheels(enable: bool = True) -> Iterator: @@ -172,8 +184,6 @@ def allow_all_wheels(enable: bool = True) -> Iterator: and set a new one, or else the results from the previous non-patched calls will interfere. """ - from pdm.models.pip_shims import PipWheel - if not enable: yield return @@ -191,27 +201,16 @@ def _find_most_preferred_tag( ) -> int: return 0 - has_find_most_preferred_tag = ( - getattr(PipWheel, "find_most_preferred_tag", None) is not None - ) - - original_wheel_supported = PipWheel.supported - original_support_index_min = PipWheel.support_index_min - if has_find_most_preferred_tag: - original_find = PipWheel.find_most_preferred_tag - else: - original_find = None - PipWheel.supported = _wheel_supported PipWheel.support_index_min = _wheel_support_index_min - if has_find_most_preferred_tag: + if _has_find_most_preferred_tag: PipWheel.find_most_preferred_tag = _find_most_preferred_tag print("Monkey patching wheel", _wheel_supported) yield print("Restoring wheel monkey patching", original_wheel_supported) PipWheel.supported = original_wheel_supported PipWheel.support_index_min = original_support_index_min - if has_find_most_preferred_tag: + if _has_find_most_preferred_tag: PipWheel.find_most_preferred_tag = original_find From cc099f17a980970b94d5fd6073c3150b9e167fbd Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 19:19:08 +0800 Subject: [PATCH 11/13] fix tests --- pdm/utils.py | 2 -- tests/test_integration.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pdm/utils.py b/pdm/utils.py index 6357426687..f2f500fbed 100644 --- a/pdm/utils.py +++ b/pdm/utils.py @@ -205,9 +205,7 @@ def _find_most_preferred_tag( PipWheel.support_index_min = _wheel_support_index_min if _has_find_most_preferred_tag: PipWheel.find_most_preferred_tag = _find_most_preferred_tag - print("Monkey patching wheel", _wheel_supported) yield - print("Restoring wheel monkey patching", original_wheel_supported) PipWheel.supported = original_wheel_supported PipWheel.support_index_min = original_support_index_min if _has_find_most_preferred_tag: diff --git a/tests/test_integration.py b/tests/test_integration.py index 6897ce0520..55ed40c755 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -11,7 +11,7 @@ def test_basic_integration(python_version, core, tmp_path, invoke): project = core.create_project(tmp_path) project.root.joinpath("foo.py").write_text("import django\n") additional_args = ["--no-self"] if python_version == "2.7" else [] - invoke(["init"], input="\ny\n\n\n\n\n\n>=2.7\n", obj=project, strict=True) + invoke(["init"], input="\ny\n\n\n\n\n\n\n", obj=project, strict=True) invoke(["use", "-f", python_version], obj=project, strict=True) project._environment = None invoke(["add", "django", "-v"] + additional_args, obj=project, strict=True) From b4e42cdb591f584174a39923a482134b14933ad7 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 19:29:35 +0800 Subject: [PATCH 12/13] fix integration test --- tests/test_integration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 55ed40c755..21f9a8ba21 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -11,8 +11,9 @@ def test_basic_integration(python_version, core, tmp_path, invoke): project = core.create_project(tmp_path) project.root.joinpath("foo.py").write_text("import django\n") additional_args = ["--no-self"] if python_version == "2.7" else [] - invoke(["init"], input="\ny\n\n\n\n\n\n\n", obj=project, strict=True) invoke(["use", "-f", python_version], obj=project, strict=True) + invoke(["init", "-n"], obj=project, strict=True) + project.meta["name"] = "test-project" project._environment = None invoke(["add", "django", "-v"] + additional_args, obj=project, strict=True) with cd(project.root): From 0e96838064ff9f70d15e88154812aac240f3527c Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 17 Feb 2022 19:44:52 +0800 Subject: [PATCH 13/13] don't build for wheel links --- pdm/models/candidates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pdm/models/candidates.py b/pdm/models/candidates.py index 01d93d1cd9..d7d50548f6 100644 --- a/pdm/models/candidates.py +++ b/pdm/models/candidates.py @@ -315,9 +315,9 @@ def direct_url(self) -> dict[str, Any] | None: def build(self) -> str: """Call PEP 517 build hook to build the candidate into a wheel""" - if self.wheel and self._wheel_compatible(self.wheel): - return self.wheel self.obtain(allow_all=False) + if self.wheel: + return self.wheel cached = self._get_cached_wheel() if cached: self.wheel = cached.file_path