revert: don't cache the decompressed content of wheels unless being told so

news/2803.feature.md

@@ -0,0 +1 @@
+Revert the package cache introduced in 2.13. Don't cache the decompressed contents of wheels unless being told so.

src/pdm/installers/installers.py

@@ -148,32 +148,28 @@ def _get_link_method(cache_method: str) -> LinkMethod:
     return "copy"
 
 
-def install_package(
-    package: CachedPackage,
+def install_wheel(
+    wheel: Path,
     environment: BaseEnvironment,
     direct_url: dict[str, Any] | None = None,
-    install_links: bool = True,
+    install_links: bool = False,
     rename_pth: bool = False,
 ) -> str:
     """Only create .pth files referring to the cached package.
     If the cache doesn't exist, create one.
     """
     interpreter = str(environment.interpreter.executable)
     script_kind = environment.script_kind
-    # the cache_method can be any of "symlink", "hardlink" and "pth"
+    # the cache_method can be any of "symlink", "hardlink", "copy" and "pth"
     cache_method: str = environment.project.config["install.cache_method"]
-    dist_name = package.path.name.split("-")[0]
+    dist_name = wheel.name.split("-")[0]
     link_method: LinkMethod | None
     if not install_links or dist_name == "editables":
         link_method = "copy"
     else:
         link_method = _get_link_method(cache_method)
 
     additional_metadata: dict[str, bytes] = {}
-    if link_method == "symlink":
-        # Track usage when symlink is used
-        additional_metadata["REFER_TO"] = package.path.as_posix().encode()
-
     if direct_url is not None:
         additional_metadata["direct_url.json"] = json.dumps(direct_url, indent=2).encode()
 
@@ -184,10 +180,18 @@ def install_package(
         link_method=link_method,
         rename_pth=rename_pth,
     )
-    source = PackageWheelSource(package)
-    dist_info_dir = install(source, destination=destination, additional_metadata=additional_metadata)
-    if link_method == "symlink":
-        package.add_referrer(dist_info_dir)
+    if install_links:
+        package = environment.project.package_cache.cache_wheel(wheel)
+        source = PackageWheelSource(package)
+        if link_method == "symlink":
+            # Track usage when symlink is used
+            additional_metadata["REFER_TO"] = package.path.as_posix().encode()
+        dist_info_dir = install(source, destination=destination, additional_metadata=additional_metadata)
+        if link_method == "symlink":
+            package.add_referrer(dist_info_dir)
+    else:
+        with WheelFile.open(wheel) as source:
+            dist_info_dir = install(source, destination=destination, additional_metadata=additional_metadata)
     return dist_info_dir
 
 
@@ -202,17 +206,3 @@ def install(
     _install(source, destination, additional_metadata=additional_metadata or {})
     root_scheme = _process_WHEEL_file(source)
     return os.path.join(destination.scheme_dict[root_scheme], source.dist_info_dir)
-
-
-def install_wheel(wheel: str, environment: BaseEnvironment, direct_url: dict[str, Any] | None = None) -> str:
-    """Install a wheel into the environment, return the .dist-info path"""
-    destination = SchemeDictionaryDestination(
-        scheme_dict=environment.get_paths(_get_dist_name(wheel)),
-        interpreter=str(environment.interpreter.executable),
-        script_kind=environment.script_kind,
-    )
-    additional_metadata = {}
-    if direct_url is not None:
-        additional_metadata["direct_url.json"] = json.dumps(direct_url, indent=2).encode()
-    with WheelFile.open(wheel) as source:
-        return install(source, destination, additional_metadata=additional_metadata)

src/pdm/installers/manager.py

@@ -5,7 +5,7 @@
 from pdm import termui
 from pdm.compat import Distribution
 from pdm.exceptions import UninstallError
-from pdm.installers.installers import install_package
+from pdm.installers.installers import install_wheel
 from pdm.installers.uninstallers import BaseRemovePaths, StashedRemovePaths
 
 if TYPE_CHECKING:
@@ -29,8 +29,8 @@
     def install(self, candidate: Candidate) -> Distribution:
         """Install a candidate into the environment, return the distribution"""
         prepared = candidate.prepare(self.environment)
-        dist_info = install_package(
-            prepared.get_cached_package(),
+        dist_info = install_wheel(
+            prepared.build(),
             self.environment,
             direct_url=prepared.direct_url(),
             install_links=self.use_install_cache,
@@ -51,7 +51,7 @@
             remove_path.remove()
             remove_path.commit()
         except OSError as e:
-            termui.logger.info("Error occurred during uninstallation, roll back the changes now.")
+            termui.logger.warn("Error occurred during uninstallation, roll back the changes now.")
             remove_path.rollback()
             raise UninstallError(e) from e
 
@@ -67,5 +67,6 @@
             paths_to_remove.remove()
             paths_to_remove.commit()
         except OSError as e:
-            termui.logger.info("Error occurred during overwriting, roll back the changes now.")
+            termui.logger.warn("Error occurred during overwriting, roll back the changes now.")
+            paths_to_remove.rollback()
             raise UninstallError(e) from e

src/pdm/models/caches.py

@@ -18,7 +18,7 @@
 from pdm.models.cached_package import CachedPackage
 from pdm.models.candidates import Candidate
 from pdm.termui import logger
-from pdm.utils import atomic_open_for_write, create_tracked_tempdir, get_file_hash
+from pdm.utils import atomic_open_for_write, create_tracked_tempdir
 
 if TYPE_CHECKING:
     from packaging.tags import Tag
@@ -275,22 +275,18 @@ class PackageCache:
     def __init__(self, root: Path) -> None:
         self.root = root
 
-    def cache_wheel(self, wheel: Path, checksum: str | None = None) -> CachedPackage:
+    def cache_wheel(self, wheel: Path) -> CachedPackage:
         """Create a CachedPackage instance from a wheel file"""
         import zipfile
 
         from installer.utils import make_file_executable
 
-        if checksum is None:
-            checksum = get_file_hash(wheel)
-        parts = (checksum[:2],)  # shard by the first two chars of the checksum
-        dest = self.root.joinpath(*parts, f"{wheel.name}.cache")
+        dest = self.root.joinpath(f"{wheel.name}.cache")
         pkg = CachedPackage(dest, original_wheel=wheel)
         if dest.exists():
             return pkg
         dest.mkdir(parents=True, exist_ok=True)
         with pkg.lock():
-            dest.joinpath(".checksum").write_text(checksum)
             logger.info("Unpacking wheel into cached location %s", dest)
             with zipfile.ZipFile(wheel) as zf:
                 try:
@@ -305,35 +301,18 @@ def cache_wheel(self, wheel: Path, checksum: str | None = None) -> CachedPackage
                     raise
         return pkg
 
-    def iter_packages(self) -> Iterable[tuple[str, CachedPackage]]:
+    def iter_packages(self) -> Iterable[CachedPackage]:
         for path in self.root.rglob("*.whl.cache"):
-            hash_parts = path.parent.relative_to(self.root).parts
             p = CachedPackage(path)
             with p.lock():  # ensure the package is not being created
                 pass
-            if not path.joinpath(".checksum").exists():
-                checksum = "".join(hash_parts)
-                path.joinpath(".checksum").write_text(checksum)
-            else:
-                checksum = p.checksum
-            yield checksum, p
+            yield p
 
     def cleanup(self) -> int:
         """Remove unused cached packages"""
         count = 0
-        for _, pkg in self.iter_packages():
+        for pkg in self.iter_packages():
             if not any(os.path.exists(fn) for fn in pkg.referrers):
                 pkg.cleanup()
                 count += 1
         return count
-
-    def match_link(self, link: Link) -> CachedPackage | None:
-        """Match a link to a cached package"""
-        if not link.is_wheel:
-            return None
-        given_hash = link_hashes.get("sha256", []) if (link_hashes := link.hash_option) else []
-        for checksum, p in self.iter_packages():
-            if p.path.stem == link.filename and (not given_hash or checksum in given_hash):
-                logger.debug("Using package from cache location: %s", p.path)
-                return p
-        return None

src/pdm/models/candidates.py

@@ -18,7 +18,6 @@
 from pdm.compat import importlib_metadata as im
 from pdm.exceptions import BuildError, CandidateNotFound, InvalidPyVersion, PDMWarning
 from pdm.models.backends import get_backend, get_backend_by_spec
-from pdm.models.cached_package import CachedPackage
 from pdm.models.reporter import BaseReporter
 from pdm.models.requirements import (
     FileRequirement,
@@ -34,7 +33,6 @@
     comparable_version,
     convert_hashes,
     filtered_sources,
-    get_file_hash,
     get_rev_from_url,
     normalize_name,
     path_to_url,
@@ -317,7 +315,7 @@
         self.req = self.candidate.req
         self.link = self._replace_url_vars(self.candidate.link)
 
-        self._cached: CachedPackage | None = None
+        self._cached: Path | None = None
         self._source_dir: Path | None = None
         self._unpacked_dir: Path | None = None
         self._metadata_dir: str | None = None
@@ -399,27 +397,13 @@
         else:
             return None
 
-    def get_cached_package(self) -> CachedPackage:
-        """Call PEP 517 build hook to build the candidate into a cached package"""
+    def build(self) -> Path:
+        """Call PEP 517 build hook to build the candidate into a wheel"""
         self._obtain(allow_all=False)
         if self._cached:
             return self._cached
-        wheel = self.build()
-        checksum_path = Path(f"{wheel}.sha256")
-        if checksum_path.exists():
-            checksum = checksum_path.read_text().strip()
-        else:
-            checksum = None
-        self._cached = self.environment.project.package_cache.cache_wheel(wheel, checksum)
-        return self._cached
-
-    def build(self) -> Path:
-        """Call PEP 517 build hook to build the candidate into a wheel"""
-        self._obtain(allow_all=False, prefer_wheel=True)
-        if self._cached and self._cached.original_wheel:
-            return self._cached.original_wheel
         if not self.req.editable:
-            cached, checksum = self._get_build_cache()
+            cached = self._get_build_cache()
             if cached:
                 return cached
         assert self._source_dir, "Source directory isn't ready yet"
@@ -429,23 +413,18 @@
         os.makedirs(build_dir, exist_ok=True)
         termui.logger.info("Running PEP 517 backend to build a wheel for %s", self.link)
         self.reporter.report_build_start(self.link.filename)  # type: ignore[union-attr]
-        wheel = Path(builder.build(build_dir, metadata_directory=self._metadata_dir))
-        checksum = get_file_hash(wheel)
-        with open(f"{wheel}.sha256", "w") as f:
-            f.write(checksum)
+        self._cached = Path(builder.build(build_dir, metadata_directory=self._metadata_dir))
         self.reporter.report_build_end(self.link.filename)  # type: ignore[union-attr]
-        return wheel
+        return self._cached
 
-    def _obtain(self, allow_all: bool = False, unpack: bool = True, prefer_wheel: bool = False) -> None:
+    def _obtain(self, allow_all: bool = False, unpack: bool = True) -> None:
         """Fetch the link of the candidate and unpack to local if necessary.
 
         :param allow_all: If true, don't validate the wheel tag nor hashes
         :param unpack: Whether to download and unpack the link if it's not local
-        :param prefer_wheel: Prefer wheel over cached package
         """
-        if self._cached and self._wheel_compatible(self._cached.path.stem, allow_all):
-            if not prefer_wheel or self._cached.original_wheel:
-                return
+        if self._cached and self._wheel_compatible(self._cached.name, allow_all):
+            return
 
         if self._source_dir and self._source_dir.exists():
             return
@@ -468,17 +447,11 @@
                     )
                 if not self.candidate.link:
                     self.candidate.link = self.link
-        # Find if there is already a CachedPackage for the link
-        package = self.environment.project.package_cache.match_link(self.link)
-        if package is not None:
-            self._cached = package
-            if not prefer_wheel:
-                return
-        # If not, find if there is any build cache for the candidate
+        # find if there is any build cache for the candidate
         if allow_all and not self.req.editable:
-            cached, checksum = self._get_build_cache()
+            cached = self._get_build_cache()
             if cached:
-                self._cached = self.environment.project.package_cache.cache_wheel(cached, checksum)
+                self._cached = cached
                 return
         # If not, download and unpack the link
         if unpack:
@@ -505,17 +478,18 @@
                     unpack_reporter=self.reporter.report_unpack,
                 )
         if self.link.is_wheel:
-            checksum = hashes["sha256"][0] if (hashes := self.link.hash_option) and "sha256" in hashes else None
-            self._cached = self.environment.project.package_cache.cache_wheel(result, checksum)
+            self._cached = result
         else:
             self._source_dir = Path(build_dir)
             self._unpacked_dir = result
 
     def prepare_metadata(self, force_build: bool = False) -> im.Distribution:
         self._obtain(allow_all=True, unpack=False)
+        if self._metadata_dir:
+            return im.PathDistribution(Path(self._metadata_dir))
 
         if self._cached:
-            return self._get_metadata_from_cached(self._cached)
+            return self._get_metadata_from_wheel(self._cached)
 
         assert self.link is not None
         if self.link.dist_info_metadata:
@@ -526,7 +500,7 @@
 
         self._unpack(validate_hashes=False)
         if self._cached:  # check again if the wheel is downloaded to local
-            return self._get_metadata_from_cached(self._cached)
+            return self._get_metadata_from_wheel(self._cached)
 
         assert self._unpacked_dir, "Source directory isn't ready yet"
         pyproject_toml = self._unpacked_dir / "pyproject.toml"
@@ -550,9 +524,11 @@
                 return None
         return MetadataDistribution(resp.text)
 
-    def _get_metadata_from_cached(self, cached: CachedPackage) -> im.Distribution:
+    def _get_metadata_from_wheel(self, wheel: Path) -> im.Distribution:
         # Get metadata from METADATA inside the wheel
-        return im.PathDistribution(cached.dist_info)
+        metadata_parent = self.environment.project.core.create_temp_dir(prefix="pdm-meta-")
+        dist_info = self._metadata_dir = _get_wheel_metadata_from_wheel(wheel, metadata_parent)
+        return im.PathDistribution(Path(dist_info))
 
     def _get_metadata_from_project(self, pyproject_toml: Path) -> im.Distribution | None:
         # Try getting from PEP 621 metadata
@@ -667,19 +643,13 @@
             return _egg_info_re.search(link.filename) is not None
         return False
 
-    def _get_build_cache(self) -> tuple[Path | None, str | None]:
+    def _get_build_cache(self) -> Path | None:
         wheel_cache = self.environment.project.make_wheel_cache()
         assert self.candidate.link
         cache_entry = wheel_cache.get(self.candidate.link, self.candidate.name, self.environment.target_python)
         if cache_entry is not None:
             termui.logger.info("Using cached wheel: %s", cache_entry)
-            if (hash_file := cache_entry.with_name(f"{cache_entry.name}.sha256")).exists():
-                cache_hash = hash_file.read_text().strip()
-            else:
-                cache_hash = get_file_hash(cache_entry)
-                hash_file.write_text(cache_hash)
-            return cache_entry, cache_hash
-        return cache_entry, None
+        return cache_entry
 
     def _get_build_dir(self) -> str:
         original_link = self.candidate.link

src/pdm/pytest.py

@@ -314,7 +314,7 @@ def build_env(build_env_wheels: Iterable[Path], tmp_path_factory: pytest.TempPat
     p = Core().create_project(d)
     env = PythonEnvironment(p, prefix=str(d), python=sys.executable)
     for wheel in build_env_wheels:
-        install_wheel(str(wheel), env)
+        install_wheel(wheel, env)
     return d
 
 

tests/cli/test_install.py

@@ -121,7 +121,7 @@ def test_sync_with_index_change(project, index, pdm):
     </body>
     </html>
     """
-    pdm(["lock"], obj=project, strict=True)
+    pdm(["lock"], obj=project, strict=True, cleanup=False)
     file_hashes = project.lockfile["package"][0]["files"]
     assert [e["hash"] for e in file_hashes] == [
         "sha256:90e49598b553d8746c4dc7d9442e0359d038c3039d802c91c0a55505da318c63"