Merge pull request #541 from pdm-project/build-env

news/541.dep.md

@@ -0,0 +1 @@
+Switch from `toml` to `tomli`.

news/541.feature.md

@@ -0,0 +1 @@
+Seperate the build env into two different levels for better caching.

pdm/builders/base.py

@@ -10,7 +10,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Iterable, Mapping
 
-import toml
+import tomli
 from pep517.wrappers import Pep517HookCaller
 from pip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet
 
@@ -88,11 +88,15 @@ def log_subprocessor(
 
 
 class _Prefix:
-    def __init__(self, executable: str, path: str) -> None:
-        self.path = path
-        paths = get_sys_config_paths(executable, vars={"base": path, "platbase": path})
-        self.bin_dir = paths["scripts"]
-        self.lib_dirs = [paths["platlib"], paths["purelib"]]
+    def __init__(self, executable: str, shared: str, overlay: str) -> None:
+        self.bin_dirs: list[str] = []
+        self.lib_dirs: list[str] = []
+        for path in (overlay, shared):
+            paths = get_sys_config_paths(
+                executable, vars={"base": path, "platbase": path}
+            )
+            self.bin_dirs.append(paths["scripts"])
+            self.lib_dirs.extend([paths["platlib"], paths["purelib"]])
         self.site_dir = os.path.join(path, "site")
         if not os.path.isdir(self.site_dir):
             os.makedirs(self.site_dir)
@@ -116,35 +120,45 @@ def __init__(self, executable: str, path: str) -> None:
                 """
                 )
             )
+        self.shared = shared
+        self.overlay = overlay
 
 
 class EnvBuilder:
     """A simple PEP 517 builder for an isolated environment"""
 
-    _env_cache: dict[str, str] = {}
+    _shared_envs: dict[int, str] = {}
+    _overlay_envs: dict[str, str] = {}
 
     DEFAULT_BACKEND = {
         "build-backend": "setuptools.build_meta:__legacy__",
         "requires": ["setuptools >= 40.8.0", "wheel"],
     }
 
     @classmethod
-    def get_env_path(cls, src_dir: str | Path) -> str:
-        key = os.path.normpath(src_dir).rstrip("\\/")
-        if key not in cls._env_cache:
-            cls._env_cache[key] = create_tracked_tempdir(prefix="pdm-build-env-")
-        return cls._env_cache[key]
+    def get_shared_env(cls, key: int) -> str:
+        if key in cls._shared_envs:
+            logger.debug("Reusing shared build env: %s", cls._shared_envs[key])
+            return cls._shared_envs[key]
+        # Postpone the cache after installation is done
+        return create_tracked_tempdir("-shared", "pdm-build-env-")
+
+    @classmethod
+    def get_overlay_env(cls, key: str) -> str:
+        if key not in cls._overlay_envs:
+            cls._overlay_envs[key] = create_tracked_tempdir(
+                "-overlay", "pdm-build-env-"
+            )
+        return cls._overlay_envs[key]
 
     def __init__(self, src_dir: str | Path, environment: Environment) -> None:
         self._env = environment
-        self._path = self.get_env_path(src_dir)
         self.executable = self._env.interpreter.executable
         self.src_dir = src_dir
-        self._prefix = _Prefix(self.executable, self._path)
         logger.debug("Preparing isolated env for PEP 517 build...")
         try:
             with open(os.path.join(src_dir, "pyproject.toml"), encoding="utf8") as f:
-                spec = toml.load(f)
+                spec = tomli.load(f)
         except FileNotFoundError:
             spec = {}
         except Exception as e:
@@ -159,6 +173,12 @@ def __init__(self, src_dir: str | Path, environment: Environment) -> None:
 
         self._backend = self._build_system["build-backend"]
 
+        self._prefix = _Prefix(
+            self.executable,
+            shared=self.get_shared_env(hash(frozenset(self._build_system["requires"]))),
+            overlay=self.get_overlay_env(os.path.normcase(self.src_dir).rstrip("\\/")),
+        )
+
         self._hook = Pep517HookCaller(
             src_dir,
             self._backend,
@@ -169,7 +189,7 @@ def __init__(self, src_dir: str | Path, environment: Environment) -> None:
 
     @property
     def _env_vars(self) -> dict[str, str]:
-        paths = [self._prefix.bin_dir]
+        paths = self._prefix.bin_dirs
         if "PATH" in os.environ:
             paths.append(os.getenv("PATH", ""))
         return {
@@ -205,10 +225,11 @@ def check_requirements(self, reqs: Iterable[str]) -> Iterable[str]:
             raise BuildError(f"Conflicting requirements: {', '.join(conflicting)}")
         return missing
 
-    def install(self, requirements: Iterable[str]) -> None:
+    def install(self, requirements: Iterable[str], shared: bool = False) -> None:
         missing = self.check_requirements(requirements)
         if not missing:
             return
+        path = self._prefix.shared if shared else self._prefix.overlay
 
         with tempfile.NamedTemporaryFile(
             "w+", prefix="pdm-build-reqs-", suffix=".txt", delete=False
@@ -219,13 +240,20 @@ def install(self, requirements: Iterable[str]) -> None:
                 "install",
                 "--ignore-installed",
                 "--prefix",
-                self._path,
+                path,
             ]
             cmd.extend(prepare_pip_source_args(self._env.project.sources))
             cmd.extend(["-r", req_file.name])
             self.subprocess_runner(cmd, isolated=False)
             os.unlink(req_file.name)
 
+        if shared:
+            # The shared env is prepared and is safe to be cached now. This is to make
+            # sure no broken env is returned when run in parallel mode.
+            key = hash(frozenset(requirements))
+            if key not in self._shared_envs:
+                self._shared_envs[key] = path
+
     def build(
         self, out_dir: str, config_settings: Mapping[str, Any] | None = None
     ) -> str:

pdm/builders/sdist.py

@@ -10,7 +10,7 @@ class EnvSdistBuilder(EnvBuilder):
     def build(
         self, out_dir: str, config_settings: Optional[Mapping[str, Any]] = None
     ) -> str:
-        self.install(self._build_system["requires"])
+        self.install(self._build_system["requires"], shared=True)
         requires = self._hook.get_requires_for_build_sdist(config_settings)
         self.install(requires)
         filename = self._hook.build_sdist(out_dir, config_settings)

pdm/builders/wheel.py

@@ -10,7 +10,7 @@ class EnvWheelBuilder(EnvBuilder):
     def build(
         self, out_dir: str, config_settings: Optional[Mapping[str, Any]] = None
     ) -> str:
-        self.install(self._build_system["requires"])
+        self.install(self._build_system["requires"], shared=True)
         requires = self._hook.get_requires_for_build_wheel(config_settings)
         self.install(requires)
         filename = self._hook.build_wheel(out_dir, config_settings)

pdm/cli/completions/pdm.ps1

@@ -131,7 +131,7 @@ def get_packages(lines):
     return [PACKAGE_REGEX.match(line).group() for line in lines]
 
 with open('pyproject.toml', encoding='utf8') as f:
-    data = toml.load(f)
+    data = tomli.load(f)
 packages = get_packages(data.get('project', {}).get('dependencies', []))
 for reqs in data.get('project', {}).get('optional-dependencies', {}).values():
     packages.extend(get_packages(reqs))

pdm/cli/completions/pdm.zsh

@@ -339,7 +339,7 @@ def get_packages(lines):
     return [PACKAGE_REGEX.match(line).group() for line in lines]
 
 with open('pyproject.toml', encoding='utf8') as f:
-    data = toml.load(f)
+    data = tomli.load(f)
 packages = get_packages(data.get('project', {}).get('dependencies', []))
 for reqs in data.get('project', {}).get('optional-dependencies', {}).values():
     packages.extend(get_packages(reqs))

pdm/cli/utils.py

@@ -430,7 +430,7 @@ def format_resolution_impossible(err: ResolutionImpossible) -> str:
     result.append(
         "Please make sure the package names are correct. If so, you can either "
         "loosen the version constraints of these dependencies, or "
-        "set a narrower `requires-python` range in the pyproject.toml."
+        "set a narrower `requires-python` range in the pyproject.tomli."
     )
     return "\n".join(result)
 

pdm/formats/flit.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, Mapping, Optional, Tuple, cast
 
-import toml
+import tomli
 
 from pdm.formats.base import (
     MetaConverter,
@@ -22,8 +22,8 @@
 def check_fingerprint(project: Optional[Project], filename: PathLike) -> bool:
     with open(filename, encoding="utf-8") as fp:
         try:
-            data = toml.load(fp)
-        except toml.TomlDecodeError:
+            data = tomli.load(fp)
+        except tomli.TOMLDecodeError:
             return False
 
     return "tool" in data and "flit" in data["tool"]
@@ -147,7 +147,7 @@ def convert(
         os.path.dirname(os.path.abspath(filename))
     ):
         converter = FlitMetaConverter(
-            toml.load(fp)["tool"]["flit"], project.core.ui if project else None
+            tomli.load(fp)["tool"]["flit"], project.core.ui if project else None
         )
         return converter.convert()
 

pdm/formats/legacy.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union, cast
 
-import toml
+import tomli
 
 from pdm._types import RequirementDict, Source
 from pdm.formats.base import (
@@ -22,8 +22,8 @@
 def check_fingerprint(project: Project, filename: PathLike) -> bool:
     with open(filename, encoding="utf-8") as fp:
         try:
-            data = toml.load(fp)
-        except toml.TomlDecodeError:
+            data = tomli.load(fp)
+        except tomli.TOMLDecodeError:
             return False
 
     return (
@@ -170,7 +170,7 @@ def convert(
     project: Project, filename: Path, options: Optional[Namespace]
 ) -> Tuple[Mapping[str, Any], Mapping[str, Any]]:
     with open(filename, encoding="utf-8") as fp:
-        converter = LegacyMetaConverter(toml.load(fp)["tool"]["pdm"], project.core.ui)
+        converter = LegacyMetaConverter(tomli.load(fp)["tool"]["pdm"], project.core.ui)
         return converter.convert()
 
 

pdm/formats/pipfile.py

@@ -7,7 +7,7 @@
 from os import PathLike
 from typing import Any
 
-import toml
+import tomli
 from packaging.markers import default_environment
 
 from pdm._types import RequirementDict
@@ -45,7 +45,7 @@ def convert(
     project: Project, filename: PathLike, options: Namespace | None
 ) -> tuple[dict[str, Any], dict[str, Any]]:
     with open(filename, encoding="utf-8") as fp:
-        data = toml.load(fp)
+        data = tomli.load(fp)
     result = {}
     settings = {}
     if "pipenv" in data:

pdm/formats/poetry.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Mapping
 
-import toml
+import tomli
 
 from pdm._types import RequirementDict, Source
 from pdm.formats.base import (
@@ -32,8 +32,8 @@
 def check_fingerprint(project: Project | None, filename: Path | str) -> bool:
     with open(filename, encoding="utf-8") as fp:
         try:
-            data = toml.load(fp)
-        except toml.TomlDecodeError:
+            data = tomli.load(fp)
+        except tomli.TOMLDecodeError:
             return False
 
     return "tool" in data and "poetry" in data["tool"]
@@ -202,7 +202,7 @@ def convert(
         os.path.dirname(os.path.abspath(filename))
     ):
         converter = PoetryMetaConverter(
-            toml.load(fp)["tool"]["poetry"], project.core.ui if project else None
+            tomli.load(fp)["tool"]["poetry"], project.core.ui if project else None
         )
         return converter.convert()
 

pdm/formats/requirements.py

@@ -79,11 +79,11 @@ def parse_requirement_file(
 
 
 def check_fingerprint(project: Project, filename: PathLike) -> bool:
-    import toml
+    import tomli
 
     with open(filename, encoding="utf-8") as fp:
         try:
-            toml.load(fp)
+            tomli.load(fp)
         except ValueError:
             # the file should be a requirements.txt if it not a TOML document.
             return True

pyproject.toml

@@ -10,7 +10,7 @@ requires-python = ">=3.7"
 license = {text = "MIT"}
 dependencies = [
     "appdirs",
-    "atoml~=1.0",
+    "atoml>=1.0.3",
     "click>=7",
     "distlib>=0.3.1",
     "importlib-metadata; python_version < \"3.8\"",
@@ -22,6 +22,7 @@ dependencies = [
     "resolvelib>=0.7.0,<0.8.0",
     "shellingham<2.0.0,>=1.3.2",
     "wheel<1.0.0,>=0.36.2",
+    "tomli~=1.0",
 ]
 name = "pdm"
 description = "Python Development Master"