Add venv-based build isolation behind a flag

src/pip/_internal/build_env/__init__.py

@@ -0,0 +1,19 @@
+"""Build Environment used for isolation during sdist building
+"""
+
+import sys
+
+if sys.version_info >= (3, 8):
+    from typing import Literal
+else:
+    from pip._vendor.typing_extensions import Literal
+
+from pip._internal.build_env._base import BuildEnvironment as BuildEnvironment
+from pip._internal.build_env._base import get_runnable_pip as get_runnable_pip
+from pip._internal.build_env._custom import (
+    CustomBuildEnvironment as CustomBuildEnvironment,
+)
+from pip._internal.build_env._noop import NoOpBuildEnvironment as NoOpBuildEnvironment
+from pip._internal.build_env._venv import VenvBuildEnvironment as VenvBuildEnvironment
+
+BuildIsolationMode = Literal["noop", "custom", "venv"]

src/pip/_internal/build_env/_base.py

@@ -0,0 +1,130 @@
+import abc
+import logging
+import os
+import pathlib
+from types import TracebackType
+from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple, Type
+
+from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.version import Version
+
+from pip import __file__ as pip_location
+from pip._internal.metadata import get_default_environment, get_environment
+from pip._internal.utils.logging import VERBOSE, getLogger
+
+if TYPE_CHECKING:
+    from pip._internal.index.package_finder import PackageFinder
+
+logger = getLogger(__name__)
+
+
+def get_runnable_pip() -> str:
+    """Get a file to pass to a Python executable, to run the currently-running pip.
+
+    This is used to run a pip subprocess, for installing requirements into the build
+    environment.
+    """
+    source = pathlib.Path(pip_location).resolve().parent
+
+    if not source.is_dir():
+        # This would happen if someone is using pip from inside a zip file. In that
+        # case, we can use that directly.
+        return str(source)
+
+    return os.fsdecode(source / "__pip-runner__.py")
+
+
+def iter_install_flags(finder: "PackageFinder") -> Iterable[str]:
+    logging_level = logger.getEffectiveLevel()
+    if logging_level <= logging.DEBUG:
+        yield "-vv"
+    elif logging_level <= VERBOSE:
+        yield "-v"
+
+    for format_control in ("no_binary", "only_binary"):
+        formats = getattr(finder.format_control, format_control)
+        format_control_key = format_control.replace("_", "-")
+        yield f"--{format_control_key}"
+        yield ",".join(sorted(formats)) or ":none:"
+
+    index_urls = finder.index_urls
+    if index_urls:
+        yield "--index-url"
+        yield index_urls[0]
+        for extra_index in index_urls[1:]:
+            yield "--extra-index-url"
+            yield extra_index
+    else:
+        yield "--no-index"
+    for link in finder.find_links:
+        yield "--find-links"
+        yield link
+
+    for host in finder.trusted_hosts:
+        yield "--trusted-host"
+        yield host
+    if finder.allow_all_prereleases:
+        yield "--pre"
+    if finder.prefer_binary:
+        yield "--prefer-binary"
+
+
+class BuildEnvironment(metaclass=abc.ABCMeta):
+    lib_dirs: List[str]
+
+    def __init__(self) -> None:
+        ...
+
+    @abc.abstractmethod
+    def __enter__(self) -> None:
+        ...
+
+    @abc.abstractmethod
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        ...
+
+    def check_requirements(
+        self, reqs: Iterable[str]
+    ) -> Tuple[Set[Tuple[str, str]], Set[str]]:
+        missing = set()
+        conflicting = set()
+        if reqs:
+            env = (
+                get_environment(self.lib_dirs)
+                if self.lib_dirs
+                else get_default_environment()
+            )
+            for req_str in reqs:
+                req = Requirement(req_str)
+                # We're explicitly evaluating with an empty extra value, since build
+                # environments are not provided any mechanism to select specific extras.
+                if req.marker is not None and not req.marker.evaluate({"extra": ""}):
+                    continue
+                dist = env.get_distribution(req.name)
+                if not dist:
+                    missing.add(req_str)
+                    continue
+                if isinstance(dist.version, Version):
+                    installed_req_str = f"{req.name}=={dist.version}"
+                else:
+                    installed_req_str = f"{req.name}==={dist.version}"
+                if not req.specifier.contains(dist.version, prereleases=True):
+                    conflicting.add((installed_req_str, req_str))
+                # FIXME: Consider direct URL?
+        return conflicting, missing
+
+    @abc.abstractmethod
+    def install_requirements(
+        self,
+        finder: "PackageFinder",
+        requirements: Iterable[str],
+        prefix_as_string: str,
+        *,
+        kind: str,
+    ) -> None:
+        raise NotImplementedError()

src/pip/_internal/build_env.py → src/pip/_internal/build_env/_custom.py

@@ -1,35 +1,30 @@
-"""Build Environment used for isolation during sdist building
-"""
-
 import logging
 import os
-import pathlib
 import site
 import sys
 import textwrap
 from collections import OrderedDict
 from sysconfig import get_paths
 from types import TracebackType
-from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple, Type
+from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Type
 
 from pip._vendor.certifi import where
-from pip._vendor.packaging.requirements import Requirement
-from pip._vendor.packaging.version import Version
 
-from pip import __file__ as pip_location
 from pip._internal.cli.spinners import open_spinner
 from pip._internal.locations import (
     get_isolated_environment_lib_paths,
     get_platlib,
     get_purelib,
 )
-from pip._internal.metadata import get_default_environment, get_environment
 from pip._internal.utils.subprocess import call_subprocess
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
 
+from ._base import BuildEnvironment, get_runnable_pip, iter_install_flags
+
 if TYPE_CHECKING:
     from pip._internal.index.package_finder import PackageFinder
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -44,22 +39,6 @@ def __init__(self, path: str) -> None:
         self.lib_dirs = get_isolated_environment_lib_paths(path)
 
 
-def get_runnable_pip() -> str:
-    """Get a file to pass to a Python executable, to run the currently-running pip.
-
-    This is used to run a pip subprocess, for installing requirements into the build
-    environment.
-    """
-    source = pathlib.Path(pip_location).resolve().parent
-
-    if not source.is_dir():
-        # This would happen if someone is using pip from inside a zip file. In that
-        # case, we can use that directly.
-        return str(source)
-
-    return os.fsdecode(source / "__pip-runner__.py")
-
-
 def _get_system_sitepackages() -> Set[str]:
     """Get system site packages
 
@@ -80,7 +59,7 @@ def _get_system_sitepackages() -> Set[str]:
     return {os.path.normcase(path) for path in system_sites}
 
 
-class BuildEnvironment:
+class CustomBuildEnvironment(BuildEnvironment):
     """Creates and manages an isolated environment to install build deps"""
 
     def __init__(self) -> None:
@@ -92,10 +71,10 @@ def __init__(self) -> None:
         )
 
         self._bin_dirs: List[str] = []
-        self._lib_dirs: List[str] = []
+        self.lib_dirs: List[str] = []
         for prefix in reversed(list(self._prefixes.values())):
             self._bin_dirs.append(prefix.bin_dir)
-            self._lib_dirs.extend(prefix.lib_dirs)
+            self.lib_dirs.extend(prefix.lib_dirs)
 
         # Customize site to:
         # - ensure .pth files are honored
@@ -134,7 +113,7 @@ def __init__(self) -> None:
                     assert not path in sys.path
                     site.addsitedir(path)
                 """
-                ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)
+                ).format(system_sites=system_sites, lib_dirs=self.lib_dirs)
             )
 
     def __enter__(self) -> None:
@@ -170,40 +149,6 @@ def __exit__(
             else:
                 os.environ[varname] = old_value
 
-    def check_requirements(
-        self, reqs: Iterable[str]
-    ) -> Tuple[Set[Tuple[str, str]], Set[str]]:
-        """Return 2 sets:
-        - conflicting requirements: set of (installed, wanted) reqs tuples
-        - missing requirements: set of reqs
-        """
-        missing = set()
-        conflicting = set()
-        if reqs:
-            env = (
-                get_environment(self._lib_dirs)
-                if hasattr(self, "_lib_dirs")
-                else get_default_environment()
-            )
-            for req_str in reqs:
-                req = Requirement(req_str)
-                # We're explicitly evaluating with an empty extra value, since build
-                # environments are not provided any mechanism to select specific extras.
-                if req.marker is not None and not req.marker.evaluate({"extra": ""}):
-                    continue
-                dist = env.get_distribution(req.name)
-                if not dist:
-                    missing.add(req_str)
-                    continue
-                if isinstance(dist.version, Version):
-                    installed_req_str = f"{req.name}=={dist.version}"
-                else:
-                    installed_req_str = f"{req.name}==={dist.version}"
-                if not req.specifier.contains(dist.version, prereleases=True):
-                    conflicting.add((installed_req_str, req_str))
-                # FIXME: Consider direct URL?
-        return conflicting, missing
-
     def install_requirements(
         self,
         finder: "PackageFinder",
@@ -217,62 +162,20 @@ def install_requirements(
         prefix.setup = True
         if not requirements:
             return
-        self._install_requirements(
-            get_runnable_pip(),
-            finder,
-            requirements,
-            prefix,
-            kind=kind,
-        )
 
-    @staticmethod
-    def _install_requirements(
-        pip_runnable: str,
-        finder: "PackageFinder",
-        requirements: Iterable[str],
-        prefix: _Prefix,
-        *,
-        kind: str,
-    ) -> None:
-        args: List[str] = [
+        args = [
             sys.executable,
-            pip_runnable,
+            get_runnable_pip(),
             "install",
             "--ignore-installed",
             "--no-user",
             "--prefix",
             prefix.path,
             "--no-warn-script-location",
+            *iter_install_flags(finder),
+            "--",
+            *requirements,
         ]
-        if logger.getEffectiveLevel() <= logging.DEBUG:
-            args.append("-v")
-        for format_control in ("no_binary", "only_binary"):
-            formats = getattr(finder.format_control, format_control)
-            args.extend(
-                (
-                    "--" + format_control.replace("_", "-"),
-                    ",".join(sorted(formats or {":none:"})),
-                )
-            )
-
-        index_urls = finder.index_urls
-        if index_urls:
-            args.extend(["-i", index_urls[0]])
-            for extra_index in index_urls[1:]:
-                args.extend(["--extra-index-url", extra_index])
-        else:
-            args.append("--no-index")
-        for link in finder.find_links:
-            args.extend(["--find-links", link])
-
-        for host in finder.trusted_hosts:
-            args.extend(["--trusted-host", host])
-        if finder.allow_all_prereleases:
-            args.append("--pre")
-        if finder.prefer_binary:
-            args.append("--prefer-binary")
-        args.append("--")
-        args.extend(requirements)
         extra_environ = {"_PIP_STANDALONE_CERT": where()}
         with open_spinner(f"Installing {kind}") as spinner:
             call_subprocess(
@@ -281,34 +184,3 @@ def _install_requirements(
                 spinner=spinner,
                 extra_environ=extra_environ,
             )
-
-
-class NoOpBuildEnvironment(BuildEnvironment):
-    """A no-op drop-in replacement for BuildEnvironment"""
-
-    def __init__(self) -> None:
-        pass
-
-    def __enter__(self) -> None:
-        pass
-
-    def __exit__(
-        self,
-        exc_type: Optional[Type[BaseException]],
-        exc_val: Optional[BaseException],
-        exc_tb: Optional[TracebackType],
-    ) -> None:
-        pass
-
-    def cleanup(self) -> None:
-        pass
-
-    def install_requirements(
-        self,
-        finder: "PackageFinder",
-        requirements: Iterable[str],
-        prefix_as_string: str,
-        *,
-        kind: str,
-    ) -> None:
-        raise NotImplementedError()