Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Faster build and installation of packages #6205

Merged
merged 8 commits into from
Feb 5, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add a wheel builder
sdispater authored and radoering committed Feb 5, 2023
commit eefb804664f715204d930e3d01f7a65d8ded79a2
44 changes: 42 additions & 2 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 13 additions & 11 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -50,6 +50,7 @@ python = "^3.7"
poetry-core = "1.5.0"
poetry-plugin-export = "^1.3.0"
"backports.cached-property" = { version = "^1.0.2", python = "<3.8" }
build = "^0.10.0"
cachecontrol = { version = "^0.12.9", extras = ["filecache"] }
cleo = "^2.0.0"
crashtest = "^0.4.1"
@@ -66,6 +67,7 @@ packaging = ">=20.4"
pexpect = "^4.7.0"
pkginfo = "^1.9.4"
platformdirs = "^2.5.2"
pyproject-hooks = "^1.0.0"
requests = "^2.18"
requests-toolbelt = ">=0.9.1,<0.11.0"
shellingham = "^1.5"
@@ -167,22 +169,22 @@ enable_error_code = [
# warning.
[[tool.mypy.overrides]]
module = [
'poetry.console.commands.self.show.plugins',
'poetry.plugins.plugin_manager',
'poetry.repositories.installed_repository',
'poetry.utils.env',
'poetry.console.commands.self.show.plugins',
'poetry.plugins.plugin_manager',
'poetry.repositories.installed_repository',
'poetry.utils.env',
]
warn_unused_ignores = false

[[tool.mypy.overrides]]
module = [
'cachecontrol.*',
'lockfile.*',
'pexpect.*',
'requests_toolbelt.*',
'shellingham.*',
'virtualenv.*',
'xattr.*',
'cachecontrol.*',
'lockfile.*',
'pexpect.*',
'requests_toolbelt.*',
'shellingham.*',
'virtualenv.*',
'xattr.*',
]
ignore_missing_imports = true

8 changes: 6 additions & 2 deletions src/poetry/config/config.py
Original file line number Diff line number Diff line change
@@ -100,7 +100,6 @@ def validator(cls, policy: str) -> bool:

logger = logging.getLogger(__name__)


_default_config: Config | None = None


@@ -124,7 +123,11 @@ class Config:
"prefer-active-python": False,
"prompt": "{project_name}-py{python_version}",
},
"experimental": {"new-installer": True, "system-git-client": False},
"experimental": {
"new-installer": True,
"system-git-client": False,
"wheel-installer": True,
},
"installer": {"parallel": True, "max-workers": None, "no-binary": None},
}

@@ -267,6 +270,7 @@ def _get_normalizer(name: str) -> Callable[[str], Any]:
"virtualenvs.options.prefer-active-python",
"experimental.new-installer",
"experimental.system-git-client",
"experimental.wheel-installer",
"installer.parallel",
}:
return boolean_normalizer
151 changes: 151 additions & 0 deletions src/poetry/installation/chef.py
Original file line number Diff line number Diff line change
@@ -2,28 +2,179 @@

import hashlib
import json
import tarfile
import tempfile
import zipfile

from contextlib import redirect_stdout
from io import StringIO
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Callable
from typing import Collection

from build import BuildBackendException
from build import ProjectBuilder
from build.env import IsolatedEnv as BaseIsolatedEnv
from poetry.core.utils.helpers import temporary_directory
from pyproject_hooks import quiet_subprocess_runner # type: ignore[import]

from poetry.installation.chooser import InvalidWheelName
from poetry.installation.chooser import Wheel
from poetry.utils.env import ephemeral_environment


if TYPE_CHECKING:
from contextlib import AbstractContextManager

from poetry.core.packages.utils.link import Link

from poetry.config.config import Config
from poetry.utils.env import Env


class ChefError(Exception):
...


class ChefBuildError(ChefError):
...


class IsolatedEnv(BaseIsolatedEnv):
def __init__(self, env: Env, config: Config) -> None:
self._env = env
self._config = config

@property
def executable(self) -> str:
return str(self._env.python)

@property
def scripts_dir(self) -> str:
return str(self._env._bin_dir)

def install(self, requirements: Collection[str]) -> None:
from cleo.io.null_io import NullIO
from poetry.core.packages.dependency import Dependency
from poetry.core.packages.project_package import ProjectPackage

from poetry.config.config import Config
from poetry.factory import Factory
from poetry.installation.installer import Installer
from poetry.packages.locker import Locker
from poetry.repositories.installed_repository import InstalledRepository

# We build Poetry dependencies from the requirements
package = ProjectPackage("__root__", "0.0.0")
package.python_versions = ".".join(str(v) for v in self._env.version_info[:3])
for requirement in requirements:
dependency = Dependency.create_from_pep_508(requirement)
package.add_dependency(dependency)

pool = Factory.create_pool(self._config)
installer = Installer(
NullIO(),
self._env,
package,
Locker(self._env.path.joinpath("poetry.lock"), {}),
pool,
Config.create(),
InstalledRepository.load(self._env),
)
installer.update(True)
installer.run()


class Chef:
def __init__(self, config: Config, env: Env) -> None:
self._config = config
self._env = env
self._cache_dir = (
Path(config.get("cache-dir")).expanduser().joinpath("artifacts")
)

def prepare(self, archive: Path, output_dir: Path | None = None) -> Path:
if not self._should_prepare(archive):
return archive

if archive.is_dir():
tmp_dir = tempfile.mkdtemp(prefix="poetry-chef-")

return self._prepare(archive, Path(tmp_dir))

return self._prepare_sdist(archive, destination=output_dir)

def _prepare(self, directory: Path, destination: Path) -> Path:
with ephemeral_environment(self._env.python) as venv:
env = IsolatedEnv(venv, self._config)
builder = ProjectBuilder(
directory,
python_executable=env.executable,
scripts_dir=env.scripts_dir,
runner=quiet_subprocess_runner,
)
env.install(builder.build_system_requires)
env.install(
builder.build_system_requires | builder.get_requires_for_build("wheel")
)

stdout = StringIO()
with redirect_stdout(stdout):
try:
return Path(
builder.build(
"wheel",
destination.as_posix(),
)
)
except BuildBackendException as e:
raise ChefBuildError(str(e))

def _prepare_sdist(self, archive: Path, destination: Path | None = None) -> Path:
from poetry.core.packages.utils.link import Link

suffix = archive.suffix
context: Callable[
[str], AbstractContextManager[zipfile.ZipFile | tarfile.TarFile]
]
if suffix == ".zip":
context = zipfile.ZipFile
else:
context = tarfile.open

with temporary_directory() as tmp_dir:
with context(archive.as_posix()) as archive_archive:
archive_archive.extractall(tmp_dir)

archive_dir = Path(tmp_dir)

elements = list(archive_dir.glob("*"))

if len(elements) == 1 and elements[0].is_dir():
sdist_dir = elements[0]
else:
sdist_dir = archive_dir / archive.name.rstrip(suffix)
if not sdist_dir.is_dir():
sdist_dir = archive_dir

if destination is None:
destination = self.get_cache_directory_for_link(Link(archive.as_uri()))

destination.mkdir(parents=True, exist_ok=True)

return self._prepare(
sdist_dir,
destination,
)

def _should_prepare(self, archive: Path) -> bool:
return archive.is_dir() or not self._is_wheel(archive)

@classmethod
def _is_wheel(cls, archive: Path) -> bool:
return archive.suffix == ".whl"

def get_cached_archive_for_link(self, link: Link) -> Path | None:
archives = self.get_cached_archives_for_link(link)
if not archives:
Loading