From 064dc9daba81177386d520c8f36183689fc03ff7 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Thu, 27 Aug 2020 03:24:08 +0200 Subject: [PATCH] Add support for building pinned artifacts This change allows poetry projects to be configured via pyproject.toml to optionally enable version pinning and also to include nested dependencies for application wheels. --- poetry/core/json/schemas/poetry-schema.json | 26 ++ poetry/core/lock/__init__.py | 0 poetry/core/lock/categories.py | 11 + poetry/core/lock/locker.py | 439 ++++++++++++++++++++ poetry/core/masonry/builders/builder.py | 2 +- poetry/core/masonry/builders/sdist.py | 1 + poetry/core/masonry/metadata.py | 28 +- poetry/core/packages/dependency.py | 19 +- poetry/core/poetry.py | 31 +- 9 files changed, 545 insertions(+), 12 deletions(-) create mode 100644 poetry/core/lock/__init__.py create mode 100644 poetry/core/lock/categories.py create mode 100644 poetry/core/lock/locker.py diff --git a/poetry/core/json/schemas/poetry-schema.json b/poetry/core/json/schemas/poetry-schema.json index 81664910f..d544b3a54 100644 --- a/poetry/core/json/schemas/poetry-schema.json +++ b/poetry/core/json/schemas/poetry-schema.json @@ -578,6 +578,9 @@ }, "script": { "$ref": "#/definitions/build-script" + }, + "metadata": { + "$ref": "#/definitions/build-metadata" } } }, @@ -586,6 +589,29 @@ {"$ref": "#/definitions/build-script"}, {"$ref": "#/definitions/build-config"} ] + }, + "build-metadata-dependencies": { + "type": "object", + "description": "Package dependency metadata configuration", + "properties": { + "lock": { + "type": "boolean", + "description": "Lock (pin) all dependencies", + "default": false + }, + "nested": { + "type": "boolean", + "description": "Include nested dependencies", + "default": false + } + } + }, + "build-metadata": { + "type": "object", + "description": "Build metadata configuration", + "properties": { + "dependencies": {"$ref": "#/definitions/build-metadata-dependencies"} + } } } } diff --git a/poetry/core/lock/__init__.py b/poetry/core/lock/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poetry/core/lock/categories.py b/poetry/core/lock/categories.py new file mode 100644 index 000000000..c55d8a73c --- /dev/null +++ b/poetry/core/lock/categories.py @@ -0,0 +1,11 @@ +from enum import Enum + + +class LockCategory(Enum): + MAIN = "main" + DEV = "dev" + + def __eq__(self, other): + if not isinstance(other, Enum): + return self.value == other + super(LockCategory, self).__eq__(other) diff --git a/poetry/core/lock/locker.py b/poetry/core/lock/locker.py new file mode 100644 index 000000000..0d94ffb1f --- /dev/null +++ b/poetry/core/lock/locker.py @@ -0,0 +1,439 @@ +import itertools +import json +import logging +import re + +from copy import deepcopy +from hashlib import sha256 +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Union + +from tomlkit import document +from tomlkit import inline_table +from tomlkit import item +from tomlkit import table +from tomlkit.container import Container as TOMLContainer +from tomlkit.exceptions import TOMLKitError + +from poetry.core.lock.categories import LockCategory +from poetry.core.packages.package import Dependency +from poetry.core.packages.package import Package +from poetry.core.pyproject import PyProjectTOML +from poetry.core.semver import parse_constraint +from poetry.core.semver.version import Version +from poetry.core.toml import TOMLFile +from poetry.core.utils._compat import Path # noqa +from poetry.core.version.markers import parse_marker + + +logger = logging.getLogger(__name__) + + +Data = Union[Dict[str, Any], TOMLContainer] + + +class Locker(object): + + _VERSION = "1.1" + + _relevant_keys = ["dependencies", "dev-dependencies", "source", "extras"] + + def __init__(self, lock, local_config=None): # type: (Path, Optional[Data]) -> None + self._lock = TOMLFile(lock) + self._local_config = local_config + self._lock_data = None + self._content_hash = self._get_content_hash() + + @property + def lock(self): # type: () -> TOMLFile + return self._lock + + @property + def lock_data(self): + if self._lock_data is None: + self._lock_data = self._get_lock_data() + + return self._lock_data + + def is_locked(self): # type: () -> bool + """ + Checks whether the locker has been locked (lockfile found). + """ + if not self._lock.exists(): + return False + + return "package" in self.lock_data + + def is_fresh(self): # type: () -> bool + """ + Checks whether the lock file is still up to date with the current hash. + """ + lock = self._lock.read() + metadata = lock.get("metadata", {}) + + if "content-hash" in metadata: + return self._content_hash == lock["metadata"]["content-hash"] + + return False + + def get_packages( + self, names=None, categories=None + ): # type: (Optional[List[str]], Optional[List[LockCategory]]) -> List[Package] + """ + Get locked packages. Filters by categories if specified. + + :param names: Package names to filter on. + :param categories: Package categories to filter on. + """ + packages = [] + + if not self.is_locked(): + return packages + + locked_packages = [ + pkg + for pkg in self.lock_data["package"] + if (names is None or pkg["name"] in names) + and (categories is None or pkg["category"] in categories) + ] + lock_metadata = self.lock_data["metadata"] + + for info in locked_packages: + packages.append(self._load_package(info, lock_metadata)) + + return packages + + def get_project_dependencies( + self, project_requires, pinned_versions=False, with_nested=False + ): # type: (List[Dependency], bool, bool) -> Any + packages = self.get_packages(categories=[LockCategory.MAIN]) + + # group packages entries by name, this is required because requirement might use + # different constraints + packages_by_name = {} + for pkg in packages: + if pkg.name not in packages_by_name: + packages_by_name[pkg.name] = [] + packages_by_name[pkg.name].append(pkg) + + def __get_locked_package( + _dependency, + ): # type: (Dependency) -> Optional[Package] + """ + Internal helper to identify corresponding locked package using dependency + version constraints. + """ + for _package in packages_by_name.get(_dependency.name, []): + if _dependency.constraint.allows(_package.version): + return _package + return None + + project_level_dependencies = set() + dependencies = [] + + for dependency in project_requires: + dependency = deepcopy(dependency) + if pinned_versions: + locked_package = __get_locked_package(dependency) + if locked_package: + dependency.constraint = locked_package.to_dependency().constraint + project_level_dependencies.add(dependency.name) + dependencies.append(dependency) + + if not with_nested: + # return only with project level dependencies + return dependencies + + nested_dependencies = [] + + for pkg in packages: # type: Package + for requirement in pkg.requires: # type: Dependency + if requirement.name in project_level_dependencies: + # project level dependencies take precedence + continue + + if pinned_versions: + requirement.constraint = ( + __get_locked_package(requirement).to_dependency().constraint + ) + + # dependencies use extra to indicate that it was activated via parent + # package's extras + marker = requirement.marker.without_extras() + for project_requirement in project_requires: + if ( + pkg.name == project_requirement.name + and project_requirement.constraint.allows(pkg.version) + ): + requirement.marker = marker.intersect( + project_requirement.marker + ) + break + else: + # this dependency was not from a project requirement + requirement.marker = marker.intersect(pkg.marker) + + if requirement not in nested_dependencies: + nested_dependencies.append(requirement) + + return sorted( + itertools.chain(dependencies, nested_dependencies), + key=lambda x: x.name.lower(), + ) + + def set_lock_data(self, root, packages): # type: (...) -> bool + files = table() + packages = self._lock_packages(packages) + # Retrieving hashes + for package in packages: + if package["name"] not in files: + files[package["name"]] = [] + + for f in package["files"]: + file_metadata = inline_table() + for k, v in sorted(f.items()): + file_metadata[k] = v + + files[package["name"]].append(file_metadata) # noqa + + if files[package["name"]]: + files[package["name"]] = item(files[package["name"]]).multiline(True) + + del package["files"] + + lock = document() + lock["package"] = packages + + if root.extras: + lock["extras"] = { + extra: [dep.pretty_name for dep in deps] + for extra, deps in root.extras.items() + } + + lock["metadata"] = { + "lock-version": self._VERSION, + "python-versions": root.python_versions, + "content-hash": self._content_hash, + "files": files, + } + + if not self.is_locked() or lock != self.lock_data: + self._write_lock_data(lock) + + return True + + return False + + def _write_lock_data(self, data): + self.lock.write(data) + + # Checking lock file data consistency + if data != self.lock.read(): + raise RuntimeError("Inconsistent lock file data.") + + self._lock_data = None + + def _get_content_hash(self): # type: () -> str + """ + Returns the sha256 hash of the sorted content of the pyproject file. + """ + if self._local_config is None: + return "" + + content = self._local_config + + relevant_content = {} + for key in self._relevant_keys: + relevant_content[key] = content.get(key) + + content_hash = sha256( + json.dumps(relevant_content, sort_keys=True).encode() + ).hexdigest() + + return content_hash + + def _get_lock_data(self): # type: () -> dict + if not self._lock.exists(): + raise RuntimeError("No lockfile found. Unable to read locked packages") + + try: + lock_data = self._lock.read() + except TOMLKitError as e: + raise RuntimeError("Unable to read the lock file ({}).".format(e)) + + lock_version = Version.parse(lock_data["metadata"].get("lock-version", "1.0")) + current_version = Version.parse(self._VERSION) + # We expect the locker to be able to read lock files + # from the same semantic versioning range + accepted_versions = parse_constraint( + "^{}".format(Version(current_version.major, 0)) + ) + lock_version_allowed = accepted_versions.allows(lock_version) + if lock_version_allowed and current_version < lock_version: + logger.warning( + "The lock file might not be compatible with the current version of Poetry.\n" + "Upgrade Poetry to ensure the lock file is read properly or, alternatively, " + "regenerate the lock file with the `poetry lock` command." + ) + elif not lock_version_allowed: + raise RuntimeError( + "The lock file is not compatible with the current version of Poetry.\n" + "Upgrade Poetry to be able to read the lock file or, alternatively, " + "regenerate the lock file with the `poetry lock` command." + ) + + return lock_data + + def _lock_packages(self, packages): # type: (List[Package]) -> list + locked = [] + + for package in sorted(packages, key=lambda x: x.name): + spec = self._dump_package(package) + + locked.append(spec) + + return locked + + @staticmethod + def _load_package(info, lock_metadata): # type: (Data, Data) -> Package + package = Package( + name=info["name"], version=info["version"], pretty_version=info["version"] + ) + package.description = info.get("description", "") + package.category = info["category"] + package.optional = info["optional"] + if lock_metadata and "hashes" in lock_metadata: + # Old lock so we create dummy files from the hashes + package.files = [ + {"name": h, "hash": h} for h in lock_metadata["hashes"][info["name"]] + ] + else: + package.files = lock_metadata["files"][info["name"]] + + package.python_versions = info["python-versions"] + extras = info.get("extras", {}) + if extras: + for name, deps in extras.items(): + package.extras[name] = [] + + for dep in deps: + m = re.match(r"^(.+?)(?:\s+\((.+)\))?$", dep) + dep_name = m.group(1) + constraint = m.group(2) or "*" + + package.extras[name].append(Dependency(dep_name, constraint)) + + if "marker" in info: + package.marker = parse_marker(info["marker"]) + else: + # Compatibility for old locks + if "requirements" in info: + dep = Dependency("foo", "0.0.0") + for name, value in info["requirements"].items(): + if name == "python": + dep.python_versions = value + elif name == "platform": + dep.platform = value + + split_dep = dep.to_pep_508(False).split(";") + if len(split_dep) > 1: + package.marker = parse_marker(split_dep[1].strip()) + + for dep_name, constraint in info.get("dependencies", {}).items(): + if isinstance(constraint, list): + for c in constraint: + package.add_dependency(dep_name, c) + + continue + + package.add_dependency(dep_name, constraint) + + if "develop" in info: + package.develop = info["develop"] + + if "source" in info: + package.source_type = info["source"].get("type", "") + package.source_url = info["source"]["url"] + package.source_reference = info["source"]["reference"] + + return package + + @staticmethod + def _dump_package(package): # type: (Package) -> dict + dependencies = {} + for dependency in sorted(package.requires, key=lambda d: d.name): + if dependency.is_optional() and not dependency.is_activated(): + continue + + if dependency.pretty_name not in dependencies: + dependencies[dependency.pretty_name] = [] + + constraint = inline_table() + constraint["version"] = str(dependency.pretty_constraint) + + if dependency.extras: + constraint["extras"] = sorted(dependency.extras) + + if dependency.is_optional(): + constraint["optional"] = True + + if not dependency.marker.is_any(): + constraint["markers"] = str(dependency.marker) + + dependencies[dependency.pretty_name].append(constraint) + + # All the constraints should have the same type, + # but we want to simplify them if it's possible + for dependency, constraints in tuple(dependencies.items()): + if all(len(constraint) == 1 for constraint in constraints): + dependencies[dependency] = [ + constraint["version"] for constraint in constraints + ] + + data = { + "name": package.pretty_name, + "version": package.pretty_version, + "description": package.description or "", + "category": package.category, + "optional": package.optional, + "python-versions": package.python_versions, + "files": sorted(package.files, key=lambda x: x["file"]), + } + + if package.extras: + extras = {} + for name, deps in package.extras.items(): + extras[name] = [ + str(dep) if not dep.constraint.is_any() else dep.name + for dep in deps + ] + + data["extras"] = extras + + if dependencies: + for k, constraints in dependencies.items(): + if len(constraints) == 1: + dependencies[k] = constraints[0] + + data["dependencies"] = dependencies + + if package.source_url: + data["source"] = { + "url": package.source_url, + "reference": package.source_reference, + } + if package.source_type: + data["source"]["type"] = package.source_type + if package.source_type == "directory": + data["develop"] = package.develop + + return data + + @classmethod + def load(cls, lock, pyproject_file=None): # type: (Path, Optional[Path]) -> Locker + if pyproject_file and pyproject_file.exists(): + return cls(lock, PyProjectTOML(pyproject_file).poetry_config) + return cls(lock) diff --git a/poetry/core/masonry/builders/builder.py b/poetry/core/masonry/builders/builder.py index a829a7cbb..b96cfac9d 100644 --- a/poetry/core/masonry/builders/builder.py +++ b/poetry/core/masonry/builders/builder.py @@ -87,7 +87,7 @@ def __init__( includes=includes, ) - self._meta = Metadata.from_package(self._package) + self._meta = Metadata.from_poetry_project(self._poetry) @property def executable(self): # type: () -> Path diff --git a/poetry/core/masonry/builders/sdist.py b/poetry/core/masonry/builders/sdist.py index ae6cc677b..f7dcc6ab1 100644 --- a/poetry/core/masonry/builders/sdist.py +++ b/poetry/core/masonry/builders/sdist.py @@ -313,6 +313,7 @@ def find_files_to_add( # Include project files additional_files.add("pyproject.toml") + additional_files.add("poetry.lock") # add readme if it is specified if "readme" in self._poetry.local_config: diff --git a/poetry/core/masonry/metadata.py b/poetry/core/masonry/metadata.py index 1a7fe559a..c9ef3a81c 100644 --- a/poetry/core/masonry/metadata.py +++ b/poetry/core/masonry/metadata.py @@ -3,6 +3,22 @@ from poetry.core.version.helpers import format_python_constraint +class MetadataConfig: + def __init__( + self, dependency_lock=False, dependency_nested=False + ): # type: (bool, bool) -> None + self._dependency_lock = dependency_lock + self._dependency_nested = dependency_nested + + @property + def dependency_lock(self): + return self._dependency_lock + + @property + def dependency_nested(self): + return self._dependency_nested + + class Metadata: metadata_version = "2.1" @@ -39,7 +55,8 @@ class Metadata: provides_extra = [] @classmethod - def from_package(cls, package): # type: (...) -> Metadata + def from_poetry_project(cls, poetry): # type: (...) -> Metadata + package = poetry.package meta = cls() meta.name = canonicalize_name(package.name) @@ -67,7 +84,14 @@ def from_package(cls, package): # type: (...) -> Metadata if package.python_versions != "*": meta.requires_python = format_python_constraint(package.python_constraint) - meta.requires_dist = [d.to_pep_508() for d in package.requires] + meta.requires_dist = [ + d.to_pep_508() + for d in poetry.locker.get_project_dependencies( + project_requires=poetry.package.requires, + pinned_versions=poetry.build_metadata_config.dependency_lock, + with_nested=poetry.build_metadata_config.dependency_nested, + ) + ] # Version 2.1 if package.readme: diff --git a/poetry/core/packages/dependency.py b/poetry/core/packages/dependency.py index 6a25ccdf9..17cc6786f 100755 --- a/poetry/core/packages/dependency.py +++ b/poetry/core/packages/dependency.py @@ -44,13 +44,8 @@ def __init__( features=extras, ) - try: - if not isinstance(constraint, VersionConstraint): - self._constraint = parse_constraint(constraint) - else: - self._constraint = constraint - except ValueError: - self._constraint = parse_constraint("*") + self._constraint = None + self.constraint = constraint self._pretty_constraint = str(constraint) self._optional = optional @@ -86,6 +81,16 @@ def name(self): def constraint(self): return self._constraint + @constraint.setter + def constraint(self, value): + try: + if not isinstance(value, VersionConstraint): + self._constraint = parse_constraint(value) + else: + self._constraint = value + except ValueError: + self._constraint = parse_constraint("*") + @property def pretty_constraint(self): return self._pretty_constraint diff --git a/poetry/core/poetry.py b/poetry/core/poetry.py index af04a0dad..87b273868 100644 --- a/poetry/core/poetry.py +++ b/poetry/core/poetry.py @@ -3,23 +3,28 @@ from typing import TYPE_CHECKING from typing import Any +from typing import Optional +from poetry.core.lock.locker import Locker from poetry.core.pyproject import PyProjectTOML from poetry.core.utils._compat import Path # noqa if TYPE_CHECKING: + from poetry.core.masonry.metadata import MetadataConfig # noqa from poetry.core.packages import ProjectPackage # noqa from poetry.core.pyproject.toml import PyProjectTOMLFile # noqa class Poetry(object): def __init__( - self, file, local_config, package, - ): # type: (Path, dict, "ProjectPackage") -> None + self, file, local_config, package, locker=None + ): # type: (Path, dict, "ProjectPackage", Optional[Locker]) -> None self._pyproject = PyProjectTOML(file) self._package = package self._local_config = local_config + self._locker = locker or Locker(file.parent / "poetry.lock", local_config) + self._build_metadata_config = None @property def pyproject(self): # type: () -> PyProjectTOML @@ -37,5 +42,27 @@ def package(self): # type: () -> "ProjectPackage" def local_config(self): # type: () -> dict return self._local_config + @property + def locker(self): # type: () -> Optional[Locker] + return self._locker + + @property + def build_metadata_config(self): # type: () -> "MetadataConfig" + if self._build_metadata_config is None: + from poetry.core.masonry.metadata import MetadataConfig # noqa + + try: + config = self._local_config.get("build", {}).get("metadata", {}) + dependency = config.get("dependencies", {}) + + self._build_metadata_config = MetadataConfig( + dependency_lock=dependency.get("lock", False), + dependency_nested=dependency.get("nested", False), + ) + except AttributeError: + self._build_metadata_config = MetadataConfig() + + return self._build_metadata_config + def get_project_config(self, config, default=None): # type: (str, Any) -> Any return self._local_config.get("config", {}).get(config, default)