Skip to content

Commit

Permalink
Implement inline package constraints
Browse files Browse the repository at this point in the history
  • Loading branch information
joerick committed Jan 31, 2025
1 parent 69f8175 commit 870a1dd
Show file tree
Hide file tree
Showing 10 changed files with 171 additions and 47 deletions.
12 changes: 8 additions & 4 deletions cibuildwheel/linux.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ def build_in_container(
container: OCIContainer,
container_project_path: PurePath,
container_package_dir: PurePath,
local_tmp_dir: Path,
) -> None:
container_output_dir = PurePosixPath("/output")

Expand Down Expand Up @@ -201,6 +202,7 @@ def build_in_container(

for config in platform_configs:
log.build_start(config.identifier)
local_identifier_tmp_dir = local_tmp_dir / config.identifier
build_options = options.build_options(config.identifier)
build_frontend = build_options.build_frontend or BuildFrontendConfig("pip")
use_uv = build_frontend.name == "build[uv]" and Version(config.version) >= Version("3.8")
Expand All @@ -211,12 +213,13 @@ def build_in_container(
log.step("Setting up build environment...")

if build_options.dependency_constraints:
constraints_file = build_options.dependency_constraints.get_for_python_version(
config.version
local_constraints_file = build_options.dependency_constraints.get_for_python_version(
version=config.version,
tmp_dir=local_identifier_tmp_dir,
)
container_constraints_file = PurePosixPath("/constraints.txt")

container.copy_into(constraints_file, container_constraints_file)
container.copy_into(local_constraints_file, container_constraints_file)
dependency_constraint_flags = ["-c", container_constraints_file]

env = container.get_environment()
Expand Down Expand Up @@ -428,7 +431,7 @@ def build_in_container(
log.step_end()


def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
def build(options: Options, tmp_path: Path) -> None:
python_configurations = get_python_configurations(
options.globals.build_selector, options.globals.architectures
)
Expand Down Expand Up @@ -482,6 +485,7 @@ def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
container=container,
container_project_path=container_project_path,
container_package_dir=container_package_dir,
local_tmp_dir=tmp_path,
)

except subprocess.CalledProcessError as error:
Expand Down
6 changes: 4 additions & 2 deletions cibuildwheel/macos.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,9 @@ def build(options: Options, tmp_path: Path) -> None:
if build_options.dependency_constraints:
dependency_constraint_flags = [
"-c",
build_options.dependency_constraints.get_for_python_version(config.version),
build_options.dependency_constraints.get_for_python_version(
version=config.version, tmp_dir=identifier_tmp_dir
),
]

base_python, env = setup_python(
Expand Down Expand Up @@ -476,7 +478,7 @@ def build(options: Options, tmp_path: Path) -> None:
build_env["VIRTUALENV_PIP"] = pip_version
if build_options.dependency_constraints:
constraint_path = build_options.dependency_constraints.get_for_python_version(
config.version
version=config.version, tmp_dir=identifier_tmp_dir
)
combine_constraints(
build_env, constraint_path, identifier_tmp_dir if use_uv else None
Expand Down
22 changes: 12 additions & 10 deletions cibuildwheel/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -690,7 +690,6 @@ def build_options(self, identifier: str | None) -> BuildOptions:
"config-settings", option_format=ShlexTableFormat(sep=" ", pair_sep="=")
)

dependency_versions = self.reader.get("dependency-versions")
test_command = self.reader.get("test-command", option_format=ListFormat(sep=" && "))
before_test = self.reader.get("before-test", option_format=ListFormat(sep=" && "))
test_sources = shlex.split(
Expand Down Expand Up @@ -736,15 +735,18 @@ def build_options(self, identifier: str | None) -> BuildOptions:
with contextlib.suppress(KeyError):
environment.add(env_var_name, self.env[env_var_name], prepend=True)

if dependency_versions == "pinned":
dependency_constraints: None | (
DependencyConstraints
) = DependencyConstraints.with_defaults()
elif dependency_versions == "latest":
dependency_constraints = None
else:
dependency_versions_path = Path(dependency_versions)
dependency_constraints = DependencyConstraints(dependency_versions_path)
dependency_versions_str = self.reader.get(
"dependency-versions",
env_plat=True,
option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False),
)
try:
dependency_constraints = DependencyConstraints.from_config_string(
dependency_versions_str
)
except ValueError as e:
msg = f"Failed to parse dependency versions. {e}"
raise errors.ConfigurationError(msg) from e

if test_extras:
test_extras = f"[{test_extras}]"
Expand Down
2 changes: 1 addition & 1 deletion cibuildwheel/pyodide.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ def build(options: Options, tmp_path: Path) -> None:
dependency_constraint_flags: Sequence[PathOrStr] = []
if build_options.dependency_constraints:
constraints_path = build_options.dependency_constraints.get_for_python_version(
config.version, variant="pyodide"
version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir
)
dependency_constraint_flags = ["-c", constraints_path]

Expand Down
70 changes: 57 additions & 13 deletions cibuildwheel/util/packaging.py
Original file line number Diff line number Diff line change
@@ -1,27 +1,75 @@
from __future__ import annotations

import shlex
from collections.abc import Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from pathlib import Path, PurePath
from typing import Any, Literal, TypeVar

from packaging.utils import parse_wheel_filename

from . import resources
from .cmd import call
from .helpers import parse_key_value_string


@dataclass()
class DependencyConstraints:
def __init__(self, base_file_path: Path):
assert base_file_path.exists()
self.base_file_path = base_file_path.resolve()
base_file_path: Path | None = None
packages: list[str] | None = None

def __post_init__(self) -> None:
if self.packages is not None and self.base_file_path is not None:
msg = "Cannot specify both a file and packages in the dependency constraints"
raise ValueError(msg)

if self.base_file_path is not None:
assert self.base_file_path.exists()
self.base_file_path = self.base_file_path.resolve()

@staticmethod
def with_defaults() -> DependencyConstraints:
return DependencyConstraints(base_file_path=resources.CONSTRAINTS)

@staticmethod
def from_config_string(config_string: str) -> DependencyConstraints | None:
config_dict = parse_key_value_string(config_string, ["file"], ["packages"])
file_or_keywords = config_dict.get("file")
packages = config_dict.get("packages")

if file_or_keywords and packages:
msg = "Cannot specify both a file and packages in dependency-versions"
raise ValueError(msg)

if packages:
return DependencyConstraints(packages=packages)

if file_or_keywords and len(file_or_keywords) > 1:
msg = "Only one file or keyword can be specified in dependency-versions"
raise ValueError(msg)

file_or_keyword = file_or_keywords[0] if file_or_keywords else None

if file_or_keyword == "latest":
return None

if file_or_keyword == "pinned" or not file_or_keyword:
return DependencyConstraints.with_defaults()

return DependencyConstraints(base_file_path=Path(file_or_keyword))

def get_for_python_version(
self, version: str, *, variant: Literal["python", "pyodide"] = "python"
self, *, version: str, variant: Literal["python", "pyodide"] = "python", tmp_dir: Path
) -> Path:
if self.packages:
constraint_file = tmp_dir / "constraints.txt"
constraint_file.write_text("\n".join(self.packages))
return constraint_file

assert (
self.base_file_path is not None
), "DependencyConstraints should have either a file or packages"

version_parts = version.split(".")

# try to find a version-specific dependency file e.g. if
Expand All @@ -35,19 +83,15 @@ def get_for_python_version(
else:
return self.base_file_path

def __repr__(self) -> str:
return f"{self.__class__.__name__}({self.base_file_path!r})"

def __eq__(self, o: object) -> bool:
if not isinstance(o, DependencyConstraints):
return False

return self.base_file_path == o.base_file_path

def options_summary(self) -> Any:
if self == DependencyConstraints.with_defaults():
return "pinned"
elif self.packages:
return {"packages": " ".join(shlex.quote(p) for p in self.packages)}
else:
assert (
self.base_file_path is not None
), "DependencyConstraints should have either a file or packages"
return self.base_file_path.name


Expand Down
8 changes: 6 additions & 2 deletions cibuildwheel/windows.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,10 @@ def build(options: Options, tmp_path: Path) -> None:
if build_options.dependency_constraints:
dependency_constraint_flags = [
"-c",
build_options.dependency_constraints.get_for_python_version(config.version),
build_options.dependency_constraints.get_for_python_version(
version=config.version,
tmp_dir=identifier_tmp_dir,
),
]

# install Python
Expand Down Expand Up @@ -418,7 +421,8 @@ def build(options: Options, tmp_path: Path) -> None:

if build_options.dependency_constraints:
constraints_path = build_options.dependency_constraints.get_for_python_version(
config.version
version=config.version,
tmp_dir=identifier_tmp_dir,
)
combine_constraints(build_env, constraints_path, identifier_tmp_dir)

Expand Down
6 changes: 5 additions & 1 deletion docs/options.md
Original file line number Diff line number Diff line change
Expand Up @@ -1406,6 +1406,10 @@ Platform-specific environment variables are also available:<br/>

# Choose a specific pyodide-build version
CIBW_DEPENDENCY_VERSIONS_PYODIDE: "packages: pyodide-build==0.29.1"

# Use shell-style quoting around spaces in paths or package specifiers
CIBW_DEPENDENCY_VERSIONS: "'./constraints file.txt'"
CIBW_DEPENDENCY_VERSIONS: "packages: 'pip >=16.0.0, !=17'"
```

!!! tab examples "pyproject.toml"
Expand All @@ -1419,7 +1423,7 @@ Platform-specific environment variables are also available:<br/>
dependency-versions = "latest"

# Use your own pip constraints file
dependency-versions = "./constraints.txt"
dependency-versions = { file = "./constraints.txt" }

# Specify requirements inline
dependency-versions = { packages = ["auditwheel==6.2.0"] }
Expand Down
31 changes: 21 additions & 10 deletions test/test_dependency_versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import platform
import re
import shlex
import textwrap
from pathlib import Path

Expand Down Expand Up @@ -96,7 +97,8 @@ def test_pinned_versions(tmp_path, python_version, build_frontend_env_nouv):
assert set(actual_wheels) == set(expected_wheels)


def test_dependency_constraints_file(tmp_path, build_frontend_env_nouv):
@pytest.mark.parametrize("method", ["inline", "file"])
def test_dependency_constraints(method, tmp_path, build_frontend_env_nouv):
if utils.platform == "linux":
pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead")

Expand All @@ -108,15 +110,24 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env_nouv):
"delocate": "0.10.3",
}

constraints_file = tmp_path / "constraints file.txt"
constraints_file.write_text(
textwrap.dedent(
"""
pip=={pip}
delocate=={delocate}
""".format(**tool_versions)
if method == "file":
constraints_file = tmp_path / "constraints file.txt"
constraints_file.write_text(
textwrap.dedent(
"""
pip=={pip}
delocate=={delocate}
""".format(**tool_versions)
)
)
)
dependency_version_option = shlex.quote(str(constraints_file))
elif method == "inline":
dependency_version_option = "packages: " + " ".join(
f"{k}=={v}" for k, v in tool_versions.items()
)
else:
msg = f"Unknown method: {method}"
raise ValueError(msg)

build_environment = {}

Expand All @@ -131,7 +142,7 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env_nouv):
project_dir,
add_env={
"CIBW_ENVIRONMENT": cibw_environment_option,
"CIBW_DEPENDENCY_VERSIONS": str(constraints_file),
"CIBW_DEPENDENCY_VERSIONS": dependency_version_option,
"CIBW_SKIP": "cp36-*",
**build_frontend_env_nouv,
},
Expand Down
21 changes: 17 additions & 4 deletions unit_test/dependency_constraints_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,32 @@
from cibuildwheel.util.packaging import DependencyConstraints


def test_defaults():
def test_defaults(tmp_path: Path) -> None:
dependency_constraints = DependencyConstraints.with_defaults()

project_root = Path(__file__).parents[1]
resources_dir = project_root / "cibuildwheel" / "resources"

assert dependency_constraints.base_file_path
assert dependency_constraints.base_file_path.samefile(resources_dir / "constraints.txt")
assert dependency_constraints.get_for_python_version("3.99").samefile(
assert dependency_constraints.get_for_python_version(version="3.99", tmp_dir=tmp_path).samefile(
resources_dir / "constraints.txt"
)
assert dependency_constraints.get_for_python_version("3.9").samefile(
assert dependency_constraints.get_for_python_version(version="3.9", tmp_dir=tmp_path).samefile(
resources_dir / "constraints-python39.txt"
)
assert dependency_constraints.get_for_python_version("3.6").samefile(
assert dependency_constraints.get_for_python_version(version="3.6", tmp_dir=tmp_path).samefile(
resources_dir / "constraints-python36.txt"
)


def test_inline_packages(tmp_path: Path) -> None:
dependency_constraints = DependencyConstraints(
base_file_path=None,
packages=["foo==1.2.3", "bar==4.5.6"],
)

constraint_file = dependency_constraints.get_for_python_version(version="x.x", tmp_dir=tmp_path)
constraints_file_contents = constraint_file.read_text()

assert constraints_file_contents == "foo==1.2.3\nbar==4.5.6"
Loading

0 comments on commit 870a1dd

Please sign in to comment.