refactor: include manylinux, minor redesign

cibuildwheel/__main__.py

@@ -16,7 +16,7 @@
 import cibuildwheel.windows
 from cibuildwheel.architecture import Architecture, allowed_architectures_check
 from cibuildwheel.environment import EnvironmentParseError, parse_environment
-from cibuildwheel.options import ConfigOptions
+from cibuildwheel.options import ConfigNamespace, ConfigOptions
 from cibuildwheel.projectfiles import get_requires_python_str
 from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
 from cibuildwheel.util import (
@@ -71,7 +71,6 @@ def main() -> None:
 
     parser.add_argument(
         "--output-dir",
-        default=os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse"),
         help="Destination folder for the wheels.",
     )
 
@@ -139,20 +138,24 @@ def main() -> None:
         sys.exit(2)
 
     package_dir = Path(args.package_dir)
-    output_dir = Path(args.output_dir)
 
     options = ConfigOptions(package_dir, platform=platform)
+    output_dir = Path(
+        args.output_dir
+        if args.output_dir is not None
+        else options("output-dir", namespace=ConfigNamespace.MAIN)
+    )
 
-    build_config = options("build", platform_variants=False) or "*"
-    skip_config = options("skip", platform_variants=False)
-    test_skip = options("test-skip", platform_variants=False)
+    build_config = options("build", namespace=ConfigNamespace.MAIN) or "*"
+    skip_config = options("skip", namespace=ConfigNamespace.MAIN)
+    test_skip = options("test-skip", namespace=ConfigNamespace.MAIN)
 
     archs_config_str = options("archs") if args.archs is None else args.archs
 
     environment_config = options("environment")
     before_all = options("before-all")
     before_build = options("before-build")
-    repair_command = options("repair-command")
+    repair_command = options("repair-wheel-command")
 
     dependency_versions = options("dependency-versions")
     test_command = options("test-command")
@@ -235,8 +238,7 @@ def main() -> None:
         for build_platform in ["x86_64", "i686", "pypy_x86_64", "aarch64", "ppc64le", "s390x"]:
             pinned_images = all_pinned_docker_images[build_platform]
 
-            config_name = f"CIBW_MANYLINUX_{build_platform.upper()}_IMAGE"
-            config_value = os.environ.get(config_name)
+            config_value = options(f"{build_platform}-image", namespace=ConfigNamespace.MANYLINUX)
 
             if config_value is None:
                 # default to manylinux2010 if it's available, otherwise manylinux2014

cibuildwheel/options.py

@@ -1,6 +1,7 @@
+import enum
 import os
 from pathlib import Path
-from typing import Dict, Mapping, Tuple
+from typing import Any, Dict, Mapping, Tuple
 
 import toml
 
@@ -26,6 +27,12 @@ def _dig_first(*pairs: Tuple[Mapping[str, Setting], str]) -> Setting:
     return dict_like.get(key, _dig_first(*others)) if others else dict_like[key]
 
 
+class ConfigNamespace(enum.Enum):
+    PLATFORM = enum.auto()  # Available in "global" and plat-specific namespace
+    MAIN = enum.auto()  # Available without a namespace
+    MANYLINUX = enum.auto()  # Only in the manylinux namespace
+
+
 class ConfigOptions:
     """
     Gets options from the environment, optionally scoped by the platform.
@@ -42,80 +49,111 @@ class ConfigOptions:
 
     def __init__(self, project_path: Path, *, platform: str) -> None:
         self.platform = platform
-        self.global_options: Dict[str, Setting] = {}
-        self.platform_options: Dict[str, Setting] = {}
+        self.config: Dict[str, Any] = {}
 
         # Open defaults.toml and load tool.cibuildwheel.global, then update with tool.cibuildwheel.<platform>
-        self._load_file(DIR.joinpath("resources", "defaults.toml"), check=False)
+        self._load_file(DIR.joinpath("resources", "defaults.toml"), update=False)
 
         # Open pyproject.toml if it exists and load from there
         pyproject_toml = project_path.joinpath("pyproject.toml")
-        self._load_file(pyproject_toml, check=True)
-
-        # Open cibuildwheel.toml if it exists and load from there
-        cibuildwheel_toml = project_path.joinpath("cibuildwheel.toml")
-        self._load_file(cibuildwheel_toml, check=True)
+        self._load_file(pyproject_toml, update=True)
 
     def _update(
-        self, old_dict: Dict[str, Setting], new_dict: Dict[str, Setting], *, check: bool
+        self,
+        old_dict: Dict[str, Any],
+        new_dict: Dict[str, Any],
+        *,
+        update: bool,
+        path: str = "",
     ) -> None:
         """
         Updates a dict with a new dict - optionally checking to see if the key
         is unexpected based on the current global options - call this with
         check=False when loading the defaults.toml file, and all future files
         will be forced to have no new keys.
         """
-        for key in new_dict:
-            if check and key not in self.global_options:
-                raise ConfigOptionError(f"Key not supported, problem in config file: {key}")
-
-            old_dict[key] = new_dict[key]
 
-    def _load_file(self, filename: Path, *, check: bool) -> None:
+        for key in new_dict:
+            # Check to see if key is already present (in global too if a platform)
+            if update:
+                options = set(self.config[path] if path else self.config)
+                if path in PLATFORMS:
+                    options |= set(self.config["global"])
+
+                if key not in options:
+                    raise ConfigOptionError(
+                        f"Key not supported, problem in config file: {path} {key}"
+                    )
+
+            # This is recursive; update dicts (subsections) if needed. Only handles one level.
+            if isinstance(new_dict[key], dict):
+                if path:
+                    raise ConfigOptionError(
+                        f"Nested keys not supported, {key} should not be in {path}"
+                    )
+
+                if key not in old_dict:
+                    old_dict[key] = {}
+
+                self._update(old_dict[key], new_dict[key], update=update, path=key)
+            else:
+                old_dict[key] = new_dict[key]
+
+    def _load_file(self, filename: Path, *, update: bool) -> None:
         """
-        Load a toml file, global and current platform. Raise an error if any unexpected
-        sections are present in tool.cibuildwheel, and pass on check to _update.
+        Load a toml file, global and current platform. Raise an error if any
+        unexpected sections are present in tool.cibuildwheel if updating, and
+        raise if any are missing if not.
         """
         # Only load if present.
         try:
             config = toml.load(filename)
         except FileNotFoundError:
+            assert update, "Missing default.toml, this should not happen!"
             return
 
         # If these sections are not present, go on.
-        if not config.get("tool", {}).get("cibuildwheel"):
+        tool_cibuildwheel = config.get("tool", {}).get("cibuildwheel")
+        if not tool_cibuildwheel:
+            assert update, "Malformed internal default.toml, this should not happen!"
             return
 
-        unsupported = set(config["tool"]["cibuildwheel"]) - (PLATFORMS | {"global"})
-        if unsupported:
-            raise ConfigOptionError(f"Unsupported configuration section(s): {unsupported}")
+        self._update(self.config, tool_cibuildwheel, update=update)
 
-        self._update(self.global_options, config["tool"]["cibuildwheel"]["global"], check=check)
-        self._update(
-            self.platform_options, config["tool"]["cibuildwheel"][self.platform], check=check
-        )
-
-    def __call__(self, name: str, *, platform_variants: bool = True) -> Setting:
+    def __call__(
+        self, name: str, *, namespace: ConfigNamespace = ConfigNamespace.PLATFORM
+    ) -> Setting:
         """
         Get and return envvar for name or the override or the default.
         """
-        if name not in self.global_options:
-            raise ConfigOptionError(
-                f"{name} was not loaded from the cibuildwheel/resources/defaults.toml file"
-            )
 
-        envvar = f"CIBW_{name.upper().replace('-', '_')}"
+        # Get config settings for the requested namespace and current platform
+        if namespace == ConfigNamespace.MAIN:
+            config = self.config
+        elif namespace == ConfigNamespace.MANYLINUX:
+            config = self.config["manylinux"]
+        elif namespace == ConfigNamespace.PLATFORM:
+            config = {**self.config["global"], **self.config[self.platform]}
+
+        if name not in config:
+            raise ConfigOptionError(f"{name} must be in cibuildwheel/resources/defaults.toml file")
+
+        # Environment variable form
+        if namespace == ConfigNamespace.MANYLINUX:
+            envvar = f"CIBW_MANYLINUX_{name.upper().replace('-', '_')}"
+        else:
+            envvar = f"CIBW_{name.upper().replace('-', '_')}"
 
-        if platform_variants:
+        # Let environment variable override setting in config
+        if namespace == ConfigNamespace.PLATFORM:
             plat_envvar = f"{envvar}_{self.platform.upper()}"
             return _dig_first(
                 (os.environ, plat_envvar),
-                (self.platform_options, name),
                 (os.environ, envvar),
-                (self.global_options, name),
+                (config, name),
             )
         else:
             return _dig_first(
                 (os.environ, envvar),
-                (self.global_options, name),
+                (config, name),
             )

cibuildwheel/resources/defaults.toml

@@ -1,31 +1,39 @@
-[tool.cibuildwheel.global]
+[tool.cibuildwheel]
 build = "*"
 skip = ""
 test-skip = ""
+output-dir = "wheelhouse"
+
+
+[tool.cibuildwheel.manylinux]
+x86_64-image = "manylinux2010"
+i686-image = "manylinux2010"
+pypy_x86_64-image = "manylinux2010"
+aarch64-image = "manylinux2014"
+ppc64le-image = "manylinux2014"
+s390x-image = "manylinux2014"
 
-archs = "auto"
 
+[tool.cibuildwheel.global]
+archs = "auto"
+dependency-versions = "pinned"
 environment = ""
+build-verbosity = ""
 
 before-all = ""
 before-build = ""
-
-repair-command = ""
-
-dependency-versions = "pinned"
+repair-wheel-command = ""
 
 test-command = ""
 before-test = ""
 test-requires = ""
 test-extras = ""
 
-build-verbosity = ""
-
 
 [tool.cibuildwheel.linux]
-repair-command = "auditwheel repair -w {dest_dir} {wheel}"
+repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"
 
 [tool.cibuildwheel.macos]
-repair-command = "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}"
+repair-wheel-command = "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}"
 
 [tool.cibuildwheel.windows]

unit_test/main_tests/main_options_test.py

@@ -199,7 +199,7 @@ def test_test_command(
 
     main()
 
-    assert intercepted_build_args.args[0].test_command == test_command
+    assert intercepted_build_args.args[0].test_command == (test_command or "")
 
 
 @pytest.mark.parametrize("before_build", [None, "before --build"])
@@ -216,7 +216,7 @@ def test_before_build(
 
     main()
 
-    assert intercepted_build_args.args[0].before_build == before_build
+    assert intercepted_build_args.args[0].before_build == (before_build or "")
 
 
 @pytest.mark.parametrize("build_verbosity", [None, 0, 2, -2, 4, -4])
@@ -280,7 +280,4 @@ def test_before_all(before_all, platform_specific, platform, intercepted_build_a
 
     main()
 
-    if before_all is None:
-        before_all = ""
-
-    assert intercepted_build_args.args[0].before_all == before_all
+    assert intercepted_build_args.args[0].before_all == (before_all or "")

unit_test/options_toml_test.py

@@ -0,0 +1,70 @@
+import pytest
+
+from cibuildwheel.options import ConfigNamespace, ConfigOptions
+
+PYPROJECT_1 = """
+[tool.cibuildwheel]
+build = "cp39*"
+
+[tool.cibuildwheel.manylinux]
+x86_64-image = "manylinux1"
+
+[tool.cibuildwheel.global]
+test-command = "pyproject"
+test-requires = "something"
+
+[tool.cibuildwheel.macos]
+test-requires = "else"
+
+[tool.cibuildwheel.linux]
+test-requires = "other"
+"""
+
+
+@pytest.fixture(params=["linux", "macos", "windows"])
+def platform(request):
+    return request.param
+
+
+def test_simple_settings(tmp_path, platform):
+    with tmp_path.joinpath("pyproject.toml").open("w") as f:
+        f.write(PYPROJECT_1)
+
+    options = ConfigOptions(tmp_path, platform=platform)
+
+    assert options("build", namespace=ConfigNamespace.MAIN) == "cp39*"
+    assert options("output-dir", namespace=ConfigNamespace.MAIN) == "wheelhouse"
+
+    assert options("test-command") == "pyproject"
+    assert options("archs") == "auto"
+    assert (
+        options("test-requires")
+        == {"windows": "something", "macos": "else", "linux": "other"}[platform]
+    )
+
+    assert options("x86_64-image", namespace=ConfigNamespace.MANYLINUX) == "manylinux1"
+    assert options("i686-image", namespace=ConfigNamespace.MANYLINUX) == "manylinux2010"
+
+
+def test_envvar_override(tmp_path, platform, monkeypatch):
+    monkeypatch.setenv("CIBW_BUILD", "cp38*")
+    monkeypatch.setenv("CIBW_MANYLINUX_X86_64_IMAGE", "manylinux2014")
+    monkeypatch.setenv("CIBW_TEST_COMMAND", "mytest")
+    monkeypatch.setenv("CIBW_TEST_REQUIRES", "docs")
+    monkeypatch.setenv("CIBW_TEST_REQUIRES_LINUX", "scod")
+
+    with tmp_path.joinpath("pyproject.toml").open("w") as f:
+        f.write(PYPROJECT_1)
+
+    options = ConfigOptions(tmp_path, platform=platform)
+
+    assert options("archs") == "auto"
+
+    assert options("build", namespace=ConfigNamespace.MAIN) == "cp38*"
+    assert options("x86_64-image", namespace=ConfigNamespace.MANYLINUX) == "manylinux2014"
+    assert options("i686-image", namespace=ConfigNamespace.MANYLINUX) == "manylinux2010"
+
+    assert (
+        options("test-requires") == {"windows": "docs", "macos": "docs", "linux": "scod"}[platform]
+    )
+    assert options("test-command") == "mytest"