diff --git a/CHANGELOG.md b/CHANGELOG.md index 87f50f595d6..ba7b18866ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix regression in 0.14.0 leading to incorrect code coverage being computed for `#[pyfunction]`s. [#1726](https://github.com/PyO3/pyo3/pull/1726) - Fix incorrect FFI definition of `Py_Buffer` on PyPy. [#1737](https://github.com/PyO3/pyo3/pull/1737) - Fix incorrect calculation of `dictoffset` on 32-bit Windows. [#1475](https://github.com/PyO3/pyo3/pull/1475) +- Fix regression in 0.13.2 leading to linking to incorrect Python library on Windows "gnu" targets. [#1759](https://github.com/PyO3/pyo3/pull/1759) ## [0.14.1] - 2021-07-04 diff --git a/build.rs b/build.rs index c4ac6bf9623..4b5101289fe 100644 --- a/build.rs +++ b/build.rs @@ -1,9 +1,11 @@ use std::{env, process::Command}; use pyo3_build_config::{ - bail, cargo_env_var, ensure, env_var, - errors::{Context, Result}, - InterpreterConfig, PythonImplementation, PythonVersion, + bail, ensure, + pyo3_build_script_impl::{ + cargo_env_var, env_var, errors::Result, resolve_interpreter_config, InterpreterConfig, + PythonVersion, + }, }; /// Minimum Python version PyO3 supports. @@ -20,118 +22,29 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { Ok(()) } -fn ensure_target_architecture(interpreter_config: &InterpreterConfig) -> Result<()> { - // Try to check whether the target architecture matches the python library - let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") - .unwrap() - .as_str() - { - "64" => "64-bit", - "32" => "32-bit", - x => bail!("unexpected Rust target pointer width: {}", x), - }; - - // The reason we don't use platform.architecture() here is that it's not - // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. - // Similarly, sys.maxsize is not reliable on Windows. See - // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 - // and https://stackoverflow.com/a/3411134/823869. - let python_target = match interpreter_config.calcsize_pointer { - Some(8) => "64-bit", - Some(4) => "32-bit", - None => { - // Unset, e.g. because we're cross-compiling. Don't check anything - // in this case. - return Ok(()); - } - Some(n) => bail!("unexpected Python calcsize_pointer value: {}", n), - }; - - ensure!( - rust_target == python_target, - "Your Rust target architecture ({}) does not match your python interpreter ({})", - rust_target, - python_target - ); - - Ok(()) -} - -fn get_rustc_link_lib(config: &InterpreterConfig) -> Result { - let link_name = if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "windows" { - if config.abi3 { - // Link against python3.lib for the stable ABI on Windows. - // See https://www.python.org/dev/peps/pep-0384/#linkage - // - // This contains only the limited ABI symbols. - "pythonXY:python3".to_owned() - } else if cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap() == "gnu" { - // https://packages.msys2.org/base/mingw-w64-python - format!( - "pythonXY:python{}.{}", - config.version.major, config.version.minor - ) - } else { - format!( - "pythonXY:python{}{}", - config.version.major, config.version.minor - ) - } - } else { - match config.implementation { - PythonImplementation::CPython => match &config.ld_version { - Some(ld_version) => format!("python{}", ld_version), - None => bail!("failed to configure `ld_version` when compiling for unix"), - }, - PythonImplementation::PyPy => format!("pypy{}-c", config.version.major), - } - }; - - Ok(format!( - "cargo:rustc-link-lib={link_model}{link_name}", - link_model = if config.shared { "" } else { "static=" }, - link_name = link_name - )) -} - -fn rustc_minor_version() -> Option { - let rustc = env::var_os("RUSTC")?; - let output = Command::new(rustc).arg("--version").output().ok()?; - let version = core::str::from_utf8(&output.stdout).ok()?; - let mut pieces = version.split('.'); - if pieces.next() != Some("rustc 1") { - return None; +fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result<()> { + if let Some(pointer_width) = interpreter_config.pointer_width { + // Try to check whether the target architecture matches the python library + let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") + .unwrap() + .as_str() + { + "64" => 64, + "32" => 32, + x => bail!("unexpected Rust target pointer width: {}", x), + }; + + ensure!( + rust_target == pointer_width, + "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)", + rust_target, + pointer_width + ); } - pieces.next()?.parse().ok() + Ok(()) } -fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()> { - let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap(); - let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some(); - match (is_extension_module, target_os.as_str()) { - (_, "windows") => { - // always link on windows, even with extension module - println!("{}", get_rustc_link_lib(interpreter_config)?); - // Set during cross-compiling. - if let Some(libdir) = &interpreter_config.libdir { - println!("cargo:rustc-link-search=native={}", libdir); - } - // Set if we have an interpreter to use. - if let Some(base_prefix) = &interpreter_config.base_prefix { - println!("cargo:rustc-link-search=native={}\\libs", base_prefix); - } - } - (false, _) | (_, "android") => { - // other systems, only link libs if not extension module - // android always link. - println!("{}", get_rustc_link_lib(interpreter_config)?); - if let Some(libdir) = &interpreter_config.libdir { - println!("cargo:rustc-link-search=native={}", libdir); - } - } - _ => {} - } - +fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> { if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() { if !interpreter_config.shared { bail!( @@ -152,35 +65,74 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<() // TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies` // currently cause `auto-initialize` to be enabled in CI. - // Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this. - if interpreter_config.is_pypy() && env::var_os("PYO3_CI").is_none() { - bail!("The `auto-initialize` feature is not supported with PyPy."); + // Once MSRV is 1.51 or higher, use cargo's `resolver = "2"` instead. + if interpreter_config.implementation.is_pypy() && env::var_os("PYO3_CI").is_none() { + bail!("the `auto-initialize` feature is not supported with PyPy"); + } + } + Ok(()) +} + +fn rustc_minor_version() -> Option { + let rustc = env::var_os("RUSTC")?; + let output = Command::new(rustc).arg("--version").output().ok()?; + let version = core::str::from_utf8(&output.stdout).ok()?; + let mut pieces = version.split('.'); + if pieces.next() != Some("rustc 1") { + return None; + } + pieces.next()?.parse().ok() +} + +fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()> { + let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap(); + let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some(); + if target_os == "windows" || target_os == "android" || !is_extension_module { + // windows and android - always link + // other systems - only link if not extension module + println!( + "cargo:rustc-link-lib={link_model}{alias}{lib_name}", + link_model = if interpreter_config.shared { + "" + } else { + "static=" + }, + alias = if target_os == "windows" { + "pythonXY:" + } else { + "" + }, + lib_name = interpreter_config.lib_name.as_ref().ok_or( + "attempted to link to Python shared library but config does not contain lib_name" + )?, + ); + if let Some(lib_dir) = &interpreter_config.lib_dir { + println!("cargo:rustc-link-search=native={}", lib_dir); } } Ok(()) } -/// Generates the interpreter config suitable for the host / target / cross-compilation at hand. +/// Prepares the PyO3 crate for compilation. +/// +/// This loads the config from pyo3-build-config and then makes some additional checks to improve UX +/// for users. /// -/// The result is written to pyo3_build_config::PATH, which downstream scripts can read from -/// (including `pyo3-macros-backend` during macro expansion). +/// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler +/// version to enable features which aren't supported on MSRV. fn configure_pyo3() -> Result<()> { - let interpreter_config = pyo3_build_config::make_interpreter_config()?; + let interpreter_config = resolve_interpreter_config()?; + if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") { print_config_and_exit(&interpreter_config); } + ensure_python_version(&interpreter_config)?; - ensure_target_architecture(&interpreter_config)?; + ensure_target_pointer_width(&interpreter_config)?; + ensure_auto_initialize_ok(&interpreter_config)?; + emit_cargo_configuration(&interpreter_config)?; - interpreter_config.to_writer( - &mut std::fs::File::create(pyo3_build_config::PATH).with_context(|| { - format!( - "failed to create config file at {}", - pyo3_build_config::PATH - ) - })?, - )?; interpreter_config.emit_pyo3_cfgs(); let rustc_minor_version = rustc_minor_version().unwrap_or(0); @@ -200,33 +152,15 @@ fn configure_pyo3() -> Result<()> { fn print_config_and_exit(config: &InterpreterConfig) { println!("\n-- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --"); - println!("implementation: {}", config.implementation); - println!("interpreter version: {}", config.version); - println!("interpreter path: {:?}", config.executable); - println!("libdir: {:?}", config.libdir); - println!("shared: {}", config.shared); - println!("base prefix: {:?}", config.base_prefix); - println!("ld_version: {:?}", config.ld_version); - println!("pointer width: {:?}", config.calcsize_pointer); - + config + .to_writer(&mut std::io::stdout()) + .expect("failed to print config to stdout"); std::process::exit(101); } fn main() { - // Print out error messages using display, to get nicer formatting. if let Err(e) = configure_pyo3() { - use std::error::Error; - eprintln!("error: {}", e); - let mut source = e.source(); - if source.is_some() { - eprintln!("caused by:"); - let mut index = 0; - while let Some(some_source) = source { - eprintln!(" - {}: {}", index, some_source); - source = some_source.source(); - index += 1; - } - } + eprintln!("error: {}", e.report()); std::process::exit(1) } } diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 96cfc69838b..05da889d45a 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -30,23 +30,22 @@ Caused by: cargo:rerun-if-env-changed=PYO3_CROSS cargo:rerun-if-env-changed=PYO3_CROSS_LIB_DIR cargo:rerun-if-env-changed=PYO3_CROSS_PYTHON_VERSION - cargo:rerun-if-env-changed=PYO3_PYTHON - cargo:rerun-if-env-changed=VIRTUAL_ENV - cargo:rerun-if-env-changed=CONDA_PREFIX - cargo:rerun-if-env-changed=PATH cargo:rerun-if-env-changed=PYO3_PRINT_CONFIG -- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile -- - implementation: CPython - interpreter version: 3.8 - interpreter path: Some("/usr/bin/python") - libdir: Some("/usr/lib") - shared: true - base prefix: Some("/usr") - ld_version: Some("3.8") - pointer width: Some(8) + implementation=CPython + version=3.8 + shared=true + abi3=false + lib_name=python3.8 + lib_dir=/usr/lib + executable=/usr/bin/python + pointer_width=64 + build_flags=WITH_THREAD ``` +> Note: if you save the output config to a file, it is possible to manually override the contents and feed it back into PyO3 using the `PYO3_CONFIG_FILE` env var. For now, this is an advanced feature that should not be needed for most users. The format of the config file and its contents are deliberately unstable and undocumented. If you have a production use-case for this config file, please file an issue and help us stabilize it! + ## Building Python extension modules Python extension modules need to be compiled differently depending on the OS (and architecture) that they are being compiled for. As well as multiple OSes (and architectures), there are also many different Python versions which are actively supported. Packages uploaded to [PyPI](https://pypi.org/) usually want to upload prebuilt "wheels" covering many OS/arch/version combinations so that users on all these different platforms don't have to compile the package themselves. Package vendors can opt-in to the "abi3" limited Python API which allows their wheels to be used on multiple Python versions, reducing the number of wheels they need to compile, but restricts the functionality they can use. diff --git a/pyo3-build-config/build.rs b/pyo3-build-config/build.rs index b94555bb929..28d6057944c 100644 --- a/pyo3-build-config/build.rs +++ b/pyo3-build-config/build.rs @@ -1,3 +1,96 @@ +// Import some modules from this crate inline to generate the build config. +// Allow dead code because not all code in the modules is used in this build script. + +#[path = "src/impl_.rs"] +#[allow(dead_code)] +mod impl_; + +#[path = "src/errors.rs"] +#[allow(dead_code)] +mod errors; + +use std::{env, path::Path}; + +use errors::{Context, Result}; +use impl_::{ + env_var, get_abi3_version, make_interpreter_config, BuildFlags, InterpreterConfig, + PythonImplementation, +}; + +fn configure(interpreter_config: Option, name: &str) -> Result { + let target = Path::new(&env::var_os("OUT_DIR").unwrap()).join(name); + if let Some(config) = interpreter_config { + config + .to_writer(&mut std::fs::File::create(&target).with_context(|| { + format!("failed to write config file at {}", target.display()) + })?)?; + Ok(true) + } else { + std::fs::File::create(&target) + .with_context(|| format!("failed to create new file at {}", target.display()))?; + Ok(false) + } +} + +/// If PYO3_CONFIG_FILE is set, copy it into the crate. +fn config_file() -> Result> { + if let Some(path) = env_var("PYO3_CONFIG_FILE") { + let path = Path::new(&path); + println!("cargo:rerun-if-changed={}", path.display()); + // Absolute path is necessary because this build script is run with a cwd different to the + // original `cargo build` instruction. + ensure!( + path.is_absolute(), + "PYO3_CONFIG_FILE must be an absolute path" + ); + + let interpreter_config = InterpreterConfig::from_path(path) + .context("failed to parse contents of PYO3_CONFIG_FILE")?; + Ok(Some(interpreter_config)) + } else { + Ok(None) + } +} + +/// If PYO3_NO_PYTHON is set with abi3, use standard abi3 settings. +pub fn abi3_config() -> Option { + if let Some(version) = get_abi3_version() { + if env_var("PYO3_NO_PYTHON").is_some() { + return Some(InterpreterConfig { + version, + // NB PyPy doesn't support abi3 yet + implementation: PythonImplementation::CPython, + abi3: true, + lib_name: None, + lib_dir: None, + build_flags: BuildFlags::abi3(), + pointer_width: None, + executable: None, + shared: true, + }); + } + } + None +} + +fn generate_build_configs() -> Result<()> { + let mut configured = false; + configured |= configure(config_file()?, "pyo3-build-config-file.txt")?; + configured |= configure(abi3_config(), "pyo3-build-config-abi3.txt")?; + + if configured { + // Don't bother trying to find an interpreter on the host system if at least one of the + // config file or abi3 settings are present + configure(None, "pyo3-build-config.txt")?; + } else { + configure(Some(make_interpreter_config()?), "pyo3-build-config.txt")?; + } + Ok(()) +} + fn main() { - // Empty build script to force cargo to produce the "OUT_DIR" environment variable. + if let Err(e) = generate_build_configs() { + eprintln!("error: {}", e.report()); + std::process::exit(1) + } } diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index 8d2903a3ef4..61e8dc37fdd 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -1,5 +1,6 @@ /// A simple macro for returning an error. Resembles anyhow::bail. #[macro_export] +#[doc(hidden)] macro_rules! bail { ($msg: expr) => { return Err($msg.into()); }; ($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); }; @@ -7,12 +8,14 @@ macro_rules! bail { /// A simple macro for checking a condition. Resembles anyhow::ensure. #[macro_export] +#[doc(hidden)] macro_rules! ensure { ($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } }; } /// Show warning. If needed, please extend this macro to support arguments. #[macro_export] +#[doc(hidden)] macro_rules! warn { ($msg: literal) => { println!(concat!("cargo:warning=", $msg)); @@ -26,6 +29,16 @@ pub struct Error { source: Option>, } +/// Error report inspired by +/// +pub struct ErrorReport<'a>(&'a Error); + +impl Error { + pub fn report(&self) -> ErrorReport<'_> { + ErrorReport(self) + } +} + impl std::fmt::Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.value) @@ -38,6 +51,24 @@ impl std::error::Error for Error { } } +impl std::fmt::Display for ErrorReport<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use std::error::Error; + self.0.fmt(f)?; + let mut source = self.0.source(); + if source.is_some() { + writeln!(f, "\ncaused by:")?; + let mut index = 0; + while let Some(some_source) = source { + writeln!(f, " - {}: {}", index, some_source)?; + source = some_source.source(); + index += 1; + } + } + Ok(()) + } +} + impl From for Error { fn from(value: String) -> Self { Self { @@ -84,3 +115,31 @@ where }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn error_report() { + let error: Result<()> = Err(Error::from("there was an internal error")) + .with_context(|| format!("failed to do {}", "something difficult")) + .context("things went wrong"); + + assert_eq!( + error + .unwrap_err() + .report() + .to_string() + .split('\n') + .collect::>(), + vec![ + "things went wrong", + "caused by:", + " - 0: failed to do something difficult", + " - 1: there was an internal error", + "" + ] + ); + } +} diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index a8ac9972cab..66c18d538cd 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -38,24 +38,26 @@ pub fn env_var(var: &str) -> Option { /// Configuration needed by PyO3 to build for the correct Python implementation. /// -/// Usually this is queried directly from the Python interpreter. When the `PYO3_NO_PYTHON` variable -/// is set, or during cross compile situations, then alternative strategies are used to populate -/// this type. +/// Usually this is queried directly from the Python interpreter, or overridden using the +/// `PYO3_CONFIG_FILE` environment variable. +/// +/// When the `PYO3_NO_PYTHON` variable is set, or during cross compile situations, then alternative +/// strategies are used to populate this type. #[cfg_attr(test, derive(Debug, PartialEq))] pub struct InterpreterConfig { + pub implementation: PythonImplementation, pub version: PythonVersion, - pub libdir: Option, pub shared: bool, pub abi3: bool, - pub ld_version: Option, - pub base_prefix: Option, + pub lib_name: Option, + pub lib_dir: Option, pub executable: Option, - pub calcsize_pointer: Option, - pub implementation: PythonImplementation, + pub pointer_width: Option, pub build_flags: BuildFlags, } impl InterpreterConfig { + #[doc(hidden)] pub fn emit_pyo3_cfgs(&self) { // This should have been checked during pyo3-build-config build time. assert!(self.version >= MINIMUM_SUPPORTED_VERSION); @@ -67,7 +69,7 @@ impl InterpreterConfig { println!("cargo:rustc-cfg=Py_LIMITED_API"); } - if self.is_pypy() { + if self.implementation.is_pypy() { println!("cargo:rustc-cfg=PyPy"); if self.abi3 { warn!( @@ -82,73 +84,202 @@ impl InterpreterConfig { } } - pub fn is_pypy(&self) -> bool { - self.implementation == PythonImplementation::PyPy + #[doc(hidden)] + pub fn from_interpreter(interpreter: impl AsRef) -> Result { + const SCRIPT: &str = r#" +# Allow the script to run on Python 2, so that nicer error can be printed later. +from __future__ import print_function + +import os.path +import platform +import struct +import sys +from sysconfig import get_config_var, get_platform + +PYPY = platform.python_implementation() == "PyPy" + +# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue +# so that the version mismatch can be reported in a nicer way later. +base_prefix = getattr(sys, "base_prefix", None) + +if base_prefix: + # Anaconda based python distributions have a static python executable, but include + # the shared library. Use the shared library for embedding to avoid rust trying to + # LTO the static library (and failing with newer gcc's, because it is old). + ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta")) +else: + ANACONDA = False + +def print_if_set(varname, value): + if value is not None: + print(varname, value) + +# Windows always uses shared linking +WINDOWS = hasattr(platform, "win32_ver") + +# macOS framework packages use shared linking +FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) + +# unix-style shared library enabled +SHARED = bool(get_config_var("Py_ENABLE_SHARED")) + +print("implementation", platform.python_implementation()) +print("version_major", sys.version_info[0]) +print("version_minor", sys.version_info[1]) +print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print_if_set("ld_version", get_config_var("LDVERSION")) +print_if_set("libdir", get_config_var("LIBDIR")) +print_if_set("base_prefix", base_prefix) +print("executable", sys.executable) +print("calcsize_pointer", struct.calcsize("P")) +print("mingw", get_platform() == "mingw") +"#; + let output = run_python_script(interpreter.as_ref(), SCRIPT)?; + let map: HashMap = parse_script_output(&output); + let shared = map["shared"].as_str() == "True"; + + let version = PythonVersion { + major: map["version_major"] + .parse() + .context("failed to parse major version")?, + minor: map["version_minor"] + .parse() + .context("failed to parse minor version")?, + }; + + let abi3 = is_abi3(); + let implementation = map["implementation"].parse()?; + + let lib_name = if cfg!(windows) { + default_lib_name_windows(version, abi3, map["mingw"].as_str() == "True") + } else { + default_lib_name_unix( + version, + implementation, + map.get("ld_version").map(String::as_str), + ) + }; + + let lib_dir = if cfg!(windows) { + map.get("base_prefix") + .map(|base_prefix| format!("{}\\libs", base_prefix)) + } else { + map.get("libdir").cloned() + }; + + // The reason we don't use platform.architecture() here is that it's not + // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. + // Similarly, sys.maxsize is not reliable on Windows. See + // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 + // and https://stackoverflow.com/a/3411134/823869. + let calcsize_pointer: u32 = map["calcsize_pointer"] + .parse() + .context("failed to parse calcsize_pointer")?; + + Ok(InterpreterConfig { + version, + implementation, + shared, + abi3, + lib_name: Some(lib_name), + lib_dir, + executable: map.get("executable").cloned(), + pointer_width: Some(calcsize_pointer * 8), + build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation), + }) + } + + #[doc(hidden)] + pub fn from_path(path: impl AsRef) -> Result { + let path = path.as_ref(); + let config_file = std::fs::File::open(path) + .with_context(|| format!("failed to open PyO3 config file at {}", path.display()))?; + let reader = std::io::BufReader::new(config_file); + InterpreterConfig::from_reader(reader) } #[doc(hidden)] pub fn from_reader(reader: impl Read) -> Result { let reader = BufReader::new(reader); - let mut lines = reader.lines(); - - macro_rules! parse_line { - ($value:literal) => { - lines - .next() - .ok_or(concat!("reached end of config when reading ", $value))? - .context(concat!("failed to read ", $value, " from config"))? - .parse() - .context(concat!("failed to parse ", $value, " from config")) + let lines = reader.lines(); + + macro_rules! parse_value { + ($variable:ident, $value:ident) => { + $variable = Some($value.parse().context(format!( + concat!( + "failed to parse ", + stringify!($variable), + " from config value '{}'" + ), + $value + ))?) }; } - macro_rules! parse_option_line { - ($value:literal) => { - parse_option_string( - lines - .next() - .ok_or(concat!("reached end of config when reading ", $value))? - .context(concat!("failed to read ", $value, " from config"))?, - ) - .context(concat!("failed to parse ", $value, "from config")) - }; + let mut implementation = None; + let mut version = None; + let mut shared = None; + let mut abi3 = None; + let mut lib_name = None; + let mut lib_dir = None; + let mut executable = None; + let mut pointer_width = None; + let mut build_flags = None; + + for (i, line) in lines.enumerate() { + let line = line.context("failed to read line from config")?; + let mut split = line.splitn(2, '='); + let (key, value) = ( + split + .next() + .expect("first splitn value should always be present"), + split + .next() + .ok_or_else(|| format!("expected key=value pair on line {}", i + 1))?, + ); + match key { + "implementation" => parse_value!(implementation, value), + "version" => parse_value!(version, value), + "shared" => parse_value!(shared, value), + "abi3" => parse_value!(abi3, value), + "lib_name" => parse_value!(lib_name, value), + "lib_dir" => parse_value!(lib_dir, value), + "executable" => parse_value!(executable, value), + "pointer_width" => parse_value!(pointer_width, value), + "build_flags" => parse_value!(build_flags, value), + unknown => bail!("unknown config key `{}`", unknown), + } } - let major = parse_line!("major version")?; - let minor = parse_line!("minor version")?; - let libdir = parse_option_line!("libdir")?; - let shared = parse_line!("shared")?; - let abi3 = parse_line!("abi3")?; - let ld_version = parse_option_line!("ld_version")?; - let base_prefix = parse_option_line!("base_prefix")?; - let executable = parse_option_line!("executable")?; - let calcsize_pointer = parse_option_line!("calcsize_pointer")?; - let implementation = parse_line!("implementation")?; - let mut build_flags = BuildFlags(HashSet::new()); - for line in lines { - build_flags - .0 - .insert(line.context("failed to read flag from config")?.parse()?); - } + let version = version.ok_or("missing value for version")?; + let implementation = implementation.unwrap_or(PythonImplementation::CPython); + let abi3 = abi3.unwrap_or(false); + Ok(InterpreterConfig { - version: PythonVersion { major, minor }, - libdir, - shared, + implementation, + version, + shared: shared.unwrap_or(true), abi3, - ld_version, - base_prefix, + lib_name, + lib_dir, executable, - calcsize_pointer, - implementation, - build_flags, + pointer_width, + build_flags: build_flags.unwrap_or_else(|| { + if abi3 { + BuildFlags::abi3() + } else { + BuildFlags::default() + } + .fixup(version, implementation) + }), }) } #[doc(hidden)] pub fn to_writer(&self, mut writer: impl Write) -> Result<()> { macro_rules! write_line { - ($value:expr) => { - writeln!(writer, "{}", $value).context(concat!( + ($value:ident) => { + writeln!(writer, "{}={}", stringify!($value), self.$value).context(concat!( "failed to write ", stringify!($value), " to config" @@ -157,44 +288,32 @@ impl InterpreterConfig { } macro_rules! write_option_line { - ($opt:expr) => { - match &$opt { - Some(value) => writeln!(writer, "{}", value), - None => writeln!(writer, "null"), + ($value:ident) => { + if let Some(value) = &self.$value { + writeln!(writer, "{}={}", stringify!($value), value).context(concat!( + "failed to write ", + stringify!($value), + " to config" + )) + } else { + Ok(()) } - .context(concat!( - "failed to write ", - stringify!($value), - " to config" - )) }; } - write_line!(self.version.major)?; - write_line!(self.version.minor)?; - write_option_line!(self.libdir)?; - write_line!(self.shared)?; - write_line!(self.abi3)?; - write_option_line!(self.ld_version)?; - write_option_line!(self.base_prefix)?; - write_option_line!(self.executable)?; - write_option_line!(self.calcsize_pointer)?; - write_line!(self.implementation)?; - for flag in &self.build_flags.0 { - write_line!(flag)?; - } + write_line!(implementation)?; + write_line!(version)?; + write_line!(shared)?; + write_line!(abi3)?; + write_option_line!(lib_name)?; + write_option_line!(lib_dir)?; + write_option_line!(executable)?; + write_option_line!(pointer_width)?; + write_line!(build_flags)?; Ok(()) } } -fn parse_option_string(string: String) -> Result, ::Err> { - if string == "null" { - Ok(None) - } else { - string.parse().map(Some) - } -} - #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct PythonVersion { pub major: u8, @@ -211,12 +330,37 @@ impl Display for PythonVersion { } } +impl FromStr for PythonVersion { + type Err = crate::errors::Error; + + fn from_str(value: &str) -> Result { + let mut split = value.splitn(2, '.'); + let (major, minor) = ( + split + .next() + .expect("first splitn value should always be present"), + split.next().ok_or("expected major.minor version")?, + ); + Ok(Self { + major: major.parse().context("failed to parse major version")?, + minor: minor.parse().context("failed to parse minor version")?, + }) + } +} + #[derive(Debug, Copy, Clone, PartialEq)] pub enum PythonImplementation { CPython, PyPy, } +impl PythonImplementation { + #[doc(hidden)] + pub fn is_pypy(self) -> bool { + self == PythonImplementation::PyPy + } +} + impl Display for PythonImplementation { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -274,11 +418,17 @@ impl GetPrimitive for HashMap { struct CrossCompileConfig { lib_dir: PathBuf, - version: Option, + version: Option, os: String, arch: String, } +pub fn any_cross_compiling_env_vars_set() -> bool { + env::var_os("PYO3_CROSS").is_some() + || env::var_os("PYO3_CROSS_LIB_DIR").is_some() + || env::var_os("PYO3_CROSS_PYTHON_VERSION").is_some() +} + fn cross_compiling() -> Result> { let cross = env_var("PYO3_CROSS"); let cross_lib_dir = env_var("PYO3_CROSS_LIB_DIR"); @@ -333,10 +483,12 @@ fn cross_compiling() -> Result> { arch: target_arch.unwrap(), version: cross_python_version .map(|os_string| { - os_string + let utf8_str = os_string .to_str() - .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.") - .map(str::to_owned) + .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.")?; + utf8_str + .parse() + .context("failed to parse PYO3_CROSS_PYTHON_VERSION") }) .transpose()?, })) @@ -389,6 +541,7 @@ impl FromStr for BuildFlag { /// /// see Misc/SpecialBuilds.txt in the python source for what these mean. #[cfg_attr(test, derive(Debug, PartialEq))] +#[derive(Default)] pub struct BuildFlags(pub HashSet); impl BuildFlags { @@ -401,6 +554,10 @@ impl BuildFlags { BuildFlag::COUNT_ALLOCS, ]; + pub fn new() -> Self { + BuildFlags(HashSet::new()) + } + fn from_config_map(config_map: &HashMap) -> Self { Self( BuildFlags::ALL @@ -418,7 +575,7 @@ impl BuildFlags { /// Examine python's compile flags to pass to cfg by launching /// the interpreter and printing variables of interest from /// sysconfig.get_config_vars. - fn from_interpreter(interpreter: &Path) -> Result { + fn from_interpreter(interpreter: impl AsRef) -> Result { // If we're on a Windows host, then Python won't have any useful config vars if cfg!(windows) { return Ok(Self::windows_hardcoded()); @@ -431,7 +588,7 @@ impl BuildFlags { script.push_str(&format!("print(config.get('{}', '0'))\n", k)); } - let stdout = run_python_script(interpreter, &script)?; + let stdout = run_python_script(interpreter.as_ref(), &script)?; let split_stdout: Vec<&str> = stdout.trim_end().lines().collect(); ensure!( split_stdout.len() == BuildFlags::ALL.len(), @@ -453,18 +610,10 @@ impl BuildFlags { // query the interpreter directly for its build flags. let mut flags = HashSet::new(); flags.insert(BuildFlag::WITH_THREAD); - - // Uncomment these manually if your python was built with these and you want - // the cfg flags to be set in rust. - // - // flags.insert(BuildFlag::Py_DEBUG); - // flags.insert(BuildFlag::Py_REF_DEBUG); - // flags.insert(BuildFlag::Py_TRACE_REFS); - // flags.insert(BuildFlag::COUNT_ALLOCS; Self(flags) } - fn abi3() -> Self { + pub fn abi3() -> Self { let mut flags = HashSet::new(); flags.insert(BuildFlag::WITH_THREAD); Self(flags) @@ -488,6 +637,33 @@ impl BuildFlags { } } +impl Display for BuildFlags { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut first = true; + for flag in &self.0 { + if !first { + write!(f, ",")?; + } else { + first = false; + } + write!(f, "{}", flag)?; + } + Ok(()) + } +} + +impl FromStr for BuildFlags { + type Err = std::convert::Infallible; + + fn from_str(value: &str) -> Result { + let mut flags = HashSet::new(); + for flag in value.split(',') { + flags.insert(flag.parse().unwrap()); + } + Ok(BuildFlags(flags)) + } +} + fn parse_script_output(output: &str) -> HashMap { output .lines() @@ -550,7 +726,7 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool { /// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2]) /// ``` /// -/// Where get_platform returns a kebab-case formated string containing the os, the architecture and +/// Where get_platform returns a kebab-case formatted string containing the os, the architecture and /// possibly the os' kernel version (not the case on linux). However, when installed using a package /// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory. /// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`. @@ -616,7 +792,7 @@ fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec vec![f.path()], Ok(f) if starts_with(f, "build") => search_lib_dir(f.path(), cross), Ok(f) if starts_with(f, "lib.") => { @@ -637,8 +813,7 @@ fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec search_lib_dir(f.path(), cross), _ => continue, - }; - sysconfig_paths.extend(sysc); + }); } // If we got more than one file, only take those that contain the arch name. // For ubuntu 20.04 with host architecture x86_64 and a foreign architecture of armhf @@ -675,11 +850,10 @@ fn load_cross_compile_from_sysconfigdata( let major = sysconfig_data.get_numeric("version_major")?; let minor = sysconfig_data.get_numeric("version_minor")?; - let ld_version = match sysconfig_data.get("LDVERSION") { - Some(s) => s.clone(), - None => format!("{}.{}", major, minor), - }; - let calcsize_pointer = sysconfig_data.get_numeric("SIZEOF_VOID_P").ok(); + let pointer_width = sysconfig_data + .get_numeric("SIZEOF_VOID_P") + .map(|bytes_width: u32| bytes_width * 8) + .ok(); let soabi = match sysconfig_data.get("SOABI") { Some(s) => s, None => bail!("sysconfigdata did not define SOABI"), @@ -695,15 +869,18 @@ fn load_cross_compile_from_sysconfigdata( }; Ok(InterpreterConfig { + implementation, version, - libdir: cross_compile_config.lib_dir.to_str().map(String::from), shared: sysconfig_data.get_bool("Py_ENABLE_SHARED")?, abi3: is_abi3(), - ld_version: Some(ld_version), - base_prefix: None, + lib_dir: cross_compile_config.lib_dir.to_str().map(String::from), + lib_name: Some(default_lib_name_unix( + version, + implementation, + sysconfig_data.get("LDVERSION").map(String::as_str), + )), executable: None, - calcsize_pointer, - implementation, + pointer_width, build_flags: BuildFlags::from_config_map(&sysconfig_data).fixup(version, implementation), }) } @@ -711,40 +888,25 @@ fn load_cross_compile_from_sysconfigdata( fn windows_hardcoded_cross_compile( cross_compile_config: CrossCompileConfig, ) -> Result { - let (major, minor) = if let Some(version) = cross_compile_config.version { - let mut parts = version.split('.'); - match ( - parts.next().and_then(|major| major.parse().ok()), - parts.next().and_then(|minor| minor.parse().ok()), - parts.next(), - ) { - (Some(major), Some(minor), None) => (major, minor), - _ => bail!( - "Expected major.minor version (e.g. 3.9) for PYO3_CROSS_PYTHON_VERSION, got `{}`", - version - ), - } - } else if let Some(minor_version) = get_abi3_minor_version() { - (3, minor_version) - } else { - bail!("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.") - }; + let version = cross_compile_config.version.or_else(get_abi3_version) + .ok_or("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.")?; Ok(InterpreterConfig { - version: PythonVersion { major, minor }, - libdir: cross_compile_config.lib_dir.to_str().map(String::from), + implementation: PythonImplementation::CPython, + version, shared: true, abi3: is_abi3(), - ld_version: None, - base_prefix: None, + lib_name: Some(default_lib_name_windows(version, false, false)), + lib_dir: cross_compile_config.lib_dir.to_str().map(String::from), executable: None, - calcsize_pointer: None, - implementation: PythonImplementation::CPython, + pointer_width: None, build_flags: BuildFlags::windows_hardcoded(), }) } -fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result { +fn load_cross_compile_config( + cross_compile_config: CrossCompileConfig, +) -> Result { match cargo_env_var("CARGO_CFG_TARGET_FAMILY") { // Configure for unix platforms using the sysconfigdata file Some(os) if os == "unix" => load_cross_compile_from_sysconfigdata(cross_compile_config), @@ -754,7 +916,10 @@ fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result load_cross_compile_from_sysconfigdata(cross_compile_config), // Waiting for users to tell us what they expect on their target platform Some(os) => bail!( - "Unsupported target OS family for cross-compilation: {:?}", + "Unknown target OS family for cross-compilation: {:?}.\n\ + \n\ + Please set the PYO3_CONFIG_FILE environment variable to a config suitable for your \ + target interpreter.", os ), // Unknown os family - try to do something useful @@ -762,6 +927,37 @@ fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result String { + if abi3 { + WINDOWS_ABI3_LIB_NAME.to_owned() + } else if mingw { + // https://packages.msys2.org/base/mingw-w64-python + format!("python{}.{}", version.major, version.minor) + } else { + format!("python{}{}", version.major, version.minor) + } +} + +fn default_lib_name_unix( + version: PythonVersion, + implementation: PythonImplementation, + ld_version: Option<&str>, +) -> String { + match implementation { + PythonImplementation::CPython => match ld_version { + Some(ld_version) => format!("python{}", ld_version), + None => format!("python{}.{}", version.major, version.minor), + }, + PythonImplementation::PyPy => format!("pypy{}-c", version.major), + } +} + /// Run a python script using the specified interpreter binary. fn run_python_script(interpreter: &Path, script: &str) -> Result { let out = Command::new(interpreter) @@ -806,11 +1002,13 @@ fn get_venv_path() -> Option { } } -/// Attempts to locate a python interpreter. Locations are checked in the order listed: -/// 1. If `PYO3_PYTHON` is set, this intepreter is used. -/// 2. If in a virtualenv, that environment's interpreter is used. -/// 3. `python`, if this is functional a Python 3.x interpreter -/// 4. `python3`, as above +/// Attempts to locate a python interpreter. +/// +/// Locations are checked in the order listed: +/// 1. If `PYO3_PYTHON` is set, this interpreter is used. +/// 2. If in a virtualenv, that environment's interpreter is used. +/// 3. `python`, if this is functional a Python 3.x interpreter +/// 4. `python3`, as above pub fn find_interpreter() -> Result { if let Some(exe) = env_var("PYO3_PYTHON") { Ok(exe.into()) @@ -839,135 +1037,52 @@ pub fn find_interpreter() -> Result { } } -/// Extract compilation vars from the specified interpreter. -pub fn get_config_from_interpreter(interpreter: &Path) -> Result { - let script = r#" -# Allow the script to run on Python 2, so that nicer error can be printed later. -from __future__ import print_function - -import os.path -import platform -import struct -import sys -from sysconfig import get_config_var - -PYPY = platform.python_implementation() == "PyPy" - -# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue -# so that the version mismatch can be reported in a nicer way later. -base_prefix = getattr(sys, "base_prefix", None) - -if base_prefix: - # Anaconda based python distributions have a static python executable, but include - # the shared library. Use the shared library for embedding to avoid rust trying to - # LTO the static library (and failing with newer gcc's, because it is old). - ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta")) -else: - ANACONDA = False - -def print_if_set(varname, value): - if value is not None: - print(varname, value) - -# Windows always uses shared linking -WINDOWS = hasattr(platform, "win32_ver") - -# macOS framework packages use shared linking -FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) - -# unix-style shared library enabled -SHARED = bool(get_config_var("Py_ENABLE_SHARED")) - -print("version_major", sys.version_info[0]) -print("version_minor", sys.version_info[1]) -print("implementation", platform.python_implementation()) -print_if_set("libdir", get_config_var("LIBDIR")) -print_if_set("ld_version", get_config_var("LDVERSION")) -print_if_set("base_prefix", base_prefix) -print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) -print("executable", sys.executable) -print("calcsize_pointer", struct.calcsize("P")) -"#; - let output = run_python_script(interpreter, script)?; - let map: HashMap = parse_script_output(&output); - let shared = map["shared"].as_str() == "True"; - - let version = PythonVersion { - major: map["version_major"] - .parse() - .context("failed to parse major version")?, - minor: map["version_minor"] - .parse() - .context("failed to parse minor version")?, - }; - - let implementation = map["implementation"].parse()?; - - Ok(InterpreterConfig { - version, - implementation, - libdir: map.get("libdir").cloned(), - shared, - abi3: is_abi3(), - ld_version: map.get("ld_version").cloned(), - base_prefix: map.get("base_prefix").cloned(), - executable: map.get("executable").cloned(), - calcsize_pointer: Some( - map["calcsize_pointer"] - .parse() - .context("failed to parse calcsize_pointer")?, - ), - build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation), - }) +pub fn get_abi3_version() -> Option { + let minor_version = (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) + .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()); + minor_version.map(|minor| PythonVersion { major: 3, minor }) } -fn get_abi3_minor_version() -> Option { - (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) - .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()) -} +/// Lowers the configured version to the abi3 version, if set. +fn fixup_config_for_abi3( + config: &mut InterpreterConfig, + abi3_version: Option, +) -> Result<()> { + if let Some(version) = abi3_version { + ensure!( + version <= config.version, + "cannot set a minimum Python version {} higher than the interpreter version {} \ + (the minimum Python version is implied by the abi3-py3{} feature)", + version, + config.version, + version.minor, + ); -pub fn make_interpreter_config() -> Result { - let abi3_version = get_abi3_minor_version(); - - // If PYO3_NO_PYTHON is set with abi3, we can build PyO3 without calling Python. - if let Some(abi3_minor_version) = abi3_version { - if env_var("PYO3_NO_PYTHON").is_some() { - return Ok(InterpreterConfig { - version: PythonVersion { - major: 3, - minor: abi3_minor_version, - }, - implementation: PythonImplementation::CPython, - abi3: true, - libdir: None, - build_flags: BuildFlags::abi3(), - base_prefix: None, - calcsize_pointer: None, - executable: None, - ld_version: None, - shared: true, - }); - } + config.version = version; } + Ok(()) +} +/// Generates an interpreter config suitable for cross-compilation. +/// +/// This must be called from PyO3's build script, because it relies on environment variables such as +/// CARGO_CFG_TARGET_OS which aren't available at any other time. +pub fn make_cross_compile_config() -> Result> { let mut interpreter_config = if let Some(paths) = cross_compiling()? { - load_cross_compile_info(paths)? + load_cross_compile_config(paths)? } else { - get_config_from_interpreter(&find_interpreter()?)? + return Ok(None); }; + fixup_config_for_abi3(&mut interpreter_config, get_abi3_version())?; + Ok(Some(interpreter_config)) +} - // Fixup minor version if abi3-pyXX feature set - if let Some(abi3_minor_version) = abi3_version { - ensure!( - abi3_minor_version <= interpreter_config.version.minor, - "You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}", - abi3_minor_version, - interpreter_config.version.minor - ); - - interpreter_config.version.minor = abi3_minor_version; - } - +/// Generates an interpreter config which will be hard-coded into the pyo3-build-config crate. +/// Only used by `pyo3-build-config` build script. +#[allow(dead_code)] +pub fn make_interpreter_config() -> Result { + let mut interpreter_config = InterpreterConfig::from_interpreter(find_interpreter()?)?; + fixup_config_for_abi3(&mut interpreter_config, get_abi3_version())?; Ok(interpreter_config) } @@ -978,16 +1093,15 @@ mod tests { use super::*; #[test] - fn test_read_write_roundtrip() { + fn test_config_file_roundtrip() { let config = InterpreterConfig { abi3: true, - base_prefix: Some("base_prefix".into()), build_flags: BuildFlags::abi3(), - calcsize_pointer: Some(32), + pointer_width: Some(32), executable: Some("executable".into()), implementation: PythonImplementation::CPython, - ld_version: Some("ld_version".into()), - libdir: Some("libdir".into()), + lib_name: Some("lib_name".into()), + lib_dir: Some("lib_dir".into()), shared: true, version: MINIMUM_SUPPORTED_VERSION, }; @@ -1003,18 +1117,17 @@ mod tests { let config = InterpreterConfig { abi3: false, - base_prefix: None, build_flags: { let mut flags = HashSet::new(); flags.insert(BuildFlag::Py_DEBUG); flags.insert(BuildFlag::Other(String::from("Py_SOME_FLAG"))); BuildFlags(flags) }, - calcsize_pointer: None, + pointer_width: None, executable: None, implementation: PythonImplementation::PyPy, - ld_version: None, - libdir: None, + lib_dir: None, + lib_name: None, shared: true, version: PythonVersion { major: 3, @@ -1030,6 +1143,30 @@ mod tests { ); } + #[test] + fn test_config_file_defaults() { + // Only version is required + assert_eq!( + InterpreterConfig::from_reader(Cursor::new("version=3.6")).unwrap(), + InterpreterConfig { + version: PythonVersion { major: 3, minor: 6 }, + implementation: PythonImplementation::CPython, + shared: true, + abi3: false, + lib_name: None, + lib_dir: None, + executable: None, + pointer_width: None, + build_flags: BuildFlags::default(), + } + ) + } + + #[test] + fn build_flags_default() { + assert_eq!(BuildFlags::default(), BuildFlags::new()); + } + #[test] fn build_flags_from_config_map() { let mut config_map = HashMap::new(); @@ -1053,7 +1190,7 @@ mod tests { #[test] fn build_flags_fixup_py36_debug() { - let mut build_flags = BuildFlags(HashSet::new()); + let mut build_flags = BuildFlags::new(); build_flags.0.insert(BuildFlag::Py_DEBUG); build_flags = build_flags.fixup( @@ -1068,7 +1205,7 @@ mod tests { #[test] fn build_flags_fixup_py37_debug() { - let mut build_flags = BuildFlags(HashSet::new()); + let mut build_flags = BuildFlags::new(); build_flags.0.insert(BuildFlag::Py_DEBUG); build_flags = build_flags.fixup(PythonVersion::PY37, PythonImplementation::CPython); @@ -1082,7 +1219,7 @@ mod tests { #[test] fn build_flags_fixup_pypy() { - let mut build_flags = BuildFlags(HashSet::new()); + let mut build_flags = BuildFlags::new(); build_flags = build_flags.fixup( PythonVersion { major: 3, minor: 6 }, @@ -1092,4 +1229,135 @@ mod tests { // PyPy always has WITH_THREAD assert!(build_flags.0.contains(&BuildFlag::WITH_THREAD)); } + + #[test] + fn parse_script_output() { + let output = "foo bar\nbar foobar\n\n"; + let map = super::parse_script_output(output); + assert_eq!(map.len(), 2); + assert_eq!(map["foo"], "bar"); + assert_eq!(map["bar"], "foobar"); + } + + #[test] + fn config_from_interpreter() { + // Smoke test to just see whether this works + // + // PyO3's CI is dependent on Python being installed, so this should be reliable. + assert!(make_interpreter_config().is_ok()) + } + + #[test] + fn windows_hardcoded_cross_compile() { + let cross_config = CrossCompileConfig { + lib_dir: "C:\\some\\path".into(), + version: Some(PythonVersion { major: 3, minor: 6 }), + os: "os".into(), + arch: "arch".into(), + }; + + assert_eq!( + super::windows_hardcoded_cross_compile(cross_config).unwrap(), + InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { major: 3, minor: 6 }, + shared: true, + abi3: false, + lib_name: Some("python36".into()), + lib_dir: Some("C:\\some\\path".into()), + executable: None, + pointer_width: None, + build_flags: BuildFlags::windows_hardcoded() + } + ); + } + + #[test] + fn default_lib_name_windows() { + assert_eq!( + super::default_lib_name_windows(PythonVersion { major: 3, minor: 6 }, false, false), + "python36", + ); + assert_eq!( + super::default_lib_name_windows(PythonVersion { major: 3, minor: 6 }, true, false), + "python3", + ); + assert_eq!( + super::default_lib_name_windows(PythonVersion { major: 3, minor: 6 }, false, true), + "python3.6", + ); + assert_eq!( + super::default_lib_name_windows(PythonVersion { major: 3, minor: 6 }, true, true), + "python3", + ); + } + + #[test] + fn default_lib_name_unix() { + use PythonImplementation::*; + // Defaults to pythonX.Y for CPython + assert_eq!( + super::default_lib_name_unix(PythonVersion { major: 3, minor: 6 }, CPython, None), + "python3.6", + ); + assert_eq!( + super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, CPython, None), + "python3.9", + ); + // Can use ldversion to override for CPython + assert_eq!( + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + CPython, + Some("3.7md") + ), + "python3.7md", + ); + + // PyPy ignores ldversion + assert_eq!( + super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, Some("3.7md")), + "pypy3-c", + ); + } + + #[test] + fn interpreter_version_reduced_to_abi3() { + let mut config = InterpreterConfig { + abi3: true, + build_flags: BuildFlags::new(), + pointer_width: None, + executable: None, + implementation: PythonImplementation::CPython, + lib_dir: None, + lib_name: None, + shared: true, + version: PythonVersion { major: 3, minor: 7 }, + }; + + fixup_config_for_abi3(&mut config, Some(PythonVersion { major: 3, minor: 6 })).unwrap(); + assert_eq!(config.version, PythonVersion { major: 3, minor: 6 }); + } + + #[test] + fn abi3_version_cannot_be_higher_than_interpreter() { + let mut config = InterpreterConfig { + abi3: true, + build_flags: BuildFlags::new(), + pointer_width: None, + executable: None, + implementation: PythonImplementation::CPython, + lib_dir: None, + lib_name: None, + shared: true, + version: PythonVersion { major: 3, minor: 6 }, + }; + + assert!( + fixup_config_for_abi3(&mut config, Some(PythonVersion { major: 3, minor: 7 })) + .unwrap_err() + .to_string() + .contains("cannot set a minimum Python version 3.7 higher than the interpreter version 3.6") + ); + } } diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 0bc947aabed..b677b95473b 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -1,37 +1,23 @@ //! Configuration used by PyO3 for conditional support of varying Python versions. //! -//! The public APIs exposed, [`use_pyo3_cfgs`] and [`add_extension_module_link_args`] are intended -//! to be called from build scripts to simplify building crates which depend on PyO3. +//! This crate exposes two functions, [`use_pyo3_cfgs`] and [`add_extension_module_link_args`], +//! which are intended to be called from build scripts to simplify building crates which depend on +//! PyO3. +//! +//! It used internally by the PyO3 crate's build script to apply the same configuration. -#[doc(hidden)] -pub mod errors; +mod errors; mod impl_; -use once_cell::sync::OnceCell; +use std::io::Cursor; -// Used in PyO3's build.rs -#[doc(hidden)] -pub use impl_::{ - cargo_env_var, env_var, find_interpreter, get_config_from_interpreter, make_interpreter_config, - InterpreterConfig, PythonImplementation, PythonVersion, -}; +use once_cell::sync::OnceCell; -/// Reads the configuration written by PyO3's build.rs -/// -/// Because this will never change in a given compilation run, this is cached in a `once_cell`. -#[doc(hidden)] -pub fn get() -> &'static InterpreterConfig { - static CONFIG: OnceCell = OnceCell::new(); - CONFIG.get_or_init(|| { - let config_file = std::fs::File::open(PATH).expect("config file missing"); - let reader = std::io::BufReader::new(config_file); - InterpreterConfig::from_reader(reader).expect("failed to parse config file") - }) -} +use impl_::InterpreterConfig; -/// Path where PyO3's build.rs will write configuration. +// Used in `pyo3-macros-backend`; may expose this in a future release. #[doc(hidden)] -pub const PATH: &str = concat!(env!("OUT_DIR"), "/pyo3-build-config.txt"); +pub use impl_::PythonVersion; /// Adds all the [`#[cfg]` flags](index.html) to the current compilation. /// @@ -57,8 +43,140 @@ pub fn use_pyo3_cfgs() { /// This is currently a no-op on non-macOS platforms, however may emit additional linker arguments /// in future if deemed necessarys. pub fn add_extension_module_link_args() { - if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "macos" { - println!("cargo:rustc-cdylib-link-arg=-undefined"); - println!("cargo:rustc-cdylib-link-arg=dynamic_lookup"); + _add_extension_module_link_args( + &impl_::cargo_env_var("CARGO_CFG_TARGET_OS").unwrap(), + std::io::stdout(), + ) +} + +fn _add_extension_module_link_args(target_os: &str, mut writer: impl std::io::Write) { + if target_os == "macos" { + writeln!(writer, "cargo:rustc-cdylib-link-arg=-undefined").unwrap(); + writeln!(writer, "cargo:rustc-cdylib-link-arg=dynamic_lookup").unwrap(); + } +} + +/// Loads the configuration determined from the build environment. +/// +/// Because this will never change in a given compilation run, this is cached in a `once_cell`. +#[doc(hidden)] +pub fn get() -> &'static InterpreterConfig { + static CONFIG: OnceCell = OnceCell::new(); + CONFIG.get_or_init(|| { + if !CONFIG_FILE.is_empty() { + InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE)) + } else if !ABI3_CONFIG.is_empty() { + Ok(abi3_config()) + } else if impl_::any_cross_compiling_env_vars_set() { + InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH) + } else { + InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG)) + } + .expect("failed to parse PyO3 config file") + }) +} + +/// Path where PyO3's build.rs will write configuration by default. +#[doc(hidden)] +const DEFAULT_CROSS_COMPILE_CONFIG_PATH: &str = + concat!(env!("OUT_DIR"), "/pyo3-cross-compile-config.txt"); + +/// Build configuration provided by `PYO3_CONFIG_FILE`. May be empty if env var not set. +#[doc(hidden)] +const CONFIG_FILE: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config-file.txt")); + +/// Build configuration set if abi3 features enabled and `PYO3_NO_PYTHON` env var present. Empty if +/// not both present. +#[doc(hidden)] +const ABI3_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config-abi3.txt")); + +/// Build configuration discovered by `pyo3-build-config` build script. Not aware of +/// cross-compilation settings. +#[doc(hidden)] +const HOST_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config.txt")); + +fn abi3_config() -> InterpreterConfig { + let mut interpreter_config = InterpreterConfig::from_reader(Cursor::new(ABI3_CONFIG)) + .expect("failed to parse hardcoded PyO3 abi3 config"); + // If running from a build script on Windows, tweak the hardcoded abi3 config to contain + // the standard lib name (this is necessary so that abi3 extension modules using + // PYO3_NO_PYTHON on Windows can link) + if std::env::var("CARGO_CFG_TARGET_OS").map_or(false, |target_os| target_os == "windows") { + assert_eq!(interpreter_config.lib_name, None); + interpreter_config.lib_name = Some("python3".to_owned()) + } + interpreter_config +} + +/// Private exports used in PyO3's build.rs +/// +/// Please don't use these - they could change at any time. +#[doc(hidden)] +pub mod pyo3_build_script_impl { + use crate::errors::{Context, Result}; + use std::path::Path; + + use super::*; + + pub mod errors { + pub use crate::errors::*; + } + pub use crate::impl_::{ + cargo_env_var, env_var, make_cross_compile_config, InterpreterConfig, PythonVersion, + }; + + /// Gets the configuration for use from PyO3's build script. + /// + /// Differs from .get() above only in the cross-compile case, where PyO3's build script is + /// required to generate a new config (as it's the first build script which has access to the + /// correct value for CARGO_CFG_TARGET_OS). + pub fn resolve_interpreter_config() -> Result { + if !CONFIG_FILE.is_empty() { + InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE)) + } else if !ABI3_CONFIG.is_empty() { + Ok(abi3_config()) + } else if let Some(interpreter_config) = impl_::make_cross_compile_config()? { + // This is a cross compile and need to write the config file. + let path = Path::new(DEFAULT_CROSS_COMPILE_CONFIG_PATH); + let parent_dir = path.parent().ok_or_else(|| { + format!( + "failed to resolve parent directory of config file {}", + path.display() + ) + })?; + std::fs::create_dir_all(&parent_dir).with_context(|| { + format!( + "failed to create config file directory {}", + parent_dir.display() + ) + })?; + interpreter_config.to_writer(&mut std::fs::File::create(&path).with_context( + || format!("failed to create config file at {}", path.display()), + )?)?; + Ok(interpreter_config) + } else { + InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extension_module_link_args() { + let mut buf = Vec::new(); + + // Does nothing on non-mac + _add_extension_module_link_args("windows", &mut buf); + assert_eq!(buf, Vec::new()); + + _add_extension_module_link_args("macos", &mut buf); + assert_eq!( + std::str::from_utf8(&buf).unwrap(), + "cargo:rustc-cdylib-link-arg=-undefined\n\ + cargo:rustc-cdylib-link-arg=dynamic_lookup\n" + ); } }