diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf0a836f4..e3a915075 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,10 +110,11 @@ jobs: with: python-version: "3.10.0" - name: test cross compiling with zig - if: matrix.os != 'windows-latest' run: | rustup target add aarch64-unknown-linux-gnu + rustup target add aarch64-unknown-linux-musl cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-gnu --zig + cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-musl --zig test-auditwheel: name: Test Auditwheel diff --git a/Cargo.lock b/Cargo.lock index 7273f72bd..3ee891dfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1048,9 +1048,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lddtree" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "738eccba56c448014efe5a639caf93eb67c3c73d0443e2be7c874bd642d9c718" +checksum = "b67aac3709c5c1f31dfc635ba91bfbd137b7d599dda3f325798cc84ba569d1cd" dependencies = [ "fs-err", "glob", @@ -1125,6 +1125,7 @@ dependencies = [ "regex", "reqwest", "rpassword", + "semver", "serde", "serde_json", "sha2 0.10.0", @@ -1821,18 +1822,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9875c23cf305cd1fd7eb77234cbb705f21ea6a72c637a5c6db5fe4b8e7f008" +checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc0db5cb2556c0e558887d9bbdcf6ac4471e83ff66cf696e5419024d1606276" +checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537" dependencies = [ "proc-macro2", "quote", @@ -1841,9 +1842,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.73" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5" +checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142" dependencies = [ "itoa 1.0.1", "ryu", diff --git a/Cargo.toml b/Cargo.toml index d37ce0ef5..0314aca1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,11 +56,12 @@ ignore = "0.4.18" dialoguer = "0.9.0" console = "0.15.0" minijinja = "0.10.0" -lddtree = "0.2.0" +lddtree = "0.2.5" cc = "1.0.72" clap = { version = "3.0.0", features = ["derive", "env", "wrap_help"] } clap_complete = "3.0.0" clap_complete_fig = "3.0.0" +semver = "1.0.4" [dev-dependencies] indoc = "1.0.3" diff --git a/src/build_context.rs b/src/build_context.rs index fe81c2304..9829d2e61 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -275,7 +275,7 @@ impl BuildContext { } })?; let external_libs = if should_repair && !self.editable { - let sysroot = get_sysroot_path(&self.target)?; + let sysroot = get_sysroot_path(&self.target).unwrap_or_else(|_| PathBuf::from("/")); find_external_libs(&artifact, &policy, sysroot).with_context(|| { if let Some(platform_tag) = platform_tag { format!("Error repairing wheel for {} compliance", platform_tag) @@ -707,10 +707,13 @@ fn relpath(to: &Path, from: &Path) -> PathBuf { /// Get sysroot path from target C compiler /// /// Currently only gcc is supported, clang doesn't have a `--print-sysroot` option -/// TODO: allow specify sysroot from environment variable? fn get_sysroot_path(target: &Target) -> Result { use crate::target::get_host_target; + if let Some(sysroot) = std::env::var_os("TARGET_SYSROOT") { + return Ok(PathBuf::from(sysroot)); + } + let host_triple = get_host_target()?; let target_triple = target.target_triple(); if host_triple != target_triple { @@ -725,6 +728,10 @@ fn get_sysroot_path(target: &Target) -> Result { let compiler = build .try_get_compiler() .with_context(|| format!("Failed to get compiler for {}", target_triple))?; + // Only GNU like compilers support `--print-sysroot` + if !compiler.is_like_gnu() { + return Ok(PathBuf::from("/")); + } let path = compiler.path(); let out = Command::new(path) .arg("--print-sysroot") diff --git a/src/compile.rs b/src/compile.rs index 48d509d3b..e3d6188ba 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -1,17 +1,13 @@ use crate::build_context::BridgeModel; use crate::python_interpreter::InterpreterKind; -use crate::target::Arch; -use crate::{BuildContext, PlatformTag, PythonInterpreter}; +use crate::zig::prepare_zig_linker; +use crate::{BuildContext, PythonInterpreter}; use anyhow::{anyhow, bail, Context, Result}; use fat_macho::FatWriter; use fs_err::{self as fs, File}; use std::collections::HashMap; use std::env; -#[cfg(target_family = "unix")] -use std::fs::OpenOptions; -use std::io::{BufReader, Read, Write}; -#[cfg(target_family = "unix")] -use std::os::unix::fs::OpenOptionsExt; +use std::io::{BufReader, Read}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::str; @@ -105,87 +101,6 @@ fn compile_universal2( Ok(result) } -/// We want to use `zig cc` as linker and c compiler. We want to call `python -m ziglang cc`, but -/// cargo only accepts a path to an executable as linker, so we add a wrapper script. We then also -/// use the wrapper script to pass arguments and substitute an unsupported argument. -/// -/// We create different files for different args because otherwise cargo might skip recompiling even -/// if the linker target changed -fn prepare_zig_linker(context: &BuildContext) -> Result<(PathBuf, PathBuf)> { - let target = &context.target; - let arch = if target.cross_compiling() { - if matches!(target.target_arch(), Arch::Armv7L) { - "armv7".to_string() - } else { - target.target_arch().to_string() - } - } else { - "native".to_string() - }; - let (zig_cc, zig_cxx, cc_args) = match context.platform_tag { - // Not sure branch even has any use case, but it doesn't hurt to support it - None | Some(PlatformTag::Linux) => ( - "./zigcc-gnu.sh".to_string(), - "./zigcxx-gnu.sh".to_string(), - format!("{}-linux-gnu", arch), - ), - Some(PlatformTag::Musllinux { x, y }) => { - println!("⚠️ Warning: zig with musl is unstable"); - ( - format!("./zigcc-musl-{}-{}.sh", x, y), - format!("./zigcxx-musl-{}-{}.sh", x, y), - format!("{}-linux-musl", arch), - ) - } - Some(PlatformTag::Manylinux { x, y }) => ( - format!("./zigcc-gnu-{}-{}.sh", x, y), - format!("./zigcxx-gnu-{}-{}.sh", x, y), - // https://github.com/ziglang/zig/issues/10050#issuecomment-956204098 - format!( - "${{@/-lgcc_s/-lunwind}} -target {}-linux-gnu.{}.{}", - arch, x, y - ), - ), - }; - - let zig_linker_dir = dirs::cache_dir() - // If the really is no cache dir, cwd will also do - .unwrap_or_else(|| PathBuf::from(".")) - .join(env!("CARGO_PKG_NAME")) - .join(env!("CARGO_PKG_VERSION")); - fs::create_dir_all(&zig_linker_dir)?; - let zig_cc = zig_linker_dir.join(zig_cc); - let zig_cxx = zig_linker_dir.join(zig_cxx); - - let mut zig_cc_file = create_linker_script(&zig_cc)?; - writeln!(&mut zig_cc_file, "#!/bin/bash")?; - writeln!(&mut zig_cc_file, "python -m ziglang cc {}", cc_args)?; - drop(zig_cc_file); - - let mut zig_cxx_file = create_linker_script(&zig_cxx)?; - writeln!(&mut zig_cxx_file, "#!/bin/bash")?; - writeln!(&mut zig_cxx_file, "python -m ziglang c++ {}", cc_args)?; - drop(zig_cxx_file); - - Ok((zig_cc, zig_cxx)) -} - -#[cfg(target_family = "unix")] -fn create_linker_script(path: &Path) -> Result { - let custom_linker_file = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .mode(0o700) - .open(path)?; - Ok(custom_linker_file) -} - -#[cfg(not(target_family = "unix"))] -fn create_linker_script(path: &Path) -> Result { - Ok(File::create(path)?) -} - fn compile_target( context: &BuildContext, python_interpreter: Option<&PythonInterpreter>, diff --git a/src/lib.rs b/src/lib.rs index cfbcfc505..7bbd5326f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,6 +37,7 @@ pub use crate::new_project::{init_project, new_project, GenerateProjectOptions}; pub use crate::pyproject_toml::PyProjectToml; pub use crate::python_interpreter::PythonInterpreter; pub use crate::target::Target; +pub use crate::zig::Zig; pub use auditwheel::PlatformTag; #[cfg(feature = "upload")] pub use { @@ -62,3 +63,4 @@ mod source_distribution; mod target; #[cfg(feature = "upload")] mod upload; +mod zig; diff --git a/src/main.rs b/src/main.rs index c93efe69a..f2c649f02 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use clap::{ArgEnum, IntoApp, Parser}; use clap_complete::Generator; use maturin::{ develop, init_project, new_project, write_dist_info, BridgeModel, BuildOptions, - GenerateProjectOptions, PathWriter, PlatformTag, PythonInterpreter, Target, + GenerateProjectOptions, PathWriter, PlatformTag, PythonInterpreter, Target, Zig, }; #[cfg(feature = "upload")] use maturin::{upload_ui, PublishOpt}; @@ -163,6 +163,9 @@ enum Opt { #[clap(name = "SHELL", parse(try_from_str))] shell: Shell, }, + /// Zig linker wrapper + #[clap(subcommand)] + Zig(Zig), } /// Backend for the PEP 517 integration. Not for human consumption @@ -449,6 +452,11 @@ fn run() -> Result<()> { } } } + Opt::Zig(subcommand) => { + subcommand + .execute() + .context("Failed to create zig wrapper script")?; + } } Ok(()) diff --git a/src/zig.rs b/src/zig.rs new file mode 100644 index 000000000..33dbaf2fa --- /dev/null +++ b/src/zig.rs @@ -0,0 +1,222 @@ +//! Using `zig cc` as c compiler and linker to target a specific glibc/musl version +//! as alternative to the manylinux docker container and for easier cross compiling + +use crate::target::Arch; +use crate::{BuildContext, PlatformTag}; +use anyhow::{bail, Context, Result}; +use fs_err as fs; +use std::env; +#[cfg(target_family = "unix")] +use std::fs::OpenOptions; +use std::io::Write; +#[cfg(target_family = "unix")] +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::process::{self, Command}; +use std::str; + +/// Zig linker wrapper +#[derive(Debug, clap::Parser)] +#[clap(name = "zig", setting = clap::AppSettings::Hidden)] +pub enum Zig { + /// `zig cc` wrapper + #[clap(name = "cc", setting = clap::AppSettings::TrailingVarArg)] + Cc { + /// `zig cc` arguments + #[clap(takes_value = true, multiple_values = true)] + args: Vec, + }, + /// `zig c++` wrapper + #[clap(name = "c++", setting = clap::AppSettings::TrailingVarArg)] + Cxx { + /// `zig c++` arguments + #[clap(takes_value = true, multiple_values = true)] + args: Vec, + }, +} + +impl Zig { + /// Execute the underlying zig command + pub fn execute(&self) -> Result<()> { + let (cmd, cmd_args) = match self { + Zig::Cc { args } => ("cc", args), + Zig::Cxx { args } => ("c++", args), + }; + // Replace libgcc_s with libunwind + let cmd_args: Vec = cmd_args + .iter() + .map(|arg| { + let arg = if arg == "-lgcc_s" { + "-lunwind".to_string() + } else if arg.starts_with('@') && arg.ends_with("linker-arguments") { + // rustc passes arguments to linker via an @-file when arguments are too long + // See https://github.com/rust-lang/rust/issues/41190 + let content = fs::read(arg.trim_start_matches('@'))?; + let link_args = str::from_utf8(&content)?.replace("-lgcc_s", "-lunwind"); + fs::write(arg.trim_start_matches('@'), link_args.as_bytes())?; + arg.to_string() + } else { + arg.to_string() + }; + Ok(arg) + }) + .collect::>()?; + let (zig, zig_args) = Self::find_zig()?; + let mut child = Command::new(zig) + .args(zig_args) + .arg(cmd) + .args(cmd_args) + .spawn() + .with_context(|| format!("Failed to run `zig {}`", cmd))?; + let status = child.wait().expect("Failed to wait on zig child process"); + if !status.success() { + process::exit(status.code().unwrap_or(1)); + } + Ok(()) + } + + /// Search for `python -m ziglang` first and for `zig` second. + /// That way we use the zig from `maturin[ziglang]` first, + /// but users or distributions can also insert their own zig + fn find_zig() -> Result<(String, Vec)> { + Self::find_zig_python() + .or_else(|_| Self::find_zig_bin()) + .context("Failed to find zig") + } + + /// Detect the plain zig binary + fn find_zig_bin() -> Result<(String, Vec)> { + let output = Command::new("zig").arg("version").output()?; + let version_str = + str::from_utf8(&output.stdout).context("`zig version` didn't return utf8 output")?; + Self::validate_zig_version(version_str)?; + Ok(("zig".to_string(), Vec::new())) + } + + /// Detect the Python ziglang package + fn find_zig_python() -> Result<(String, Vec)> { + let output = Command::new("python3") + .args(&["-m", "ziglang", "version"]) + .output()?; + let version_str = str::from_utf8(&output.stdout) + .context("`python3 -m ziglang version` didn't return utf8 output")?; + Self::validate_zig_version(version_str)?; + Ok(( + "python3".to_string(), + vec!["-m".to_string(), "ziglang".to_string()], + )) + } + + fn validate_zig_version(version: &str) -> Result<()> { + let min_ver = semver::Version::new(0, 9, 0); + let version = semver::Version::parse(version.trim())?; + if version >= min_ver { + Ok(()) + } else { + bail!( + "zig version {} is too old, need at least {}", + version, + min_ver + ) + } + } +} + +/// We want to use `zig cc` as linker and c compiler. We want to call `python -m ziglang cc`, but +/// cargo only accepts a path to an executable as linker, so we add a wrapper script. We then also +/// use the wrapper script to pass arguments and substitute an unsupported argument. +/// +/// We create different files for different args because otherwise cargo might skip recompiling even +/// if the linker target changed +pub fn prepare_zig_linker(context: &BuildContext) -> Result<(PathBuf, PathBuf)> { + let target = &context.target; + let arch = if target.cross_compiling() { + if matches!(target.target_arch(), Arch::Armv7L) { + "armv7".to_string() + } else { + target.target_arch().to_string() + } + } else { + "native".to_string() + }; + let file_ext = if cfg!(windows) { "bat" } else { "sh" }; + let (zig_cc, zig_cxx, cc_args) = match context.platform_tag { + // Not sure branch even has any use case, but it doesn't hurt to support it + None | Some(PlatformTag::Linux) => ( + format!("zigcc-gnu.{}", file_ext), + format!("zigcxx-gnu.{}", file_ext), + format!("-target {}-linux-gnu", arch), + ), + Some(PlatformTag::Musllinux { x, y }) => { + println!("⚠️ Warning: zig with musl is unstable"); + ( + format!("zigcc-musl-{}-{}.{}", x, y, file_ext), + format!("zigcxx-musl-{}-{}.{}", x, y, file_ext), + format!("-target {}-linux-musl", arch), + ) + } + Some(PlatformTag::Manylinux { x, y }) => ( + format!("zigcc-gnu-{}-{}.{}", x, y, file_ext), + format!("zigcxx-gnu-{}-{}.{}", x, y, file_ext), + // https://github.com/ziglang/zig/issues/10050#issuecomment-956204098 + format!("-target {}-linux-gnu.{}.{}", arch, x, y), + ), + }; + + let zig_linker_dir = dirs::cache_dir() + // If the really is no cache dir, cwd will also do + .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir")) + .join(env!("CARGO_PKG_NAME")) + .join(env!("CARGO_PKG_VERSION")); + fs::create_dir_all(&zig_linker_dir)?; + + let zig_cc = zig_linker_dir.join(zig_cc); + let zig_cxx = zig_linker_dir.join(zig_cxx); + write_linker_wrapper(&zig_cc, "cc", &cc_args)?; + write_linker_wrapper(&zig_cxx, "c++", &cc_args)?; + + Ok((zig_cc, zig_cxx)) +} + +#[cfg(target_family = "unix")] +fn write_linker_wrapper(path: &Path, command: &str, args: &str) -> Result<()> { + let mut custom_linker_file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .mode(0o700) + .open(path)?; + let current_exe = if let Ok(maturin) = env::var("CARGO_BIN_EXE_maturin") { + PathBuf::from(maturin) + } else { + env::current_exe()? + }; + writeln!(&mut custom_linker_file, "#!/bin/bash")?; + writeln!( + &mut custom_linker_file, + "{} zig {} -- {} $@", + current_exe.display(), + command, + args + )?; + Ok(()) +} + +/// Write a zig cc wrapper batch script for windows +#[cfg(not(target_family = "unix"))] +fn write_linker_wrapper(path: &Path, command: &str, args: &str) -> Result<()> { + let mut custom_linker_file = fs::File::create(path)?; + let current_exe = if let Ok(maturin) = env::var("CARGO_BIN_EXE_maturin") { + PathBuf::from(maturin) + } else { + env::current_exe()? + }; + writeln!( + &mut custom_linker_file, + "{} zig {} -- {} %*", + current_exe.display(), + command, + args + )?; + Ok(()) +} diff --git a/tests/common/integration.rs b/tests/common/integration.rs index bd699c680..15d8e5490 100644 --- a/tests/common/integration.rs +++ b/tests/common/integration.rs @@ -16,6 +16,9 @@ pub fn test_integration( ) -> Result<()> { maybe_mock_cargo(); + // Pass CARGO_BIN_EXE_maturin for testing purpose + std::env::set_var("CARGO_BIN_EXE_maturin", env!("CARGO_BIN_EXE_maturin")); + let target = Target::from_target_triple(None)?; let package_string = package.as_ref().join("Cargo.toml").display().to_string();