diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d0410f676..47462c064 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,7 +37,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: clippy - args: --features password-storage -- -D warnings + args: --tests --all-features -- -D warnings black: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 1bc7d2d60..0a09f6841 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,12 +181,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" - [[package]] name = "bytesize" version = "1.1.0" @@ -297,6 +291,12 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "cipher" version = "0.2.5" @@ -786,25 +786,6 @@ dependencies = [ "scroll", ] -[[package]] -name = "h2" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.11.2" @@ -849,40 +830,6 @@ dependencies = [ "digest 0.9.0", ] -[[package]] -name = "http" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" -dependencies = [ - "bytes", - "fnv", - "itoa 1.0.1", -] - -[[package]] -name = "http-body" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" - -[[package]] -name = "httpdate" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" - [[package]] name = "human-panic" version = "1.0.3" @@ -907,43 +854,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "hyper" -version = "0.14.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa 0.4.8", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" -dependencies = [ - "http", - "hyper", - "rustls", - "tokio", - "tokio-rustls", -] - [[package]] name = "idna" version = "0.2.3" @@ -1001,18 +911,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "ipnet" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" - -[[package]] -name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - [[package]] name = "itoa" version = "1.0.1" @@ -1091,7 +989,7 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "maturin" -version = "0.12.7-beta.1" +version = "0.12.7-beta.2" dependencies = [ "anyhow", "base64", @@ -1117,13 +1015,13 @@ dependencies = [ "keyring", "lddtree", "minijinja", + "multipart", "once_cell", "platform-info", "pretty_env_logger", "pyproject-toml", "python-pkginfo", "regex", - "reqwest", "rpassword", "semver", "serde", @@ -1136,6 +1034,7 @@ dependencies = [ "textwrap", "thiserror", "toml", + "ureq", "zip", ] @@ -1181,25 +1080,16 @@ dependencies = [ ] [[package]] -name = "mio" -version = "0.7.14" +name = "multipart" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" dependencies = [ - "libc", "log", - "miow", - "ntapi", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "mime", + "mime_guess", + "rand", + "tempfile", ] [[package]] @@ -1225,15 +1115,6 @@ dependencies = [ "void", ] -[[package]] -name = "ntapi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" -dependencies = [ - "winapi", -] - [[package]] name = "num" version = "0.3.1" @@ -1310,16 +1191,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" version = "0.27.1" @@ -1616,44 +1487,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "reqwest" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c4e0a76dc12a116108933f6301b95e83634e0c47b0afbed6abbaa0601e99258" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper", - "hyper-rustls", - "ipnet", - "js-sys", - "lazy_static", - "log", - "mime", - "mime_guess", - "percent-encoding", - "pin-project-lite", - "rustls", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-rustls", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots", - "winreg", -] - [[package]] name = "rfc2047-decoder" version = "0.1.2" @@ -1708,15 +1541,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustls-pemfile" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" -dependencies = [ - "base64", -] - [[package]] name = "ryu" version = "1.0.9" @@ -1846,7 +1670,7 @@ version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142" dependencies = [ - "itoa 1.0.1", + "itoa", "ryu", "serde", ] @@ -1862,18 +1686,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" -dependencies = [ - "form_urlencoded", - "itoa 0.4.8", - "ryu", - "serde", -] - [[package]] name = "sha2" version = "0.9.8" @@ -2077,46 +1889,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" -[[package]] -name = "tokio" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" -dependencies = [ - "bytes", - "libc", - "memchr", - "mio", - "num_cpus", - "pin-project-lite", - "winapi", -] - -[[package]] -name = "tokio-rustls" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" -dependencies = [ - "rustls", - "tokio", - "webpki", -] - -[[package]] -name = "tokio-util" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - [[package]] name = "toml" version = "0.5.8" @@ -2126,38 +1898,6 @@ dependencies = [ "serde", ] -[[package]] -name = "tower-service" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" - -[[package]] -name = "tracing" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" -dependencies = [ - "cfg-if 1.0.0", - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "try-lock" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" - [[package]] name = "typenum" version = "1.15.0" @@ -2227,6 +1967,23 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9399fa2f927a3d327187cbd201480cee55bee6ac5d3c77dd27f0c6814cff16d5" +dependencies = [ + "base64", + "chunked_transfer", + "flate2", + "log", + "once_cell", + "rustls", + "url", + "webpki", + "webpki-roots", +] + [[package]] name = "url" version = "2.2.2" @@ -2277,16 +2034,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "want" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" -dependencies = [ - "log", - "try-lock", -] - [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" @@ -2318,18 +2065,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" -dependencies = [ - "cfg-if 1.0.0", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.78" @@ -2428,15 +2163,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "winreg" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" -dependencies = [ - "winapi", -] - [[package]] name = "xattr" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index 8716a9703..9c6975373 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,8 +32,6 @@ keyring = { version = "1.0.0", optional = true } platform-info = "0.2.0" pretty_env_logger = { version = "0.4.0", optional = true } regex = "1.4.5" -reqwest = { version = "0.11.2", optional = true, default-features = false, features = ["blocking", "multipart"] } -rpassword = { version = "5.0.1", optional = true } serde = { version = "1.0.131", features = ["derive"] } serde_json = "1.0.70" sha2 = "0.10.0" @@ -43,8 +41,7 @@ tempfile = "3.2.0" toml = "0.5.8" zip = "0.5.5" thiserror = "1.0.24" -dirs = { version = "4.0.0", optional = true } -configparser = { version = "3.0.0", optional = true } +dirs = "4.0.0" fs-err = "2.5.0" fat-macho = { version = "0.4.4", default-features = false } once_cell = "1.7.2" @@ -62,18 +59,20 @@ clap = { version = "3.0.0", features = ["derive", "env", "wrap_help"] } clap_complete = "3.0.0" clap_complete_fig = "3.0.0" semver = "1.0.4" +# upload +configparser = { version = "3.0.0", optional = true } +multipart = { version = "0.18.0", features = ["client"], default-features = false, optional = true } +rpassword = { version = "5.0.1", optional = true } +ureq = { version = "2.3.1", optional = true } [dev-dependencies] indoc = "1.0.3" [features] -default = ["log", "upload", "rustls", "human-panic"] -upload = ["reqwest", "rpassword", "configparser", "dirs"] +default = ["log", "upload", "human-panic"] +upload = ["ureq", "multipart", "rpassword", "configparser"] password-storage = ["upload", "keyring"] log = ["pretty_env_logger"] -# We use rustls for manylinux compliance and because unlike both dynamic and -# static openssl variants it can be built and used inside a cent os 5 container -rustls = ["reqwest/rustls-tls"] # Internal feature to speed up the tests significantly faster-tests = [] diff --git a/src/auditwheel/audit.rs b/src/auditwheel/audit.rs index cfa16ce9e..b2abcd9b6 100644 --- a/src/auditwheel/audit.rs +++ b/src/auditwheel/audit.rs @@ -1,15 +1,16 @@ use super::musllinux::{find_musl_libc, get_musl_version}; use super::policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES}; -use crate::auditwheel::PlatformTag; +use crate::auditwheel::{find_external_libs, PlatformTag}; use crate::target::Target; -use anyhow::Result; +use anyhow::{bail, Context, Result}; use fs_err::File; use goblin::elf::{sym::STT_FUNC, Elf}; +use lddtree::Library; use regex::Regex; use std::collections::{HashMap, HashSet}; use std::io; use std::io::Read; -use std::path::Path; +use std::path::{Path, PathBuf}; use thiserror::Error; /// Error raised during auditing an elf file for manylinux/musllinux compatibility @@ -354,3 +355,127 @@ pub fn auditwheel_rs( }?; Ok((policy, should_repair)) } + +/// Get sysroot path from target C compiler +/// +/// Currently only gcc is supported, clang doesn't have a `--print-sysroot` option +pub fn get_sysroot_path(target: &Target) -> Result { + use crate::target::get_host_target; + use std::process::{Command, Stdio}; + + if let Some(sysroot) = std::env::var_os("TARGET_SYSROOT") { + return Ok(PathBuf::from(sysroot)); + } + + let host_triple = get_host_target()?; + let target_triple = target.target_triple(); + if host_triple != target_triple { + let mut build = cc::Build::new(); + build + // Suppress cargo metadata for example env vars printing + .cargo_metadata(false) + // opt_level, host and target are required + .opt_level(0) + .host(&host_triple) + .target(target_triple); + let compiler = build + .try_get_compiler() + .with_context(|| format!("Failed to get compiler for {}", target_triple))?; + // Only GNU like compilers support `--print-sysroot` + if !compiler.is_like_gnu() { + return Ok(PathBuf::from("/")); + } + let path = compiler.path(); + let out = Command::new(path) + .arg("--print-sysroot") + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .output() + .with_context(|| format!("Failed to run `{} --print-sysroot`", path.display()))?; + if out.status.success() { + let sysroot = String::from_utf8(out.stdout) + .context("Failed to read the sysroot path")? + .trim() + .to_owned(); + return Ok(PathBuf::from(sysroot)); + } else { + bail!( + "Failed to get the sysroot path: {}", + String::from_utf8(out.stderr)? + ); + } + } + Ok(PathBuf::from("/")) +} + +/// For the given compilation result, return the manylinux platform and the external libs +/// we need to add to repair it +pub fn get_policy_and_libs( + artifact: &Path, + platform_tag: Option, + target: &Target, +) -> Result<(Policy, Vec)> { + let (policy, should_repair) = + auditwheel_rs(artifact, target, platform_tag).with_context(|| { + if let Some(platform_tag) = platform_tag { + format!("Error ensuring {} compliance", platform_tag) + } else { + "Error checking for manylinux/musllinux compliance".to_string() + } + })?; + let external_libs = if should_repair { + let sysroot = get_sysroot_path(target).unwrap_or_else(|_| PathBuf::from("/")); + find_external_libs(&artifact, &policy, sysroot).with_context(|| { + if let Some(platform_tag) = platform_tag { + format!("Error repairing wheel for {} compliance", platform_tag) + } else { + "Error repairing wheel for manylinux/musllinux compliance".to_string() + } + })? + } else { + Vec::new() + }; + Ok((policy, external_libs)) +} + +pub fn relpath(to: &Path, from: &Path) -> PathBuf { + let mut suffix_pos = 0; + for (f, t) in from.components().zip(to.components()) { + if f == t { + suffix_pos += 1; + } else { + break; + } + } + let mut result = PathBuf::new(); + from.components() + .skip(suffix_pos) + .map(|_| result.push("..")) + .last(); + to.components() + .skip(suffix_pos) + .map(|x| result.push(x.as_os_str())) + .last(); + result +} + +#[cfg(test)] +mod test { + use crate::auditwheel::audit::relpath; + use std::path::Path; + + #[test] + fn test_relpath() { + let cases = [ + ("", "", ""), + ("/", "/usr", ".."), + ("/", "/usr/lib", "../.."), + ]; + for (from, to, expected) in cases { + let from = Path::new(from); + let to = Path::new(to); + let result = relpath(from, to); + assert_eq!(result, Path::new(expected)); + } + } +} diff --git a/src/auditwheel/mod.rs b/src/auditwheel/mod.rs index 22e298ea6..c9faf392e 100644 --- a/src/auditwheel/mod.rs +++ b/src/auditwheel/mod.rs @@ -8,4 +8,4 @@ mod repair; pub use audit::*; pub use platform_tag::PlatformTag; pub use policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES}; -pub use repair::{find_external_libs, hash_file}; +pub use repair::find_external_libs; diff --git a/src/auditwheel/policy.rs b/src/auditwheel/policy.rs index 6aa8f8cc6..ff2212b0a 100644 --- a/src/auditwheel/policy.rs +++ b/src/auditwheel/policy.rs @@ -74,16 +74,6 @@ impl PartialOrd for Policy { } impl Policy { - /// Get highest priority policy than self - pub fn higher_priority_policies(&self) -> impl Iterator { - let policies = if self.name.starts_with("musllinux") { - &MUSLLINUX_POLICIES - } else { - &MANYLINUX_POLICIES - }; - policies.iter().filter(move |p| p.priority > self.priority) - } - /// Get platform tag from this policy pub fn platform_tag(&self) -> PlatformTag { self.name.parse().expect("unknown platform tag") diff --git a/src/auditwheel/repair.rs b/src/auditwheel/repair.rs index 1fce01da3..7374744f0 100644 --- a/src/auditwheel/repair.rs +++ b/src/auditwheel/repair.rs @@ -1,10 +1,7 @@ use super::audit::AuditWheelError; use crate::auditwheel::Policy; use anyhow::Result; -use fs_err as fs; use lddtree::DependencyAnalyzer; -use sha2::{Digest, Sha256}; -use std::io; use std::path::{Path, PathBuf}; pub fn find_external_libs( @@ -30,12 +27,3 @@ pub fn find_external_libs( } Ok(ext_libs) } - -/// Calculate the sha256 of a file -pub fn hash_file(path: impl AsRef) -> Result { - let mut file = fs::File::open(path.as_ref()).map_err(AuditWheelError::IoError)?; - let mut hasher = Sha256::new(); - io::copy(&mut file, &mut hasher).map_err(AuditWheelError::IoError)?; - let hex = format!("{:x}", hasher.finalize()); - Ok(hex) -} diff --git a/src/build_context.rs b/src/build_context.rs index 9829d2e61..aaa4f8df3 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1,6 +1,5 @@ -use crate::auditwheel::{ - auditwheel_rs, find_external_libs, hash_file, patchelf, PlatformTag, Policy, -}; +use crate::auditwheel::{get_policy_and_libs, patchelf, relpath}; +use crate::auditwheel::{PlatformTag, Policy}; use crate::compile::warn_missing_py_init; use crate::module_writer::{ write_bin, write_bindings_module, write_cffi_module, write_python_part, WheelWriter, @@ -12,10 +11,11 @@ use anyhow::{anyhow, bail, Context, Result}; use cargo_metadata::Metadata; use fs_err as fs; use lddtree::Library; +use sha2::{Digest, Sha256}; use std::borrow::Cow; use std::collections::HashMap; +use std::io; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; /// The way the rust code is used in the wheel #[derive(Clone, Debug, PartialEq, Eq)] @@ -258,7 +258,7 @@ impl BuildContext { artifact: &Path, platform_tag: Option, ) -> Result<(Policy, Vec)> { - if self.skip_auditwheel { + if self.skip_auditwheel || self.editable { return Ok((Policy::default(), Vec::new())); } @@ -266,27 +266,7 @@ impl BuildContext { .map(|x| &x.target) .unwrap_or(&self.target); - let (policy, should_repair) = - auditwheel_rs(artifact, target, platform_tag).with_context(|| { - if let Some(platform_tag) = platform_tag { - format!("Error ensuring {} compliance", platform_tag) - } else { - "Error checking for manylinux/musllinux compliance".to_string() - } - })?; - let external_libs = if should_repair && !self.editable { - let sysroot = get_sysroot_path(&self.target).unwrap_or_else(|_| PathBuf::from("/")); - find_external_libs(&artifact, &policy, sysroot).with_context(|| { - if let Some(platform_tag) = platform_tag { - format!("Error repairing wheel for {} compliance", platform_tag) - } else { - "Error repairing wheel for manylinux/musllinux compliance".to_string() - } - })? - } else { - Vec::new() - }; - Ok((policy, external_libs)) + get_policy_and_libs(artifact, platform_tag, target) } fn add_external_libs( @@ -332,12 +312,11 @@ impl BuildContext { if !lib.rpath.is_empty() || !lib.runpath.is_empty() { patchelf::set_rpath(&dest_path, &libs_dir)?; } + patchelf::replace_needed(artifact, &lib.name, &new_soname)?; soname_map.insert( lib.name.clone(), (new_soname.clone(), dest_path.clone(), lib.needed.clone()), ); - - patchelf::replace_needed(artifact, &lib.name, &new_soname)?; } // we grafted in a bunch of libraries and modified their sonames, but @@ -683,95 +662,11 @@ impl BuildContext { } } -fn relpath(to: &Path, from: &Path) -> PathBuf { - let mut suffix_pos = 0; - for (f, t) in from.components().zip(to.components()) { - if f == t { - suffix_pos += 1; - } else { - break; - } - } - let mut result = PathBuf::new(); - from.components() - .skip(suffix_pos) - .map(|_| result.push("..")) - .last(); - to.components() - .skip(suffix_pos) - .map(|x| result.push(x.as_os_str())) - .last(); - result -} - -/// Get sysroot path from target C compiler -/// -/// Currently only gcc is supported, clang doesn't have a `--print-sysroot` option -fn get_sysroot_path(target: &Target) -> Result { - use crate::target::get_host_target; - - if let Some(sysroot) = std::env::var_os("TARGET_SYSROOT") { - return Ok(PathBuf::from(sysroot)); - } - - let host_triple = get_host_target()?; - let target_triple = target.target_triple(); - if host_triple != target_triple { - let mut build = cc::Build::new(); - build - // Suppress cargo metadata for example env vars printing - .cargo_metadata(false) - // opt_level, host and target are required - .opt_level(0) - .host(&host_triple) - .target(target_triple); - let compiler = build - .try_get_compiler() - .with_context(|| format!("Failed to get compiler for {}", target_triple))?; - // Only GNU like compilers support `--print-sysroot` - if !compiler.is_like_gnu() { - return Ok(PathBuf::from("/")); - } - let path = compiler.path(); - let out = Command::new(path) - .arg("--print-sysroot") - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .output() - .with_context(|| format!("Failed to run `{} --print-sysroot`", path.display()))?; - if out.status.success() { - let sysroot = String::from_utf8(out.stdout) - .context("Failed to read the sysroot path")? - .trim() - .to_owned(); - return Ok(PathBuf::from(sysroot)); - } else { - bail!( - "Failed to get the sysroot path: {}", - String::from_utf8(out.stderr)? - ); - } - } - Ok(PathBuf::from("/")) -} - -#[cfg(test)] -mod test { - use super::relpath; - use std::path::Path; - - #[test] - fn test_relpath() { - let cases = [ - ("", "", ""), - ("/", "/usr", ".."), - ("/", "/usr/lib", "../.."), - ]; - for (from, to, expected) in cases { - let from = Path::new(from); - let to = Path::new(to); - let result = relpath(from, to); - assert_eq!(result, Path::new(expected)); - } - } +/// Calculate the sha256 of a file +pub fn hash_file(path: impl AsRef) -> Result { + let mut file = fs::File::open(path.as_ref())?; + let mut hasher = Sha256::new(); + io::copy(&mut file, &mut hasher)?; + let hex = format!("{:x}", hasher.finalize()); + Ok(hex) } diff --git a/src/cargo_toml.rs b/src/cargo_toml.rs index ad2b45d11..4c1995a08 100644 --- a/src/cargo_toml.rs +++ b/src/cargo_toml.rs @@ -156,7 +156,7 @@ mod test { "# ); - let cargo_toml: CargoToml = toml::from_str(&cargo_toml).unwrap(); + let cargo_toml: CargoToml = toml::from_str(cargo_toml).unwrap(); let mut scripts = HashMap::new(); scripts.insert("ph".to_string(), "maturin:print_hello".to_string()); @@ -195,7 +195,7 @@ mod test { "# ); - let cargo_toml: CargoToml = toml::from_str(&cargo_toml).unwrap(); + let cargo_toml: CargoToml = toml::from_str(cargo_toml).unwrap(); let classifiers = vec!["Programming Language :: Python".to_string()]; @@ -226,7 +226,7 @@ mod test { "# ); - let cargo_toml: Result = toml::from_str(&cargo_toml); + let cargo_toml: Result = toml::from_str(cargo_toml); assert!(cargo_toml.is_ok()); } } diff --git a/src/lib.rs b/src/lib.rs index 7bbd5326f..c7783cc50 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,7 +23,6 @@ #![deny(missing_docs)] -pub use crate::auditwheel::{auditwheel_rs, AuditWheelError}; pub use crate::build_context::{BridgeModel, BuildContext, BuiltWheelMetadata}; pub use crate::build_options::BuildOptions; pub use crate::cargo_toml::CargoToml; @@ -37,13 +36,10 @@ pub use crate::new_project::{init_project, new_project, GenerateProjectOptions}; pub use crate::pyproject_toml::PyProjectToml; pub use crate::python_interpreter::PythonInterpreter; pub use crate::target::Target; +#[cfg(feature = "upload")] +pub use crate::upload::{upload, upload_ui, PublishOpt, Registry, UploadError}; pub use crate::zig::Zig; pub use auditwheel::PlatformTag; -#[cfg(feature = "upload")] -pub use { - crate::registry::Registry, - crate::upload::{upload, upload_ui, PublishOpt, UploadError}, -}; mod auditwheel; mod build_context; @@ -57,8 +53,6 @@ mod module_writer; mod new_project; mod pyproject_toml; mod python_interpreter; -#[cfg(feature = "upload")] -mod registry; mod source_distribution; mod target; #[cfg(feature = "upload")] diff --git a/src/metadata.rs b/src/metadata.rs index 8eca6fb5b..95eac2e86 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -717,7 +717,7 @@ mod test { "# ); - let cargo_toml_struct: CargoToml = toml::from_str(&cargo_toml).unwrap(); + let cargo_toml_struct: CargoToml = toml::from_str(cargo_toml).unwrap(); let metadata = Metadata21::from_cargo_toml(&cargo_toml_struct, "/not/exist/manifest/path").unwrap(); let actual = metadata.to_file_contents(); diff --git a/src/registry.rs b/src/registry.rs deleted file mode 100644 index 6eee1de8a..000000000 --- a/src/registry.rs +++ /dev/null @@ -1,24 +0,0 @@ -use reqwest::Url; - -/// A pip registry such as pypi or testpypi with associated credentials, used -/// for uploading wheels -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct Registry { - /// The username - pub username: String, - /// The password - pub password: String, - /// The url endpoint for legacy uploading - pub url: Url, -} - -impl Registry { - /// Creates a new registry - pub fn new(username: String, password: String, url: Url) -> Registry { - Registry { - username, - password, - url, - } - } -} diff --git a/src/target.rs b/src/target.rs index 7a640d429..bb88997e8 100644 --- a/src/target.rs +++ b/src/target.rs @@ -547,31 +547,31 @@ mod test { fn test_macosx_deployment_target() { assert_eq!( macosx_deployment_target(None, false).unwrap(), - (((10, 7), (11, 0))) + ((10, 7), (11, 0)) ); assert_eq!( macosx_deployment_target(None, true).unwrap(), - (((10, 9), (11, 0))) + ((10, 9), (11, 0)) ); assert_eq!( macosx_deployment_target(Some("10.6"), false).unwrap(), - (((10, 7), (11, 0))) + ((10, 7), (11, 0)) ); assert_eq!( macosx_deployment_target(Some("10.6"), true).unwrap(), - (((10, 9), (11, 0))) + ((10, 9), (11, 0)) ); assert_eq!( macosx_deployment_target(Some("10.9"), false).unwrap(), - (((10, 9), (11, 0))) + ((10, 9), (11, 0)) ); assert_eq!( macosx_deployment_target(Some("11.0.0"), false).unwrap(), - (((11, 0), (11, 0))) + ((11, 0), (11, 0)) ); assert_eq!( macosx_deployment_target(Some("11.1"), false).unwrap(), - (((11, 1), (11, 1))) + ((11, 1), (11, 1)) ); } } diff --git a/src/upload.rs b/src/upload.rs index 3fe66931a..3a44cf025 100644 --- a/src/upload.rs +++ b/src/upload.rs @@ -1,16 +1,14 @@ //! The uploading logic was mostly reverse engineered; I wrote it down as //! documentation at https://warehouse.readthedocs.io/api-reference/legacy/#upload-api -use crate::Registry; +use crate::build_context::hash_file; use anyhow::{bail, Context, Result}; use bytesize::ByteSize; use configparser::ini::Ini; use fs_err as fs; use fs_err::File; +use multipart::client::lazy::Multipart; use regex::Regex; -use reqwest::Url; -use reqwest::{self, blocking::multipart::Form, blocking::Client, StatusCode}; -use sha2::{Digest, Sha256}; use std::env; use std::io; use std::path::{Path, PathBuf}; @@ -47,9 +45,9 @@ pub struct PublishOpt { #[derive(Error, Debug)] #[error("Uploading to the registry failed")] pub enum UploadError { - /// Any reqwest error + /// Any ureq error #[error("Http error")] - RewqestError(#[source] reqwest::Error), + UreqError(#[source] ureq::Error), /// The registry returned a "403 Forbidden" #[error("Username or password are incorrect")] AuthenticationError, @@ -73,9 +71,32 @@ impl From for UploadError { } } -impl From for UploadError { - fn from(error: reqwest::Error) -> Self { - UploadError::RewqestError(error) +impl From for UploadError { + fn from(error: ureq::Error) -> Self { + UploadError::UreqError(error) + } +} + +/// A pip registry such as pypi or testpypi with associated credentials, used +/// for uploading wheels +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Registry { + /// The username + pub username: String, + /// The password + pub password: String, + /// The url endpoint for legacy uploading + pub url: String, +} + +impl Registry { + /// Creates a new registry + pub fn new(username: String, password: String, url: String) -> Registry { + Registry { + username, + password, + url, + } } } @@ -187,7 +208,7 @@ fn complete_registry(opt: &PublishOpt) -> Result { (None, opt.registry.clone()) }; let (username, password) = resolve_pypi_cred(opt, &pypirc, register_name); - let registry = Registry::new(username, password, Url::parse(®istry_url)?); + let registry = Registry::new(username, password, registry_url); Ok(registry) } @@ -203,10 +224,7 @@ fn canonicalize_name(name: &str) -> String { /// Uploads a single wheel to the registry pub fn upload(registry: &Registry, wheel_path: &Path) -> Result<(), UploadError> { - let mut wheel = File::open(&wheel_path)?; - let mut hasher = Sha256::new(); - io::copy(&mut wheel, &mut hasher)?; - let hash_hex = format!("{:x}", hasher.finalize()); + let hash_hex = hash_file(&wheel_path)?; let dist = python_pkginfo::Distribution::new(wheel_path) .map_err(|err| UploadError::PkgInfoError(wheel_path.to_owned(), err))?; @@ -267,57 +285,72 @@ pub fn upload(registry: &Registry, wheel_path: &Path) -> Result<(), UploadError> add_vec("requires_external", &metadata.requires_external); add_vec("project_urls", &metadata.project_urls); - let mut form = Form::new(); + let wheel = File::open(&wheel_path)?; + let wheel_name = wheel_path + .file_name() + .expect("Wheel path has a file name") + .to_string_lossy(); + + let mut form = Multipart::new(); for (key, value) in api_metadata { - form = form.text(key, value); + form.add_text(key, value); } - form = form.file("content", &wheel_path)?; - - let client = Client::new(); - let response = client - .post(registry.url.clone()) - .header( - reqwest::header::USER_AGENT, - format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")), + form.add_stream("content", &wheel, Some(wheel_name), None); + let multipart_data = form.prepare().map_err(|e| e.error)?; + + let encoded = base64::encode(&format!("{}:{}", registry.username, registry.password)); + let response = ureq::post(registry.url.as_str()) + .set( + "Content-Type", + &format!( + "multipart/form-data; boundary={}", + multipart_data.boundary() + ), ) - .multipart(form) - .basic_auth(registry.username.clone(), Some(registry.password.clone())) - .send()?; - - let status = response.status(); - if status.is_success() { - return Ok(()); - } - let err_text = response.text().unwrap_or_else(|e| { - format!( - "The registry should return some text, even in case of an error, but didn't ({})", - e + .set( + "User-Agent", + &format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")), ) - }); - // Detect FileExistsError the way twine does - // https://github.com/pypa/twine/blob/87846e5777b380d4704704a69e1f9a7a1231451c/twine/commands/upload.py#L30 - if status == StatusCode::FORBIDDEN { - if err_text.contains("overwrite artifact") { - // Artifactory (https://jfrog.com/artifactory/) - Err(UploadError::FileExistsError(err_text)) - } else { - Err(UploadError::AuthenticationError) - } - } else { - let status_string = status.to_string(); - if status == StatusCode::CONFLICT // pypiserver (https://pypi.org/project/pypiserver) + .set("Authorization", &format!("Basic {}", encoded)) + .send(multipart_data); + + match response { + Ok(_) => Ok(()), + Err(ureq::Error::Status(status, response)) => { + let err_text = response.into_string().unwrap_or_else(|e| { + format!( + "The registry should return some text, \ + even in case of an error, but didn't ({})", + e + ) + }); + // Detect FileExistsError the way twine does + // https://github.com/pypa/twine/blob/87846e5777b380d4704704a69e1f9a7a1231451c/twine/commands/upload.py#L30 + if status == 403 { + if err_text.contains("overwrite artifact") { + // Artifactory (https://jfrog.com/artifactory/) + Err(UploadError::FileExistsError(err_text)) + } else { + Err(UploadError::AuthenticationError) + } + } else { + let status_string = status.to_string(); + if status == 409 // conflict, pypiserver (https://pypi.org/project/pypiserver) // PyPI / TestPyPI - || (status == StatusCode::BAD_REQUEST && err_text.contains("already exists")) + || (status == 400 && err_text.contains("already exists")) // Nexus Repository OSS (https://www.sonatype.com/nexus-repository-oss) - || (status == StatusCode::BAD_REQUEST && err_text.contains("updating asset")) + || (status == 400 && err_text.contains("updating asset")) // # Gitlab Enterprise Edition (https://about.gitlab.com) - || (status == StatusCode::BAD_REQUEST && err_text.contains("already been taken")) - { - Err(UploadError::FileExistsError(err_text)) - } else { - Err(UploadError::StatusCodeError(status_string, err_text)) + || (status == 400 && err_text.contains("already been taken")) + { + Err(UploadError::FileExistsError(err_text)) + } else { + Err(UploadError::StatusCodeError(status_string, err_text)) + } + } } + Err(err) => Err(UploadError::UreqError(err)), } } diff --git a/tests/common/editable.rs b/tests/common/editable.rs index 1c9e3df42..603a2d308 100644 --- a/tests/common/editable.rs +++ b/tests/common/editable.rs @@ -30,7 +30,7 @@ pub fn test_editable( let mut cli = vec![ "build", "--interpreter", - &interpreter, + interpreter, "--manifest-path", &package_string, "--compatibility", @@ -84,7 +84,7 @@ pub fn test_editable( ); } - check_installed(&package.as_ref(), &python)?; + check_installed(package.as_ref(), &python)?; } Ok(()) diff --git a/tests/common/integration.rs b/tests/common/integration.rs index 15d8e5490..69dd4f303 100644 --- a/tests/common/integration.rs +++ b/tests/common/integration.rs @@ -148,7 +148,7 @@ pub fn test_integration( ); } - check_installed(&package.as_ref(), &python)?; + check_installed(package.as_ref(), &python)?; } Ok(()) diff --git a/tests/common/other.rs b/tests/common/other.rs index e82c65b10..cf64f2ba2 100644 --- a/tests/common/other.rs +++ b/tests/common/other.rs @@ -113,10 +113,7 @@ pub fn test_source_distribution( unique_name: &str, ) -> Result<()> { let manifest_path = package.as_ref().join("Cargo.toml"); - let sdist_directory = Path::new("test-crates") - .join("wheels") - .join(unique_name) - .to_path_buf(); + let sdist_directory = Path::new("test-crates").join("wheels").join(unique_name); let build_options = BuildOptions { manifest_path,