From 14638a0b31773e7fa5548f50de600b598382d2dc Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Mon, 21 Dec 2020 02:00:03 +0900 Subject: [PATCH 1/9] Split build script as `naive_build` --- openblas-src/build.rs | 136 ++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 66 deletions(-) diff --git a/openblas-src/build.rs b/openblas-src/build.rs index 96778be..4f8636c 100644 --- a/openblas-src/build.rs +++ b/openblas-src/build.rs @@ -94,72 +94,7 @@ fn main() { "Non-vcpkg builds are not supported on Windows. You must use the 'system' feature." ) } - - let output = PathBuf::from(env::var("OUT_DIR").unwrap().replace(r"\", "/")); - let mut make = Command::new("make"); - make.args(&["libs", "netlib", "shared"]) - .arg(format!("BINARY={}", binary())) - .arg(format!( - "{}_CBLAS=1", - if feature_enabled("cblas") { - "YES" - } else { - "NO" - } - )) - .arg(format!( - "{}_LAPACKE=1", - if feature_enabled("lapacke") { - "YES" - } else { - "NO" - } - )); - match env::var("OPENBLAS_ARGS") { - Ok(args) => { - make.args(args.split_whitespace()); - } - _ => (), - }; - if let Ok(num_jobs) = env::var("NUM_JOBS") { - make.arg(format!("-j{}", num_jobs)); - } - let target = match env::var("OPENBLAS_TARGET") { - Ok(target) => { - make.arg(format!("TARGET={}", target)); - target - } - _ => env::var("TARGET").unwrap(), - }; - env::remove_var("TARGET"); - let source = if feature_enabled("cache") { - PathBuf::from(format!("source_{}", target.to_lowercase())) - } else { - output.join(format!("source_{}", target.to_lowercase())) - }; - - if !source.exists() { - let source_tmp = PathBuf::from(format!("{}_tmp", source.display())); - if source_tmp.exists() { - fs::remove_dir_all(&source_tmp).unwrap(); - } - run(Command::new("cp").arg("-R").arg("source").arg(&source_tmp)); - fs::rename(&source_tmp, &source).unwrap(); - } - for name in &vec!["CC", "FC", "HOSTCC"] { - if let Ok(value) = env::var(format!("OPENBLAS_{}", name)) { - make.arg(format!("{}={}", name, value)); - } - } - run(&mut make.current_dir(&source)); - run(Command::new("make") - .arg("install") - .arg(format!("DESTDIR={}", output.display())) - .current_dir(&source)); - println!( - "cargo:rustc-link-search={}", - output.join("opt/OpenBLAS/lib").display(), - ); + naive_build(); } println!("cargo:rustc-link-lib={}=openblas", link_kind); } @@ -177,3 +112,72 @@ fn run(command: &mut Command) { } } } + +/// openblas-src 0.9.0 compatible `make` runner +fn naive_build() { + let output = PathBuf::from(env::var("OUT_DIR").unwrap().replace(r"\", "/")); + let mut make = Command::new("make"); + make.args(&["libs", "netlib", "shared"]) + .arg(format!("BINARY={}", binary())) + .arg(format!( + "{}_CBLAS=1", + if feature_enabled("cblas") { + "YES" + } else { + "NO" + } + )) + .arg(format!( + "{}_LAPACKE=1", + if feature_enabled("lapacke") { + "YES" + } else { + "NO" + } + )); + match env::var("OPENBLAS_ARGS") { + Ok(args) => { + make.args(args.split_whitespace()); + } + _ => (), + }; + if let Ok(num_jobs) = env::var("NUM_JOBS") { + make.arg(format!("-j{}", num_jobs)); + } + let target = match env::var("OPENBLAS_TARGET") { + Ok(target) => { + make.arg(format!("TARGET={}", target)); + target + } + _ => env::var("TARGET").unwrap(), + }; + env::remove_var("TARGET"); + let source = if feature_enabled("cache") { + PathBuf::from(format!("source_{}", target.to_lowercase())) + } else { + output.join(format!("source_{}", target.to_lowercase())) + }; + + if !source.exists() { + let source_tmp = PathBuf::from(format!("{}_tmp", source.display())); + if source_tmp.exists() { + fs::remove_dir_all(&source_tmp).unwrap(); + } + run(Command::new("cp").arg("-R").arg("source").arg(&source_tmp)); + fs::rename(&source_tmp, &source).unwrap(); + } + for name in &vec!["CC", "FC", "HOSTCC"] { + if let Ok(value) = env::var(format!("OPENBLAS_{}", name)) { + make.arg(format!("{}={}", name, value)); + } + } + run(&mut make.current_dir(&source)); + run(Command::new("make") + .arg("install") + .arg(format!("DESTDIR={}", output.display())) + .current_dir(&source)); + println!( + "cargo:rustc-link-search={}", + output.join("opt/OpenBLAS/lib").display(), + ); +} From 6d73b505b0fa1715f3791b051e6fd77b6a31328f Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Tue, 22 Dec 2020 02:01:48 +0900 Subject: [PATCH 2/9] Comment why openblas-build has introduced --- openblas-src/build.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openblas-src/build.rs b/openblas-src/build.rs index 4f8636c..3b217cd 100644 --- a/openblas-src/build.rs +++ b/openblas-src/build.rs @@ -114,6 +114,10 @@ fn run(command: &mut Command) { } /// openblas-src 0.9.0 compatible `make` runner +/// +/// This cannot detect that OpenBLAS skips LAPACK build due to the absense of Fortran compiler. +/// openblas-build crate can detect it by sneaking OpenBLAS build system, but only works on Linux. +/// fn naive_build() { let output = PathBuf::from(env::var("OUT_DIR").unwrap().replace(r"\", "/")); let mut make = Command::new("make"); From e4ac3c5355aa894c59ae3147f23b10f7fd865a5c Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sat, 26 Dec 2020 18:54:22 +0900 Subject: [PATCH 3/9] Remove TARGET environment variable in `make` process --- openblas-build/src/build.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/openblas-build/src/build.rs b/openblas-build/src/build.rs index e57d9b0..08daa18 100644 --- a/openblas-build/src/build.rs +++ b/openblas-build/src/build.rs @@ -241,11 +241,15 @@ impl Configure { // - This will automatically run in parallel without `-j` flag // - The `make` of OpenBLAS outputs 30k lines, // which will be redirected into `out.log` and `err.log`. + // - cargo sets `TARGET` environment variable as target triple (e.g. x86_64-unknown-linux-gnu) + // while binding build.rs, but `make` read it as CPU target specification. + // match Command::new("make") .current_dir(out_dir) .stdout(unsafe { Stdio::from_raw_fd(out.into_raw_fd()) }) // this works only for unix .stderr(unsafe { Stdio::from_raw_fd(err.into_raw_fd()) }) .args(&self.make_args()) + .env_remove("TARGET") .check_call() { Ok(_) => {} @@ -291,6 +295,14 @@ impl Configure { mod tests { use super::*; + #[ignore] + #[test] + fn build_default() { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test_build/build_default"); + let opt = Configure::default(); + let _detail = opt.build(path).unwrap(); + } + #[ignore] #[test] fn build_no_shared() { From 45c1661e0c1c608e740bdb802c5a71b86a4f9a33 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sat, 26 Dec 2020 21:30:47 +0900 Subject: [PATCH 4/9] Default setting --- openblas-src/Cargo.toml | 3 +++ openblas-src/build.rs | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/openblas-src/Cargo.toml b/openblas-src/Cargo.toml index f5682da..e3097bf 100644 --- a/openblas-src/Cargo.toml +++ b/openblas-src/Cargo.toml @@ -48,3 +48,6 @@ libc = "0.2" [target.'cfg(target_os="windows")'.build-dependencies] vcpkg = "0.2" + +[target.'cfg(target_os="linux")'.build-dependencies.openblas-build] +path = "../openblas-build" diff --git a/openblas-src/build.rs b/openblas-src/build.rs index 3b217cd..61559ba 100644 --- a/openblas-src/build.rs +++ b/openblas-src/build.rs @@ -94,7 +94,11 @@ fn main() { "Non-vcpkg builds are not supported on Windows. You must use the 'system' feature." ) } - naive_build(); + if cfg!(target_os = "linux") { + build(); + } else { + naive_build(); + } } println!("cargo:rustc-link-lib={}=openblas", link_kind); } @@ -113,6 +117,13 @@ fn run(command: &mut Command) { } } +/// Build OpenBLAS using openblas-build crate +fn build() { + let output = PathBuf::from(env::var("OUT_DIR").unwrap()).join("OpenBLAS"); + let cfg = openblas_build::Configure::default(); + cfg.build(&output).unwrap(); +} + /// openblas-src 0.9.0 compatible `make` runner /// /// This cannot detect that OpenBLAS skips LAPACK build due to the absense of Fortran compiler. From ec9ddaaeea53479710690539c2027eef44bbe8f0 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sat, 26 Dec 2020 21:39:57 +0900 Subject: [PATCH 5/9] Handle existing flags --- openblas-src/build.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/openblas-src/build.rs b/openblas-src/build.rs index 61559ba..42ae991 100644 --- a/openblas-src/build.rs +++ b/openblas-src/build.rs @@ -120,7 +120,18 @@ fn run(command: &mut Command) { /// Build OpenBLAS using openblas-build crate fn build() { let output = PathBuf::from(env::var("OUT_DIR").unwrap()).join("OpenBLAS"); - let cfg = openblas_build::Configure::default(); + let mut cfg = openblas_build::Configure::default(); + if !feature_enabled("cblas") { + cfg.no_cblas = true; + } + if !feature_enabled("lapacke") { + cfg.no_lapacke = true; + } + if feature_enabled("static") { + cfg.no_shared = true; + } else { + cfg.no_static = true; + } cfg.build(&output).unwrap(); } From a4fbe9b105e4337c5e52275421ce227443dda21d Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sat, 26 Dec 2020 21:56:15 +0900 Subject: [PATCH 6/9] Switch build script using attribute instead of cfg! --- openblas-src/build.rs | 60 +++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/openblas-src/build.rs b/openblas-src/build.rs index 42ae991..e872092 100644 --- a/openblas-src/build.rs +++ b/openblas-src/build.rs @@ -1,17 +1,9 @@ -use std::{env, fs, path::*, process::Command}; +use std::{env, path::*, process::Command}; fn feature_enabled(feature: &str) -> bool { env::var(format!("CARGO_FEATURE_{}", feature.to_uppercase())).is_ok() } -fn binary() -> &'static str { - if cfg!(target_pointer_width = "32") { - "32" - } else { - "64" - } -} - /// Add path where pacman (on msys2) install OpenBLAS /// /// - `pacman -S mingw-w64-x86_64-openblas` will install @@ -94,32 +86,15 @@ fn main() { "Non-vcpkg builds are not supported on Windows. You must use the 'system' feature." ) } - if cfg!(target_os = "linux") { - build(); - } else { - naive_build(); - } + build(); } println!("cargo:rustc-link-lib={}=openblas", link_kind); } -fn run(command: &mut Command) { - println!("Running: `{:?}`", command); - match command.status() { - Ok(status) => { - if !status.success() { - panic!("Failed: `{:?}` ({})", command, status); - } - } - Err(error) => { - panic!("Failed: `{:?}` ({})", command, error); - } - } -} - /// Build OpenBLAS using openblas-build crate +#[cfg(target_os = "linux")] fn build() { - let output = PathBuf::from(env::var("OUT_DIR").unwrap()).join("OpenBLAS"); + let output = PathBuf::from(env::var("OUT_DIR").unwrap()); let mut cfg = openblas_build::Configure::default(); if !feature_enabled("cblas") { cfg.no_cblas = true; @@ -140,7 +115,10 @@ fn build() { /// This cannot detect that OpenBLAS skips LAPACK build due to the absense of Fortran compiler. /// openblas-build crate can detect it by sneaking OpenBLAS build system, but only works on Linux. /// -fn naive_build() { +#[cfg(target_os = "macos")] +fn build() { + use std::fs; + let output = PathBuf::from(env::var("OUT_DIR").unwrap().replace(r"\", "/")); let mut make = Command::new("make"); make.args(&["libs", "netlib", "shared"]) @@ -206,4 +184,26 @@ fn naive_build() { "cargo:rustc-link-search={}", output.join("opt/OpenBLAS/lib").display(), ); + + fn run(command: &mut Command) { + println!("Running: `{:?}`", command); + match command.status() { + Ok(status) => { + if !status.success() { + panic!("Failed: `{:?}` ({})", command, status); + } + } + Err(error) => { + panic!("Failed: `{:?}` ({})", command, error); + } + } + } + + fn binary() -> &'static str { + if cfg!(target_pointer_width = "32") { + "32" + } else { + "64" + } + } } From 29519bb034841dd6fbe28f5c33d3527298ad03f4 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sat, 26 Dec 2020 23:21:50 +0900 Subject: [PATCH 7/9] Use not(target_os = "linux") --- openblas-src/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openblas-src/build.rs b/openblas-src/build.rs index e872092..4e032d3 100644 --- a/openblas-src/build.rs +++ b/openblas-src/build.rs @@ -115,7 +115,7 @@ fn build() { /// This cannot detect that OpenBLAS skips LAPACK build due to the absense of Fortran compiler. /// openblas-build crate can detect it by sneaking OpenBLAS build system, but only works on Linux. /// -#[cfg(target_os = "macos")] +#[cfg(not(target_os = "linux"))] fn build() { use std::fs; From f885d4aceb52981128bda5c260f9cacc1c6c011f Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sat, 26 Dec 2020 23:23:14 +0900 Subject: [PATCH 8/9] Install gfortran on Linux CI --- .github/workflows/openblas-src.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/openblas-src.yml b/.github/workflows/openblas-src.yml index 8dc4293..ff36280 100644 --- a/.github/workflows/openblas-src.yml +++ b/.github/workflows/openblas-src.yml @@ -82,6 +82,10 @@ jobs: - uses: actions/checkout@v1 with: submodules: 'recursive' + - name: Install GCC-Fortran by apt + run: | + apt update + apt install -y gfortran - name: Install OpenBLAS by apt run: | apt update From 9093fa541c92297fdc45b023cc2574561510202a Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sun, 27 Dec 2020 01:36:23 +0900 Subject: [PATCH 9/9] Add link search path --- openblas-src/build.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/openblas-src/build.rs b/openblas-src/build.rs index 4e032d3..db4e7be 100644 --- a/openblas-src/build.rs +++ b/openblas-src/build.rs @@ -108,6 +108,7 @@ fn build() { cfg.no_static = true; } cfg.build(&output).unwrap(); + println!("cargo:rustc-link-search={}", output.display()); } /// openblas-src 0.9.0 compatible `make` runner