-
Notifications
You must be signed in to change notification settings - Fork 152
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #645 from eddyb/build-std
Build the stdlib from rust-src sources.
- Loading branch information
Showing
2 changed files
with
152 additions
and
72 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,4 @@ | ||
target | ||
*.pyc | ||
/cache/ | ||
/rust.git/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,11 +1,10 @@ | ||
use anyhow::{anyhow, Context}; | ||
use chrono::{DateTime, Utc}; | ||
use collector::Sha; | ||
use std::ffi::OsStr; | ||
use std::fmt; | ||
use std::fs::{self, File}; | ||
use std::io::{BufReader, Read}; | ||
use std::path::{Path, PathBuf}; | ||
use std::path::PathBuf; | ||
use tar::Archive; | ||
use xz2::bufread::XzDecoder; | ||
|
||
|
@@ -40,10 +39,116 @@ impl Sysroot { | |
}; | ||
|
||
download.get_and_extract(ModuleVariant::Rustc)?; | ||
download.get_and_extract(ModuleVariant::Std)?; | ||
// HACK(eddyb) commented out because we build our own stdlib | ||
// (see `fn build_std` below). | ||
// download.get_and_extract(ModuleVariant::Std)?; | ||
download.get_and_extract(ModuleVariant::Cargo)?; | ||
download.get_and_extract(ModuleVariant::RustSrc)?; | ||
|
||
download.into_sysroot() | ||
let sysroot_dir = download.directory.join(&download.rust_sha); | ||
let sysroot = download.into_sysroot()?; | ||
|
||
// FIXME(eddyb) remove this once we no longer need to | ||
// build our own stdlib (see `fn build_std` below). | ||
sysroot.build_std(sysroot_dir)?; | ||
|
||
Ok(sysroot) | ||
} | ||
|
||
/// Build `std`+`test`+`proc_macro` in a similar way to Cargo's `-Zbuild-std` | ||
/// feature, but only once, and move the resulting libraries into the sysroot. | ||
/// | ||
/// We only need this until https://github.com/rust-lang/cargo/pull/8073 | ||
/// reaches beta, because then `rust-lang/rust` builds will have that | ||
/// treatment. For now, we only have access to that Cargo change here, | ||
/// using the newly built Cargo. | ||
/// | ||
/// For more background on why we need this, see this comment: | ||
/// https://github.com/rust-lang/rust/issues/69060#issuecomment-604928032 | ||
/// (in short, Cargo used to include `rustc -vV` output, which contains | ||
/// the commit hash, into `-Cmetadata`, producing different `std`s, | ||
/// and making the perf runs incomparable, up to several % of difference). | ||
fn build_std(&self, sysroot_dir: PathBuf) -> anyhow::Result<()> { | ||
// Make sure everything below gets absolute directories. | ||
let sysroot_dir = sysroot_dir.canonicalize()?; | ||
|
||
let sysroot_rustlib_dir = sysroot_dir.join("lib/rustlib"); | ||
let rust_src_dir = sysroot_rustlib_dir.join("src/rust"); | ||
|
||
// HACK(eddyb) add a top-level `Cargo.toml` that has the necessary | ||
// `patch.crates-io` entries for `rustc-std-workspace-{core,alloc,std}`. | ||
// (maybe `rust-src` should include such a `Cargo.toml`?) | ||
fs::write( | ||
rust_src_dir.join("Cargo.toml"), | ||
"\ | ||
[workspace] | ||
members = ['src/libtest'] | ||
[patch.crates-io] | ||
# See comments in `tools/rustc-std-workspace-core/README.md` for what's going on | ||
# here | ||
rustc-std-workspace-core = { path = 'src/tools/rustc-std-workspace-core' } | ||
rustc-std-workspace-alloc = { path = 'src/tools/rustc-std-workspace-alloc' } | ||
rustc-std-workspace-std = { path = 'src/tools/rustc-std-workspace-std' } | ||
", | ||
)?; | ||
|
||
// HACK(eddyb) we need `std` to run the build scripts to build `std`. | ||
let vanilla_sysroot_dir = { | ||
let vanilla_download = SysrootDownload { | ||
directory: sysroot_dir.join("vanilla-sysroot"), | ||
rust_sha: self.sha.clone(), | ||
triple: self.triple.clone(), | ||
}; | ||
vanilla_download.get_and_extract(ModuleVariant::Std)?; | ||
vanilla_download.directory.join(vanilla_download.rust_sha) | ||
}; | ||
|
||
let rustflags = format!( | ||
"--sysroot={sysroot} --remap-path-prefix={remap_from}={remap_to}", | ||
sysroot = vanilla_sysroot_dir.display(), | ||
remap_from = rust_src_dir.display(), | ||
remap_to = "/rustc/REDACTED_SHA_HASH/" | ||
); | ||
|
||
// Run Cargo to produce `$local_build_target_dir/release/deps/lib*.rlib`. | ||
let local_build_target_dir = sysroot_dir.join("build-std-target"); | ||
let cargo_status = std::process::Command::new(&self.cargo) | ||
.env("RUSTC", &self.rustc) | ||
.env("RUSTFLAGS", rustflags) | ||
.env("__CARGO_DEFAULT_LIB_METADATA", "rustc-perf-std") | ||
.args(&["build", "--release"]) | ||
.arg("--target-dir") | ||
.arg(&local_build_target_dir) | ||
.args(&["--features", "panic-unwind", "--features", "backtrace"]) | ||
.arg("--manifest-path") | ||
.arg(rust_src_dir.join("src/libtest/Cargo.toml")) | ||
.status()?; | ||
if !cargo_status.success() { | ||
return Err(anyhow!( | ||
"unable to build stdlib for {} triple {}", | ||
self.sha, | ||
self.triple | ||
)); | ||
} | ||
|
||
// Move all of the `rlib` files into the main sysroot. | ||
let sysroot_target_lib_dir = sysroot_rustlib_dir.join(&self.triple).join("lib"); | ||
for entry in fs::read_dir(local_build_target_dir.join("release/deps"))? { | ||
let entry = entry?; | ||
let path = entry.path(); | ||
if let (Some(name), Some(ext)) = (path.file_name(), path.extension()) { | ||
if ext == "rlib" { | ||
fs::rename(&path, sysroot_target_lib_dir.join(name))?; | ||
} | ||
} | ||
} | ||
|
||
// Clean up, to avoid accidental usage of these directories. | ||
fs::remove_dir_all(vanilla_sysroot_dir)?; | ||
fs::remove_dir_all(local_build_target_dir)?; | ||
|
||
Ok(()) | ||
} | ||
} | ||
|
||
|
@@ -66,14 +171,15 @@ struct SysrootDownload { | |
triple: String, | ||
} | ||
|
||
const MODULE_URL: &str = | ||
"https://rust-lang-ci2.s3.amazonaws.com/rustc-builds/@SHA@/@MODULE@-nightly-@[email protected]"; | ||
const BASE_URL: &str = "https://rust-lang-ci2.s3.amazonaws.com/rustc-builds"; | ||
|
||
// FIXME(eddyb) rename to just `Component`. | ||
#[derive(Debug, Copy, Clone, PartialEq, Eq)] | ||
enum ModuleVariant { | ||
Cargo, | ||
Rustc, | ||
Std, | ||
RustSrc, | ||
} | ||
|
||
impl fmt::Display for ModuleVariant { | ||
|
@@ -82,47 +188,45 @@ impl fmt::Display for ModuleVariant { | |
ModuleVariant::Cargo => write!(f, "cargo"), | ||
ModuleVariant::Rustc => write!(f, "rustc"), | ||
ModuleVariant::Std => write!(f, "rust-std"), | ||
ModuleVariant::RustSrc => write!(f, "rust-src"), | ||
} | ||
} | ||
} | ||
|
||
impl ModuleVariant { | ||
fn url(&self, sysroot: &SysrootDownload, triple: &str) -> String { | ||
MODULE_URL | ||
.replace("@MODULE@", &self.to_string()) | ||
.replace("@SHA@", &sysroot.rust_sha) | ||
.replace("@TRIPLE@", triple) | ||
let suffix = if *self == ModuleVariant::RustSrc { | ||
String::new() | ||
} else { | ||
format!("-{}", triple) | ||
}; | ||
format!( | ||
"{base}/{sha}/{module}-nightly{suffix}.tar.xz", | ||
base = BASE_URL, | ||
module = self, | ||
sha = sysroot.rust_sha, | ||
suffix = suffix, | ||
) | ||
} | ||
} | ||
|
||
impl SysrootDownload { | ||
fn into_sysroot(self) -> anyhow::Result<Sysroot> { | ||
let sysroot_bin_dir = self.directory.join(&self.rust_sha).join("bin"); | ||
let sysroot_bin = |name| { | ||
let path = sysroot_bin_dir.join(name); | ||
path.canonicalize().with_context(|| { | ||
format!( | ||
"failed to canonicalize {} path for {}: {:?}", | ||
name, self.rust_sha, path | ||
) | ||
}) | ||
}; | ||
|
||
Ok(Sysroot { | ||
rustc: self | ||
.directory | ||
.join(&self.rust_sha) | ||
.join("rustc/bin/rustc") | ||
.canonicalize() | ||
.with_context(|| { | ||
format!("failed to canonicalize rustc path for {}", self.rust_sha) | ||
})?, | ||
rustdoc: self | ||
.directory | ||
.join(&self.rust_sha) | ||
.join("rustc/bin/rustdoc") | ||
.canonicalize() | ||
.with_context(|| { | ||
format!("failed to canonicalize rustdoc path for {}", self.rust_sha) | ||
})?, | ||
cargo: { | ||
let path = self.directory.join(&self.rust_sha).join("cargo/bin/cargo"); | ||
path.canonicalize().with_context(|| { | ||
format!( | ||
"failed to canonicalize cargo path for {}: {:?}", | ||
self.rust_sha, path | ||
) | ||
})? | ||
}, | ||
rustc: sysroot_bin("rustc")?, | ||
rustdoc: sysroot_bin("rustdoc")?, | ||
cargo: sysroot_bin("cargo")?, | ||
sha: self.rust_sha, | ||
triple: self.triple, | ||
}) | ||
|
@@ -161,19 +265,21 @@ impl SysrootDownload { | |
} | ||
|
||
return Err(anyhow!( | ||
"unable to download sha {} triple {} module {}", | ||
"unable to download sha {} triple {} module {} from {}", | ||
self.rust_sha, | ||
self.triple, | ||
variant | ||
variant, | ||
url | ||
)); | ||
} | ||
|
||
fn extract<T: Read>(&self, variant: ModuleVariant, reader: T) -> anyhow::Result<()> { | ||
let is_std = variant == ModuleVariant::Std; | ||
let mut archive = Archive::new(reader); | ||
let std_prefix = format!("rust-std-{}/lib/rustlib", self.triple); | ||
|
||
let mut to_link = Vec::new(); | ||
let prefix = if variant == ModuleVariant::Std { | ||
format!("rust-std-{}", self.triple) | ||
} else { | ||
variant.to_string() | ||
}; | ||
|
||
let unpack_into = self.directory.join(&self.rust_sha); | ||
|
||
|
@@ -184,21 +290,11 @@ impl SysrootDownload { | |
assert!(components.next().is_some(), "strip container directory"); | ||
let path = components.as_path(); | ||
|
||
let path = if is_std { | ||
if let Ok(path) = path.strip_prefix(&std_prefix) { | ||
if path.extension() == Some(OsStr::new("dylib")) { | ||
to_link.push(path.to_owned()); | ||
continue; | ||
} else { | ||
Path::new("rustc/lib/rustlib").join(path) | ||
} | ||
} else { | ||
continue; | ||
} | ||
let path = if let Ok(path) = path.strip_prefix(&prefix) { | ||
unpack_into.join(path) | ||
} else { | ||
path.into() | ||
continue; | ||
}; | ||
let path = unpack_into.join(path); | ||
fs::create_dir_all(&path.parent().unwrap()).with_context(|| { | ||
format!( | ||
"could not create intermediate directories for {}", | ||
|
@@ -208,24 +304,6 @@ impl SysrootDownload { | |
entry.unpack(path)?; | ||
} | ||
|
||
let link_dst_prefix = unpack_into.join(format!("rustc/lib/rustlib/{}/lib", self.triple)); | ||
let link_src_prefix = format!("{}/lib", self.triple); | ||
for path in to_link { | ||
let src = unpack_into.join("rustc/lib").join( | ||
path.strip_prefix(&link_src_prefix) | ||
.with_context(|| format!("stripping prefix from: {:?}", path))?, | ||
); | ||
let dst = link_dst_prefix.join(&path); | ||
fs::create_dir_all(&dst.parent().unwrap()).with_context(|| { | ||
format!( | ||
"could not create intermediate directories for {}", | ||
dst.display() | ||
) | ||
})?; | ||
log::trace!("linking {} to {}", src.display(), dst.display()); | ||
fs::hard_link(src, dst)?; | ||
} | ||
|
||
Ok(()) | ||
} | ||
} |