Auto merge of #51063 - mixi:musl-bootstrap, r=alexcrichton

Fix building rustc on and for musl hosts.

This fixes all problems I had when trying to compile rustc on a musl-based distribution (with `crt-static = false` in `config.toml`).

This is a fixed version of what ended up being #50105, making it possible to compile rustc on musl targets.

The differences to the old (now merged and subsequently reverted) pull request are:
 - The commit (6d9154a) that caused the regression for which the original commits were reverted in #50709 is left out. This means the corresponding bug #36710 is still not fixed with `+crt-static`.
 - The test for issue 36710 is skipped for musl targets (until the issue is properly fixed).
 - Building cargo-vendor if `crt-static = false` is needed was broken (cargo-vendor links to some shared libraries if they exist on the system and this produces broken binaries with `+crt-static`)

CC @alexcrichton

src/bootstrap/bin/rustc.rs

@@ -268,6 +268,15 @@ fn main() {
         if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") {
             cmd.arg(format!("-Clinker={}", host_linker));
         }
+
+        if let Ok(s) = env::var("RUSTC_HOST_CRT_STATIC") {
+            if s == "true" {
+                cmd.arg("-C").arg("target-feature=+crt-static");
+            }
+            if s == "false" {
+                cmd.arg("-C").arg("target-feature=-crt-static");
+            }
+        }
     }
 
     if env::var_os("RUSTC_PARALLEL_QUERIES").is_some() {

src/bootstrap/bootstrap.py

@@ -489,7 +489,7 @@ def bin_root(self):
         """
         return os.path.join(self.build_dir, self.build, "stage0")
 
-    def get_toml(self, key):
+    def get_toml(self, key, section=None):
         """Returns the value of the given key in config.toml, otherwise returns None
 
         >>> rb = RustBuild()
@@ -501,12 +501,29 @@ def get_toml(self, key):
 
         >>> rb.get_toml("key3") is None
         True
+
+        Optionally also matches the section the key appears in
+
+        >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
+        >>> rb.get_toml('key', 'a')
+        'value1'
+        >>> rb.get_toml('key', 'b')
+        'value2'
+        >>> rb.get_toml('key', 'c') is None
+        True
         """
+
+        cur_section = None
         for line in self.config_toml.splitlines():
+            section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
+            if section_match is not None:
+                cur_section = section_match.group(1)
+
             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
             if match is not None:
                 value = match.group(1)
-                return self.get_string(value) or value.strip()
+                if section is None or section == cur_section:
+                    return self.get_string(value) or value.strip()
         return None
 
     def cargo(self):
@@ -589,7 +606,17 @@ def build_bootstrap(self):
         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
             (os.pathsep + env["LIBRARY_PATH"]) \
             if "LIBRARY_PATH" in env else ""
-        env["RUSTFLAGS"] = "-Cdebuginfo=2"
+        env["RUSTFLAGS"] = "-Cdebuginfo=2 "
+
+        build_section = "target.{}".format(self.build_triple())
+        target_features = []
+        if self.get_toml("crt-static", build_section) == "true":
+            target_features += ["+crt-static"]
+        elif self.get_toml("crt-static", build_section) == "false":
+            target_features += ["-crt-static"]
+        if target_features:
+            env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
+
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
             os.pathsep + env["PATH"]
         if not os.path.isfile(self.cargo()):

src/bootstrap/builder.rs

@@ -698,9 +698,14 @@ impl<'a> Builder<'a> {
         let out_dir = self.stage_out(compiler, mode);
         cargo
             .env("CARGO_TARGET_DIR", out_dir)
-            .arg(cmd)
-            .arg("--target")
-            .arg(target);
+            .arg(cmd);
+
+        if cmd != "install" {
+            cargo.arg("--target")
+                 .arg(target);
+        } else {
+            assert_eq!(target, compiler.host);
+        }
 
         // Set a flag for `check` so that certain build scripts can do less work
         // (e.g. not building/requiring LLVM).
@@ -842,6 +847,10 @@ impl<'a> Builder<'a> {
             cargo.env("RUSTC_CRT_STATIC", x.to_string());
         }
 
+        if let Some(x) = self.crt_static(compiler.host) {
+            cargo.env("RUSTC_HOST_CRT_STATIC", x.to_string());
+        }
+
         // Enable usage of unstable features
         cargo.env("RUSTC_BOOTSTRAP", "1");
         self.add_rust_test_threads(&mut cargo);
@@ -1018,8 +1027,8 @@ impl<'a> Builder<'a> {
         }
 
         if self.config.rust_optimize {
-            // FIXME: cargo bench does not accept `--release`
-            if cmd != "bench" {
+            // FIXME: cargo bench/install do not accept `--release`
+            if cmd != "bench" && cmd != "install" {
                 cargo.arg("--release");
             }
         }

src/bootstrap/dist.rs

@@ -951,13 +951,16 @@ impl Step for PlainSourceTarball {
                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
             }
             if !has_cargo_vendor {
-                let mut cmd = Command::new(&builder.initial_cargo);
-                cmd.arg("install")
-                   .arg("--force")
+                let mut cmd = builder.cargo(
+                    builder.compiler(0, builder.config.build),
+                    Mode::Tool,
+                    builder.config.build,
+                    "install"
+                );
+                cmd.arg("--force")
                    .arg("--debug")
                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
-                   .arg("cargo-vendor")
-                   .env("RUSTC", &builder.initial_rustc);
+                   .arg("cargo-vendor");
                 if let Some(dir) = builder.openssl_install_dir(builder.config.build) {
                     builder.ensure(native::Openssl {
                         target: builder.config.build,

src/librustc_codegen_llvm/back/link.rs

@@ -625,6 +625,11 @@ fn link_natively(sess: &Session,
     if let Some(args) = sess.target.target.options.pre_link_args.get(&flavor) {
         cmd.args(args);
     }
+    if let Some(args) = sess.target.target.options.pre_link_args_crt.get(&flavor) {
+        if sess.crt_static() {
+            cmd.args(args);
+        }
+    }
     if let Some(ref args) = sess.opts.debugging_opts.pre_link_args {
         cmd.args(args);
     }
@@ -639,6 +644,12 @@ fn link_natively(sess: &Session,
         cmd.arg(root.join(obj));
     }
 
+    if crate_type == config::CrateTypeExecutable && sess.crt_static() {
+        for obj in &sess.target.target.options.pre_link_objects_exe_crt {
+            cmd.arg(root.join(obj));
+        }
+    }
+
     if sess.target.target.options.is_like_emscripten {
         cmd.arg("-s");
         cmd.arg(if sess.panic_strategy() == PanicStrategy::Abort {
@@ -660,6 +671,11 @@ fn link_natively(sess: &Session,
     for obj in &sess.target.target.options.post_link_objects {
         cmd.arg(root.join(obj));
     }
+    if sess.crt_static() {
+        for obj in &sess.target.target.options.post_link_objects_crt {
+            cmd.arg(root.join(obj));
+        }
+    }
     if let Some(args) = sess.target.target.options.post_link_args.get(&flavor) {
         cmd.args(args);
     }

src/librustc_target/spec/linux_musl_base.rs

@@ -15,7 +15,8 @@ pub fn opts() -> TargetOptions {
 
     // Make sure that the linker/gcc really don't pull in anything, including
     // default objects, libs, etc.
-    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-nostdlib".to_string());
+    base.pre_link_args_crt.insert(LinkerFlavor::Gcc, Vec::new());
+    base.pre_link_args_crt.get_mut(&LinkerFlavor::Gcc).unwrap().push("-nostdlib".to_string());
 
     // At least when this was tested, the linker would not add the
     // `GNU_EH_FRAME` program header to executables generated, which is required
@@ -55,9 +56,9 @@ pub fn opts() -> TargetOptions {
     //
     // Each target directory for musl has these object files included in it so
     // they'll be included from there.
-    base.pre_link_objects_exe.push("crt1.o".to_string());
-    base.pre_link_objects_exe.push("crti.o".to_string());
-    base.post_link_objects.push("crtn.o".to_string());
+    base.pre_link_objects_exe_crt.push("crt1.o".to_string());
+    base.pre_link_objects_exe_crt.push("crti.o".to_string());
+    base.post_link_objects_crt.push("crtn.o".to_string());
 
     // These targets statically link libc by default
     base.crt_static_default = true;

src/librustc_target/spec/mod.rs

@@ -426,20 +426,22 @@ pub struct TargetOptions {
     /// Linker to invoke
     pub linker: Option<String>,
 
-    /// Linker arguments that are unconditionally passed *before* any
-    /// user-defined libraries.
-    pub pre_link_args: LinkArgs,
+    /// Linker arguments that are passed *before* any user-defined libraries.
+    pub pre_link_args: LinkArgs, // ... unconditionally
+    pub pre_link_args_crt: LinkArgs, // ... when linking with a bundled crt
     /// Objects to link before all others, always found within the
     /// sysroot folder.
-    pub pre_link_objects_exe: Vec<String>, // ... when linking an executable
+    pub pre_link_objects_exe: Vec<String>, // ... when linking an executable, unconditionally
+    pub pre_link_objects_exe_crt: Vec<String>, // ... when linking an executable with a bundled crt
     pub pre_link_objects_dll: Vec<String>, // ... when linking a dylib
     /// Linker arguments that are unconditionally passed after any
     /// user-defined but before post_link_objects.  Standard platform
     /// libraries that should be always be linked to, usually go here.
     pub late_link_args: LinkArgs,
     /// Objects to link after all others, always found within the
     /// sysroot folder.
-    pub post_link_objects: Vec<String>,
+    pub post_link_objects: Vec<String>, // ... unconditionally
+    pub post_link_objects_crt: Vec<String>, // ... when linking with a bundled crt
     /// Linker arguments that are unconditionally passed *after* any
     /// user-defined libraries.
     pub post_link_args: LinkArgs,
@@ -639,6 +641,7 @@ impl Default for TargetOptions {
             is_builtin: false,
             linker: option_env!("CFG_DEFAULT_LINKER").map(|s| s.to_string()),
             pre_link_args: LinkArgs::new(),
+            pre_link_args_crt: LinkArgs::new(),
             post_link_args: LinkArgs::new(),
             asm_args: Vec::new(),
             cpu: "generic".to_string(),
@@ -672,8 +675,10 @@ impl Default for TargetOptions {
             position_independent_executables: false,
             relro_level: RelroLevel::None,
             pre_link_objects_exe: Vec::new(),
+            pre_link_objects_exe_crt: Vec::new(),
             pre_link_objects_dll: Vec::new(),
             post_link_objects: Vec::new(),
+            post_link_objects_crt: Vec::new(),
             late_link_args: LinkArgs::new(),
             link_env: Vec::new(),
             archive_format: "gnu".to_string(),
@@ -892,10 +897,13 @@ impl Target {
         key!(is_builtin, bool);
         key!(linker, optional);
         key!(pre_link_args, link_args);
+        key!(pre_link_args_crt, link_args);
         key!(pre_link_objects_exe, list);
+        key!(pre_link_objects_exe_crt, list);
         key!(pre_link_objects_dll, list);
         key!(late_link_args, link_args);
         key!(post_link_objects, list);
+        key!(post_link_objects_crt, list);
         key!(post_link_args, link_args);
         key!(link_env, env);
         key!(asm_args, list);
@@ -1097,10 +1105,13 @@ impl ToJson for Target {
         target_option_val!(is_builtin);
         target_option_val!(linker);
         target_option_val!(link_args - pre_link_args);
+        target_option_val!(link_args - pre_link_args_crt);
         target_option_val!(pre_link_objects_exe);
+        target_option_val!(pre_link_objects_exe_crt);
         target_option_val!(pre_link_objects_dll);
         target_option_val!(link_args - late_link_args);
         target_option_val!(post_link_objects);
+        target_option_val!(post_link_objects_crt);
         target_option_val!(link_args - post_link_args);
         target_option_val!(env - link_env);
         target_option_val!(asm_args);

src/test/run-make-fulldeps/issue-36710/Makefile

@@ -0,0 +1,21 @@
+-include ../tools.mk
+
+ifeq (musl,$(findstring musl,$(TARGET)))
+all: skip
+else
+all: test
+endif
+
+test: foo
+	$(call RUN,foo)
+
+skip:
+	echo "expected failure"
+
+foo: foo.rs $(call NATIVE_STATICLIB,foo)
+	$(RUSTC) $< -lfoo $(EXTRACXXFLAGS)
+
+$(TMPDIR)/libfoo.o: foo.cpp
+	$(call COMPILE_OBJ_CXX,$@,$<)
+
+.PHONY: all test skip

src/test/run-make-fulldeps/issue-36710/foo.cpp

@@ -0,0 +1,25 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#include <stdint.h>
+
+struct A {
+    A() { v = 1234; }
+    ~A() { v = 1; }
+    uint32_t v;
+};
+
+A a;
+
+extern "C" {
+    uint32_t get() {
+        return a.v;
+    }
+}

src/test/run-make-fulldeps/issue-36710/foo.rs

@@ -0,0 +1,18 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Tests that linking to C++ code with global destructors works.
+
+extern { fn get() -> u32; }
+
+fn main() {
+    let i = unsafe { get() };
+    assert_eq!(i, 1234);
+}

src/test/run-make-fulldeps/tools.mk

@@ -59,12 +59,14 @@ endif
 
 ifdef IS_MSVC
 COMPILE_OBJ = $(CC) -c -Fo:`cygpath -w $(1)` $(2)
+COMPILE_OBJ_CXX = $(CXX) -c -Fo:`cygpath -w $(1)` $(2)
 NATIVE_STATICLIB_FILE = $(1).lib
 NATIVE_STATICLIB = $(TMPDIR)/$(call NATIVE_STATICLIB_FILE,$(1))
 OUT_EXE=-Fe:`cygpath -w $(TMPDIR)/$(call BIN,$(1))` \
 	-Fo:`cygpath -w $(TMPDIR)/$(1).obj`
 else
 COMPILE_OBJ = $(CC) -c -o $(1) $(2)
+COMPILE_OBJ_CXX = $(CXX) -c -o $(1) $(2)
 NATIVE_STATICLIB_FILE = lib$(1).a
 NATIVE_STATICLIB = $(call STATICLIB,$(1))
 OUT_EXE=-o $(TMPDIR)/$(1)

src/tools/compiletest/src/runtest.rs

@@ -2529,7 +2529,7 @@ impl<'test> TestCx<'test> {
                 .env("IS_WINDOWS", "1")
                 .env("MSVC_LIB", format!("'{}' -nologo", lib.display()))
                 .env("CC", format!("'{}' {}", self.config.cc, cflags))
-                .env("CXX", &self.config.cxx);
+                .env("CXX", format!("'{}'", &self.config.cxx));
         } else {
             cmd.env("CC", format!("{} {}", self.config.cc, self.config.cflags))
                 .env("CXX", format!("{} {}", self.config.cxx, self.config.cflags))