diff --git a/src/libprofiler_builtins/build.rs b/src/libprofiler_builtins/build.rs index dd88dd933f691..8d6c7d68dfe23 100644 --- a/src/libprofiler_builtins/build.rs +++ b/src/libprofiler_builtins/build.rs @@ -27,6 +27,7 @@ fn main() { "InstrProfilingFile.c", "InstrProfilingMerge.c", "InstrProfilingMergeFile.c", + "InstrProfilingNameVar.c", "InstrProfilingPlatformDarwin.c", "InstrProfilingPlatformLinux.c", "InstrProfilingPlatformOther.c", @@ -42,6 +43,8 @@ fn main() { cfg.define("strdup", Some("_strdup")); cfg.define("open", Some("_open")); cfg.define("fdopen", Some("_fdopen")); + cfg.define("getpid", Some("_getpid")); + cfg.define("fileno", Some("_fileno")); } else { // Turn off various features of gcc and such, mostly copying // compiler-rt's build system already @@ -50,6 +53,7 @@ fn main() { cfg.flag("-fomit-frame-pointer"); cfg.flag("-ffreestanding"); cfg.define("VISIBILITY_HIDDEN", None); + cfg.define("COMPILER_RT_HAS_UNAME", Some("1")); } for src in profile_sources { diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs index 31795d0ff4c90..9a5e929d85f2d 100644 --- a/src/librustc/session/config.rs +++ b/src/librustc/session/config.rs @@ -1249,6 +1249,14 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "extra arguments to prepend to the linker invocation (space separated)"), profile: bool = (false, parse_bool, [TRACKED], "insert profiling code"), + pgo_gen: Option = (None, parse_opt_string, [TRACKED], + "Generate PGO profile data, to a given file, or to the default \ + location if it's empty."), + pgo_use: String = (String::new(), parse_string, [TRACKED], + "Use PGO profile data from the given profile file."), + disable_instrumentation_preinliner: bool = + (false, parse_bool, [TRACKED], "Disable the instrumentation pre-inliner, \ + useful for profiling / PGO."), relro_level: Option = (None, parse_relro_level, [TRACKED], "choose which RELRO level to use"), nll: bool = (false, parse_bool, [UNTRACKED], @@ -1773,6 +1781,13 @@ pub fn build_session_options_and_crate_config( ); } + if debugging_opts.pgo_gen.is_some() && !debugging_opts.pgo_use.is_empty() { + early_error( + error_format, + "options `-Z pgo-gen` and `-Z pgo-use` are exclusive", + ); + } + let mut output_types = BTreeMap::new(); if !debugging_opts.parse_only { for list in matches.opt_strs("emit") { @@ -2886,6 +2901,14 @@ mod tests { opts.debugging_opts.tls_model = Some(String::from("tls model")); assert!(reference.dep_tracking_hash() != opts.dep_tracking_hash()); + opts = reference.clone(); + opts.debugging_opts.pgo_gen = Some(String::from("abc")); + assert_ne!(reference.dep_tracking_hash(), opts.dep_tracking_hash()); + + opts = reference.clone(); + opts.debugging_opts.pgo_use = String::from("abc"); + assert_ne!(reference.dep_tracking_hash(), opts.dep_tracking_hash()); + opts = reference.clone(); opts.cg.metadata = vec![String::from("A"), String::from("B")]; assert!(reference.dep_tracking_hash() != opts.dep_tracking_hash()); diff --git a/src/librustc_llvm/diagnostic.rs b/src/librustc_llvm/diagnostic.rs index c5cdf6566921c..e73c570ed8231 100644 --- a/src/librustc_llvm/diagnostic.rs +++ b/src/librustc_llvm/diagnostic.rs @@ -121,6 +121,7 @@ impl InlineAsmDiagnostic { pub enum Diagnostic { Optimization(OptimizationDiagnostic), InlineAsm(InlineAsmDiagnostic), + PGO(DiagnosticInfoRef), /// LLVM has other types that we do not wrap here. UnknownDiagnostic(DiagnosticInfoRef), @@ -160,6 +161,10 @@ impl Diagnostic { Optimization(OptimizationDiagnostic::unpack(OptimizationFailure, di)) } + Dk::PGOProfile => { + PGO(di) + } + _ => UnknownDiagnostic(di), } } diff --git a/src/librustc_llvm/ffi.rs b/src/librustc_llvm/ffi.rs index 403fe4731f118..600cb6491ad2f 100644 --- a/src/librustc_llvm/ffi.rs +++ b/src/librustc_llvm/ffi.rs @@ -322,6 +322,7 @@ pub enum DiagnosticKind { OptimizationRemarkAnalysisAliasing, OptimizationRemarkOther, OptimizationFailure, + PGOProfile, } /// LLVMRustArchiveKind @@ -1646,7 +1647,9 @@ extern "C" { OptLevel: CodeGenOptLevel, MergeFunctions: bool, SLPVectorize: bool, - LoopVectorize: bool); + LoopVectorize: bool, + PGOGenPath: *const c_char, + PGOUsePath: *const c_char); pub fn LLVMRustAddLibraryInfo(PM: PassManagerRef, M: ModuleRef, DisableSimplifyLibCalls: bool); @@ -1741,6 +1744,7 @@ extern "C" { pub fn LLVMRustModuleCost(M: ModuleRef) -> u64; pub fn LLVMRustThinLTOAvailable() -> bool; + pub fn LLVMRustPGOAvailable() -> bool; pub fn LLVMRustWriteThinBitcodeToFile(PMR: PassManagerRef, M: ModuleRef, BC: *const c_char) -> bool; diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index baaf57c890898..802665b6ddbc5 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -784,7 +784,9 @@ impl<'a> CrateLoader<'a> { } fn inject_profiler_runtime(&mut self) { - if self.sess.opts.debugging_opts.profile { + if self.sess.opts.debugging_opts.profile || + self.sess.opts.debugging_opts.pgo_gen.is_some() + { info!("loading profiler"); let symbol = Symbol::intern("profiler_builtins"); diff --git a/src/librustc_trans/attributes.rs b/src/librustc_trans/attributes.rs index df78ccdd2298a..c968b8525a5b1 100644 --- a/src/librustc_trans/attributes.rs +++ b/src/librustc_trans/attributes.rs @@ -92,6 +92,11 @@ pub fn set_probestack(cx: &CodegenCx, llfn: ValueRef) { _ => {} } + // probestack doesn't play nice either with pgo-gen. + if cx.sess().opts.debugging_opts.pgo_gen.is_some() { + return; + } + // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. llvm::AddFunctionAttrStringValue( diff --git a/src/librustc_trans/back/link.rs b/src/librustc_trans/back/link.rs index 542cdc5baad32..75ba83a7c620a 100644 --- a/src/librustc_trans/back/link.rs +++ b/src/librustc_trans/back/link.rs @@ -1085,6 +1085,10 @@ fn link_args(cmd: &mut Linker, cmd.build_static_executable(); } + if sess.opts.debugging_opts.pgo_gen.is_some() { + cmd.pgo_gen(); + } + // FIXME (#2397): At some point we want to rpath our guesses as to // where extern libraries might live, based on the // addl_lib_search_paths diff --git a/src/librustc_trans/back/linker.rs b/src/librustc_trans/back/linker.rs index 53a9dd6a76fcd..a9095a66aaa98 100644 --- a/src/librustc_trans/back/linker.rs +++ b/src/librustc_trans/back/linker.rs @@ -117,6 +117,7 @@ pub trait Linker { fn partial_relro(&mut self); fn no_relro(&mut self); fn optimize(&mut self); + fn pgo_gen(&mut self); fn debuginfo(&mut self); fn no_default_libraries(&mut self); fn build_dylib(&mut self, out_filename: &Path); @@ -280,6 +281,24 @@ impl<'a> Linker for GccLinker<'a> { } } + fn pgo_gen(&mut self) { + if !self.sess.target.target.options.linker_is_gnu { return } + + // If we're doing PGO generation stuff and on a GNU-like linker, use the + // "-u" flag to properly pull in the profiler runtime bits. + // + // This is because LLVM otherwise won't add the needed initialization + // for us on Linux (though the extra flag should be harmless if it + // does). + // + // See https://reviews.llvm.org/D14033 and https://reviews.llvm.org/D14030. + // + // Though it may be worth to try to revert those changes upstream, since + // the overhead of the initialization should be minor. + self.cmd.arg("-u"); + self.cmd.arg("__llvm_profile_runtime"); + } + fn debuginfo(&mut self) { match self.sess.opts.debuginfo { DebugInfoLevel::NoDebugInfo => { @@ -520,6 +539,10 @@ impl<'a> Linker for MsvcLinker<'a> { // Needs more investigation of `/OPT` arguments } + fn pgo_gen(&mut self) { + // Nothing needed here. + } + fn debuginfo(&mut self) { // This will cause the Microsoft linker to generate a PDB file // from the CodeView line tables in the object files. @@ -723,6 +746,10 @@ impl<'a> Linker for EmLinker<'a> { self.cmd.args(&["--memory-init-file", "0"]); } + fn pgo_gen(&mut self) { + // noop, but maybe we need something like the gnu linker? + } + fn debuginfo(&mut self) { // Preserve names or generate source maps depending on debug info self.cmd.arg(match self.sess.opts.debuginfo { @@ -888,6 +915,9 @@ impl Linker for WasmLd { fn optimize(&mut self) { } + fn pgo_gen(&mut self) { + } + fn debuginfo(&mut self) { } diff --git a/src/librustc_trans/back/symbol_export.rs b/src/librustc_trans/back/symbol_export.rs index fd79ae7435ed1..d205e6ca4eda6 100644 --- a/src/librustc_trans/back/symbol_export.rs +++ b/src/librustc_trans/back/symbol_export.rs @@ -223,6 +223,20 @@ fn exported_symbols_provider_local<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, } } + if tcx.sess.opts.debugging_opts.pgo_gen.is_some() { + // These are weak symbols that point to the profile version and the + // profile name, which need to be treated as exported so LTO doesn't nix + // them. + const PROFILER_WEAK_SYMBOLS: [&'static str; 2] = [ + "__llvm_profile_raw_version", + "__llvm_profile_filename", + ]; + for sym in &PROFILER_WEAK_SYMBOLS { + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(sym)); + symbols.push((exported_symbol, SymbolExportLevel::C)); + } + } + if tcx.sess.crate_types.borrow().contains(&config::CrateTypeDylib) { let symbol_name = metadata_symbol_name(tcx); let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(&symbol_name)); diff --git a/src/librustc_trans/back/write.rs b/src/librustc_trans/back/write.rs index 3e7422557e9b6..2cbb88fed05c2 100644 --- a/src/librustc_trans/back/write.rs +++ b/src/librustc_trans/back/write.rs @@ -240,6 +240,9 @@ pub struct ModuleConfig { /// Some(level) to optimize binary size, or None to not affect program size. opt_size: Option, + pgo_gen: Option, + pgo_use: String, + // Flags indicating which outputs to produce. emit_no_opt_bc: bool, emit_bc: bool, @@ -274,6 +277,9 @@ impl ModuleConfig { opt_level: None, opt_size: None, + pgo_gen: None, + pgo_use: String::new(), + emit_no_opt_bc: false, emit_bc: false, emit_bc_compressed: false, @@ -492,8 +498,13 @@ unsafe extern "C" fn diagnostic_handler(info: DiagnosticInfoRef, user: *mut c_vo opt.message)); } } - - _ => (), + llvm::diagnostic::PGO(diagnostic_ref) => { + let msg = llvm::build_string(|s| { + llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) + }).expect("non-UTF8 PGO diagnostic"); + diag_handler.warn(&msg); + } + llvm::diagnostic::UnknownDiagnostic(..) => {}, } } @@ -932,6 +943,9 @@ pub fn start_async_translation(tcx: TyCtxt, modules_config.passes.push("insert-gcov-profiling".to_owned()) } + modules_config.pgo_gen = sess.opts.debugging_opts.pgo_gen.clone(); + modules_config.pgo_use = sess.opts.debugging_opts.pgo_use.clone(); + modules_config.opt_level = Some(get_llvm_opt_level(sess.opts.optimize)); modules_config.opt_size = Some(get_llvm_opt_size(sess.opts.optimize)); @@ -2046,6 +2060,8 @@ pub unsafe fn with_llvm_pmb(llmod: ModuleRef, config: &ModuleConfig, opt_level: llvm::CodeGenOptLevel, f: &mut FnMut(llvm::PassManagerBuilderRef)) { + use std::ptr; + // Create the PassManagerBuilder for LLVM. We configure it with // reasonable defaults and prepare it to actually populate the pass // manager. @@ -2053,11 +2069,27 @@ pub unsafe fn with_llvm_pmb(llmod: ModuleRef, let opt_size = config.opt_size.unwrap_or(llvm::CodeGenOptSizeNone); let inline_threshold = config.inline_threshold; - llvm::LLVMRustConfigurePassManagerBuilder(builder, - opt_level, - config.merge_functions, - config.vectorize_slp, - config.vectorize_loop); + let pgo_gen_path = config.pgo_gen.as_ref().map(|s| { + let s = if s.is_empty() { "default_%m.profraw" } else { s }; + CString::new(s.as_bytes()).unwrap() + }); + + let pgo_use_path = if config.pgo_use.is_empty() { + None + } else { + Some(CString::new(config.pgo_use.as_bytes()).unwrap()) + }; + + llvm::LLVMRustConfigurePassManagerBuilder( + builder, + opt_level, + config.merge_functions, + config.vectorize_slp, + config.vectorize_loop, + pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + ); + llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32); if opt_size != llvm::CodeGenOptSizeNone { diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs index 56eece9f31e7e..c839e5340f58d 100644 --- a/src/librustc_trans/base.rs +++ b/src/librustc_trans/base.rs @@ -708,6 +708,13 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, } } + if (tcx.sess.opts.debugging_opts.pgo_gen.is_some() || + !tcx.sess.opts.debugging_opts.pgo_use.is_empty()) && + unsafe { !llvm::LLVMRustPGOAvailable() } + { + tcx.sess.fatal("this compiler's LLVM does not support PGO"); + } + let crate_hash = tcx.crate_hash(LOCAL_CRATE); let link_meta = link::build_link_meta(crate_hash); diff --git a/src/librustc_trans/llvm_util.rs b/src/librustc_trans/llvm_util.rs index 5113b65a5c470..1c8f09ce7b3f1 100644 --- a/src/librustc_trans/llvm_util.rs +++ b/src/librustc_trans/llvm_util.rs @@ -61,6 +61,9 @@ unsafe fn configure_llvm(sess: &Session) { add("rustc"); // fake program name if sess.time_llvm_passes() { add("-time-passes"); } if sess.print_llvm_passes() { add("-debug-pass=Structure"); } + if sess.opts.debugging_opts.disable_instrumentation_preinliner { + add("-disable-preinline"); + } for arg in &sess.opts.cg.llvm_args { add(&(*arg)); diff --git a/src/rustllvm/PassWrapper.cpp b/src/rustllvm/PassWrapper.cpp index 06d1301d70003..3d5cce81278ac 100644 --- a/src/rustllvm/PassWrapper.cpp +++ b/src/rustllvm/PassWrapper.cpp @@ -44,6 +44,10 @@ #include "llvm-c/Transforms/PassManagerBuilder.h" +#if LLVM_VERSION_GE(4, 0) +#define PGO_AVAILABLE +#endif + using namespace llvm; using namespace llvm::legacy; @@ -428,12 +432,27 @@ extern "C" void LLVMRustAddAnalysisPasses(LLVMTargetMachineRef TM, extern "C" void LLVMRustConfigurePassManagerBuilder( LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, - bool MergeFunctions, bool SLPVectorize, bool LoopVectorize) { + bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, + const char* PGOGenPath, const char* PGOUsePath) { // Ignore mergefunc for now as enabling it causes crashes. // unwrap(PMBR)->MergeFunctions = MergeFunctions; unwrap(PMBR)->SLPVectorize = SLPVectorize; unwrap(PMBR)->OptLevel = fromRust(OptLevel); unwrap(PMBR)->LoopVectorize = LoopVectorize; + +#ifdef PGO_AVAILABLE + if (PGOGenPath) { + assert(!PGOUsePath); + unwrap(PMBR)->EnablePGOInstrGen = true; + unwrap(PMBR)->PGOInstrGen = PGOGenPath; + } + if (PGOUsePath) { + assert(!PGOGenPath); + unwrap(PMBR)->PGOInstrUse = PGOUsePath; + } +#else + assert(!PGOGenPath && !PGOUsePath && "Should've caught earlier"); +#endif } // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` @@ -766,6 +785,15 @@ LLVMRustThinLTOAvailable() { #endif } +extern "C" bool +LLVMRustPGOAvailable() { +#ifdef PGO_AVAILABLE + return true; +#else + return false; +#endif +} + #if LLVM_VERSION_GE(4, 0) // Here you'll find an implementation of ThinLTO as used by the Rust compiler diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index e815d151aeba9..0ef9643f4cab4 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -1021,6 +1021,7 @@ enum class LLVMRustDiagnosticKind { OptimizationRemarkAnalysisAliasing, OptimizationRemarkOther, OptimizationFailure, + PGOProfile, }; static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) { @@ -1043,6 +1044,8 @@ static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) { return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisFPCommute; case DK_OptimizationRemarkAnalysisAliasing: return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisAliasing; + case DK_PGOProfile: + return LLVMRustDiagnosticKind::PGOProfile; default: return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark) ? LLVMRustDiagnosticKind::OptimizationRemarkOther diff --git a/src/test/run-make-fulldeps/pgo-gen-lto/Makefile b/src/test/run-make-fulldeps/pgo-gen-lto/Makefile new file mode 100644 index 0000000000000..e8c695f52bec0 --- /dev/null +++ b/src/test/run-make-fulldeps/pgo-gen-lto/Makefile @@ -0,0 +1,8 @@ +-include ../tools.mk + +all: +ifeq ($(PROFILER_SUPPORT),1) + $(RUSTC) -Copt-level=3 -Clto=fat -Z pgo-gen="$(TMPDIR)/test.profraw" test.rs + $(call RUN,test) || exit 1 + [ -e "$(TMPDIR)/test.profraw" ] || (echo "No .profraw file"; exit 1) +endif diff --git a/src/test/run-make-fulldeps/pgo-gen-lto/test.rs b/src/test/run-make-fulldeps/pgo-gen-lto/test.rs new file mode 100644 index 0000000000000..3f07b46791d22 --- /dev/null +++ b/src/test/run-make-fulldeps/pgo-gen-lto/test.rs @@ -0,0 +1,11 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +fn main() {} diff --git a/src/test/run-make-fulldeps/pgo-gen/Makefile b/src/test/run-make-fulldeps/pgo-gen/Makefile new file mode 100644 index 0000000000000..7dc227b5a145a --- /dev/null +++ b/src/test/run-make-fulldeps/pgo-gen/Makefile @@ -0,0 +1,8 @@ +-include ../tools.mk + +all: +ifeq ($(PROFILER_SUPPORT),1) + $(RUSTC) -g -Z pgo-gen="$(TMPDIR)/test.profraw" test.rs + $(call RUN,test) || exit 1 + [ -e "$(TMPDIR)/test.profraw" ] || (echo "No .profraw file"; exit 1) +endif diff --git a/src/test/run-make-fulldeps/pgo-gen/test.rs b/src/test/run-make-fulldeps/pgo-gen/test.rs new file mode 100644 index 0000000000000..3f07b46791d22 --- /dev/null +++ b/src/test/run-make-fulldeps/pgo-gen/test.rs @@ -0,0 +1,11 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +fn main() {}