From 538d56af4f53665c19058cf5b78ed623d4e7d192 Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Sat, 21 Sep 2024 18:19:27 -0400 Subject: [PATCH 1/5] Add inline(usually) --- compiler/rustc_attr/src/builtin.rs | 1 + compiler/rustc_codegen_gcc/src/attributes.rs | 1 + compiler/rustc_codegen_llvm/src/attributes.rs | 3 +++ compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 4 +++- compiler/rustc_mir_transform/src/cross_crate_inline.rs | 2 +- compiler/rustc_mir_transform/src/inline.rs | 2 +- compiler/rustc_span/src/symbol.rs | 1 + 7 files changed, 11 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs index dcc4cf2478f33..0ad8748760b57 100644 --- a/compiler/rustc_attr/src/builtin.rs +++ b/compiler/rustc_attr/src/builtin.rs @@ -46,6 +46,7 @@ pub enum InlineAttr { Hint, Always, Never, + Usually, } #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)] diff --git a/compiler/rustc_codegen_gcc/src/attributes.rs b/compiler/rustc_codegen_gcc/src/attributes.rs index 5fdf2680aac88..6a5fd7d41b389 100644 --- a/compiler/rustc_codegen_gcc/src/attributes.rs +++ b/compiler/rustc_codegen_gcc/src/attributes.rs @@ -30,6 +30,7 @@ fn inline_attr<'gcc, 'tcx>( None } } + InlineAttr::Usually => Some(FnAttribute::Inline), InlineAttr::None => None, } } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 489259da85646..6147638d4d836 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -47,6 +47,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll } } InlineAttr::None => None, + InlineAttr::Usually => { + Some(llvm::CreateAttrStringValue(cx.llcx, "function-inline-cost", "0")) + } } } diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index e99c3a462711c..52461fe01d1fb 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -525,9 +525,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { .emit(); InlineAttr::None } else if list_contains_name(items, sym::always) { - InlineAttr::Always + InlineAttr::Usually } else if list_contains_name(items, sym::never) { InlineAttr::Never + } else if list_contains_name(items, sym::usually) { + InlineAttr::Usually } else { struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") .with_help("valid inline arguments are `always` and `never`") diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 42cbece32d8c9..71cb49fb2f57b 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { // #[inline(never)] to force code generation. match codegen_fn_attrs.inline { InlineAttr::Never => return false, - InlineAttr::Hint | InlineAttr::Always => return true, + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually => return true, _ => {} } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index c9f24764cc2a1..e63d088171d80 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -106,7 +106,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { changed: false, caller_is_inline_forwarder: matches!( codegen_fn_attrs.inline, - InlineAttr::Hint | InlineAttr::Always + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually ) && body_is_forwarder(body), }; let blocks = START_BLOCK..body.basic_blocks.next_index(); diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index cfe990a225fe2..1c8304a0ec616 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -2101,6 +2101,7 @@ symbols! { usize_legacy_fn_max_value, usize_legacy_fn_min_value, usize_legacy_mod, + usually, va_arg, va_copy, va_end, From 8caf2219652817181e87268953fe4cc6549a5c7f Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Thu, 26 Sep 2024 17:36:27 -0400 Subject: [PATCH 2/5] Give InlineAttr::Usually a bit more juice --- compiler/rustc_middle/src/mir/mono.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 954f746ce5b61..6c90479e64f26 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -138,7 +138,7 @@ impl<'tcx> MonoItem<'tcx> { // conflict with upstream crates as it could be an exported // symbol. match tcx.codegen_fn_attrs(instance.def_id()).inline { - InlineAttr::Always => InstantiationMode::LocalCopy, + InlineAttr::Always | InlineAttr::Usually => InstantiationMode::LocalCopy, _ => InstantiationMode::GloballyShared { may_conflict: true }, } } From 03a186c497795ef0594940f99a392139a69a9a50 Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Fri, 27 Sep 2024 18:03:51 -0400 Subject: [PATCH 3/5] Hack in alwaysinline for functions like stm32f4 emits --- compiler/rustc_codegen_llvm/src/attributes.rs | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 6147638d4d836..d38ad4bf338d8 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -327,6 +327,27 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc") } +fn should_always_inline(body: &rustc_middle::mir::Body<'_>) -> bool { + use rustc_middle::mir::*; + match body.basic_blocks.len() { + 0 => return true, + 1 => {} + 2.. => return false, + } + let block = &body.basic_blocks[START_BLOCK]; + match block.statements.len() { + 0 => { + matches!(block.terminator().kind, TerminatorKind::Return) + } + 1 => { + let statement = &block.statements[0]; + matches!(statement.kind, StatementKind::Assign(_)) + && matches!(block.terminator().kind, TerminatorKind::Return) + } + 2.. => return false, + } +} + /// Helper for `FnAbi::apply_attrs_llfn`: /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. @@ -356,7 +377,17 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( } else { codegen_fn_attrs.inline }; - to_add.extend(inline_attr(cx, inline)); + + if cx.tcx.is_mir_available(instance.def_id()) { + let body = cx.tcx.instance_mir(instance.def); + if should_always_inline(body) { + to_add.push(AttributeKind::AlwaysInline.create_attr(cx.llcx)); + } else { + to_add.extend(inline_attr(cx, inline)); + } + } else { + to_add.extend(inline_attr(cx, inline)); + } // The `uwtable` attribute according to LLVM is: // From 1ef34d6aef0679a5a15ee12d1cd34a6562db547e Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Sat, 28 Sep 2024 17:52:13 -0400 Subject: [PATCH 4/5] Only use Usually when optimizations are off --- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 52461fe01d1fb..e09ed0cf2b2a0 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -13,6 +13,7 @@ use rustc_middle::middle::codegen_fn_attrs::{ use rustc_middle::mir::mono::Linkage; use rustc_middle::query::Providers; use rustc_middle::ty::{self as ty, TyCtxt}; +use rustc_session::config::OptLevel; use rustc_session::lint; use rustc_session::parse::feature_err; use rustc_span::symbol::Ident; @@ -525,7 +526,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { .emit(); InlineAttr::None } else if list_contains_name(items, sym::always) { - InlineAttr::Usually + if tcx.sess.opts.optimize == OptLevel::No { + InlineAttr::Usually + } else { + InlineAttr::Always + } } else if list_contains_name(items, sym::never) { InlineAttr::Never } else if list_contains_name(items, sym::usually) { From 8c952070dcac734defe8a14dbd8a4cdf3799cfac Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Sat, 28 Sep 2024 21:37:10 -0400 Subject: [PATCH 5/5] Don't special-case Usually in instantiation mode selection --- compiler/rustc_middle/src/mir/mono.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 6c90479e64f26..954f746ce5b61 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -138,7 +138,7 @@ impl<'tcx> MonoItem<'tcx> { // conflict with upstream crates as it could be an exported // symbol. match tcx.codegen_fn_attrs(instance.def_id()).inline { - InlineAttr::Always | InlineAttr::Usually => InstantiationMode::LocalCopy, + InlineAttr::Always => InstantiationMode::LocalCopy, _ => InstantiationMode::GloballyShared { may_conflict: true }, } }