diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs index 309049b98f0e2..7803adb9e324d 100644 --- a/compiler/rustc_attr/src/builtin.rs +++ b/compiler/rustc_attr/src/builtin.rs @@ -46,6 +46,7 @@ pub enum InlineAttr { Hint, Always, Never, + Usually, } #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)] diff --git a/compiler/rustc_codegen_gcc/src/attributes.rs b/compiler/rustc_codegen_gcc/src/attributes.rs index 5fdf2680aac88..6a5fd7d41b389 100644 --- a/compiler/rustc_codegen_gcc/src/attributes.rs +++ b/compiler/rustc_codegen_gcc/src/attributes.rs @@ -30,6 +30,7 @@ fn inline_attr<'gcc, 'tcx>( None } } + InlineAttr::Usually => Some(FnAttribute::Inline), InlineAttr::None => None, } } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 489259da85646..6147638d4d836 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -47,6 +47,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll } } InlineAttr::None => None, + InlineAttr::Usually => { + Some(llvm::CreateAttrStringValue(cx.llcx, "function-inline-cost", "0")) + } } } diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 9bd8a84f5a3f0..48e5e491682cd 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -525,9 +525,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { .emit(); InlineAttr::None } else if list_contains_name(items, sym::always) { - InlineAttr::Always + InlineAttr::Usually } else if list_contains_name(items, sym::never) { InlineAttr::Never + } else if list_contains_name(items, sym::usually) { + InlineAttr::Usually } else { struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") .with_help("valid inline arguments are `always` and `never`") diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 42cbece32d8c9..71cb49fb2f57b 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { // #[inline(never)] to force code generation. match codegen_fn_attrs.inline { InlineAttr::Never => return false, - InlineAttr::Hint | InlineAttr::Always => return true, + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually => return true, _ => {} } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 2de75e2ef50fc..a9642a0396058 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -106,7 +106,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { changed: false, caller_is_inline_forwarder: matches!( codegen_fn_attrs.inline, - InlineAttr::Hint | InlineAttr::Always + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually ) && body_is_forwarder(body), }; let blocks = START_BLOCK..body.basic_blocks.next_index(); diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index de4532bcb9950..20b507bcdb449 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -2101,6 +2101,7 @@ symbols! { usize_legacy_fn_max_value, usize_legacy_fn_min_value, usize_legacy_mod, + usually, va_arg, va_copy, va_end, diff --git a/library/core/src/alloc/layout.rs b/library/core/src/alloc/layout.rs index 107d82267e576..ec4f07872b4b0 100644 --- a/library/core/src/alloc/layout.rs +++ b/library/core/src/alloc/layout.rs @@ -84,7 +84,8 @@ impl Layout { true } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] const fn max_size_for_align(align: Alignment) -> usize { // (power-of-two implies align != 0.) diff --git a/library/core/src/alloc/mod.rs b/library/core/src/alloc/mod.rs index aa841db045ce7..dfb3c264d6316 100644 --- a/library/core/src/alloc/mod.rs +++ b/library/core/src/alloc/mod.rs @@ -360,7 +360,8 @@ pub unsafe trait Allocator { /// Creates a "by reference" adapter for this instance of `Allocator`. /// /// The returned adapter also implements `Allocator` and will simply borrow this. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn by_ref(&self) -> &Self where Self: Sized, diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs index a3a471a57c7aa..88741f55aaeaa 100644 --- a/library/core/src/cell.rs +++ b/library/core/src/cell.rs @@ -813,12 +813,14 @@ fn panic_already_mutably_borrowed(err: BorrowError) -> ! { type BorrowFlag = isize; const UNUSED: BorrowFlag = 0; -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] fn is_writing(x: BorrowFlag) -> bool { x < UNUSED } -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] fn is_reading(x: BorrowFlag) -> bool { x > UNUSED } @@ -2079,7 +2081,8 @@ impl UnsafeCell { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_unsafe_cell_new", since = "1.32.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn new(value: T) -> UnsafeCell { UnsafeCell { value } } @@ -2095,7 +2098,8 @@ impl UnsafeCell { /// /// let five = uc.into_inner(); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "rust1", since = "1.0.0")] // When this is const stabilized, please remove `primitive_into_inner` below. #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")] @@ -2119,7 +2123,8 @@ impl UnsafeCell { /// *uc.get_mut() -= 1; /// assert_eq!(*uc.get_mut(), 41); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[unstable(feature = "unsafe_cell_from_mut", issue = "111645")] pub const fn from_mut(value: &mut T) -> &mut UnsafeCell { // SAFETY: `UnsafeCell` has the same memory layout as `T` due to #[repr(transparent)]. @@ -2142,7 +2147,8 @@ impl UnsafeCell { /// /// let five = uc.get(); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_unsafecell_get", since = "1.32.0")] #[rustc_never_returns_null_ptr] @@ -2168,7 +2174,8 @@ impl UnsafeCell { /// /// assert_eq!(*c.get_mut(), 6); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "unsafe_cell_get_mut", since = "1.50.0")] #[rustc_const_unstable(feature = "const_unsafecell_get_mut", issue = "88836")] pub const fn get_mut(&mut self) -> &mut T { @@ -2203,7 +2210,8 @@ impl UnsafeCell { /// /// assert_eq!(uc.into_inner(), 5); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "unsafe_cell_raw_get", since = "1.56.0")] #[rustc_const_stable(feature = "unsafe_cell_raw_get", since = "1.56.0")] #[rustc_diagnostic_item = "unsafe_cell_raw_get"] diff --git a/library/core/src/clone.rs b/library/core/src/clone.rs index c5f8bd7401e5e..3a908e1723c27 100644 --- a/library/core/src/clone.rs +++ b/library/core/src/clone.rs @@ -321,7 +321,7 @@ mod impls { $( #[stable(feature = "rust1", since = "1.0.0")] impl Clone for $t { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn clone(&self) -> Self { *self } @@ -347,7 +347,8 @@ mod impls { #[stable(feature = "rust1", since = "1.0.0")] impl Clone for *const T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn clone(&self) -> Self { *self } @@ -355,7 +356,8 @@ mod impls { #[stable(feature = "rust1", since = "1.0.0")] impl Clone for *mut T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn clone(&self) -> Self { *self } @@ -364,7 +366,8 @@ mod impls { /// Shared references can be cloned, but mutable references *cannot*! #[stable(feature = "rust1", since = "1.0.0")] impl Clone for &T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[rustc_diagnostic_item = "noop_method_clone"] fn clone(&self) -> Self { *self diff --git a/library/core/src/cmp.rs b/library/core/src/cmp.rs index 818a36002e759..2a88c7361270d 100644 --- a/library/core/src/cmp.rs +++ b/library/core/src/cmp.rs @@ -1520,13 +1520,13 @@ mod impls { (true, true) => Some(Equal), } } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn lt(&self, other: &$t) -> bool { (*self) < (*other) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn le(&self, other: &$t) -> bool { (*self) <= (*other) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn ge(&self, other: &$t) -> bool { (*self) >= (*other) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn gt(&self, other: &$t) -> bool { (*self) > (*other) } } )*) @@ -1558,13 +1558,13 @@ mod impls { fn partial_cmp(&self, other: &$t) -> Option { Some(crate::intrinsics::three_way_compare(*self, *other)) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn lt(&self, other: &$t) -> bool { (*self) < (*other) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn le(&self, other: &$t) -> bool { (*self) <= (*other) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn ge(&self, other: &$t) -> bool { (*self) >= (*other) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn gt(&self, other: &$t) -> bool { (*self) > (*other) } } diff --git a/library/core/src/convert/mod.rs b/library/core/src/convert/mod.rs index 432e55e8c9a4c..f4fc9b74abcb4 100644 --- a/library/core/src/convert/mod.rs +++ b/library/core/src/convert/mod.rs @@ -99,7 +99,8 @@ pub use num::FloatToInt; /// ``` #[stable(feature = "convert_id", since = "1.33.0")] #[rustc_const_stable(feature = "const_identity", since = "1.33.0")] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_diagnostic_item = "convert_identity"] pub const fn identity(x: T) -> T { x @@ -764,7 +765,8 @@ where #[stable(feature = "rust1", since = "1.0.0")] impl From for T { /// Returns the argument unchanged. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn from(t: T) -> T { t } @@ -820,7 +822,8 @@ where #[stable(feature = "rust1", since = "1.0.0")] impl AsRef<[T]> for [T] { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn as_ref(&self) -> &[T] { self } @@ -828,7 +831,8 @@ impl AsRef<[T]> for [T] { #[stable(feature = "rust1", since = "1.0.0")] impl AsMut<[T]> for [T] { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn as_mut(&mut self) -> &mut [T] { self } @@ -836,7 +840,8 @@ impl AsMut<[T]> for [T] { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for str { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn as_ref(&self) -> &str { self } @@ -844,7 +849,8 @@ impl AsRef for str { #[stable(feature = "as_mut_str_for_str", since = "1.51.0")] impl AsMut for str { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn as_mut(&mut self) -> &mut str { self } diff --git a/library/core/src/convert/num.rs b/library/core/src/convert/num.rs index 0246d0627cafe..273daa5ec9cea 100644 --- a/library/core/src/convert/num.rs +++ b/library/core/src/convert/num.rs @@ -73,7 +73,7 @@ macro_rules! impl_from { // Rustdocs on the impl block show a "[+] show undocumented items" toggle. // Rustdocs on functions do not. #[doc = $doc] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn from(small: $Small) -> Self { small as Self } diff --git a/library/core/src/default.rs b/library/core/src/default.rs index 4c30290ff263b..64dfbbc777b3a 100644 --- a/library/core/src/default.rs +++ b/library/core/src/default.rs @@ -151,7 +151,8 @@ macro_rules! default_impl { ($t:ty, $v:expr, $doc:tt) => { #[stable(feature = "rust1", since = "1.0.0")] impl Default for $t { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[doc = $doc] fn default() -> $t { $v diff --git a/library/core/src/fmt/rt.rs b/library/core/src/fmt/rt.rs index eee4a9e4c6c89..56e333d40aa5e 100644 --- a/library/core/src/fmt/rt.rs +++ b/library/core/src/fmt/rt.rs @@ -19,7 +19,8 @@ pub struct Placeholder { } impl Placeholder { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn new( position: usize, fill: char, @@ -95,7 +96,8 @@ pub struct Argument<'a> { #[rustc_diagnostic_item = "ArgumentMethods"] impl<'a> Argument<'a> { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn new<'b, T>(x: &'b T, f: fn(&T, &mut Formatter<'_>) -> Result) -> Argument<'b> { Argument { // INVARIANT: this creates an `ArgumentType<'b>` from a `&'b T` and @@ -109,47 +111,58 @@ impl<'a> Argument<'a> { } } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_display<'b, T: Display>(x: &'b T) -> Argument<'b> { Self::new(x, Display::fmt) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_debug<'b, T: Debug>(x: &'b T) -> Argument<'b> { Self::new(x, Debug::fmt) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_debug_noop<'b, T: Debug>(x: &'b T) -> Argument<'b> { Self::new(x, |_, _| Ok(())) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_octal<'b, T: Octal>(x: &'b T) -> Argument<'b> { Self::new(x, Octal::fmt) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_lower_hex<'b, T: LowerHex>(x: &'b T) -> Argument<'b> { Self::new(x, LowerHex::fmt) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_upper_hex<'b, T: UpperHex>(x: &'b T) -> Argument<'b> { Self::new(x, UpperHex::fmt) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_pointer<'b, T: Pointer>(x: &'b T) -> Argument<'b> { Self::new(x, Pointer::fmt) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_binary<'b, T: Binary>(x: &'b T) -> Argument<'b> { Self::new(x, Binary::fmt) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_lower_exp<'b, T: LowerExp>(x: &'b T) -> Argument<'b> { Self::new(x, LowerExp::fmt) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn new_upper_exp<'b, T: UpperExp>(x: &'b T) -> Argument<'b> { Self::new(x, UpperExp::fmt) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn from_usize(x: &usize) -> Argument<'_> { Argument { ty: ArgumentType::Count(*x) } } @@ -164,7 +177,8 @@ impl<'a> Argument<'a> { // it here is an explicit CFI violation. #[allow(inline_no_sanitize)] #[no_sanitize(cfi, kcfi)] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub(super) unsafe fn fmt(&self, f: &mut Formatter<'_>) -> Result { match self.ty { // SAFETY: @@ -180,7 +194,8 @@ impl<'a> Argument<'a> { } } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub(super) fn as_usize(&self) -> Option { match self.ty { ArgumentType::Count(count) => Some(count), @@ -198,7 +213,8 @@ impl<'a> Argument<'a> { /// let f = format_args!("{}", "a"); /// println!("{f}"); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn none() -> [Self; 0] { [] } @@ -215,7 +231,8 @@ pub struct UnsafeArg { impl UnsafeArg { /// See documentation where `UnsafeArg` is required to know when it is safe to /// create and use `UnsafeArg`. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub unsafe fn new() -> Self { Self { _private: () } } diff --git a/library/core/src/future/async_drop.rs b/library/core/src/future/async_drop.rs index 16ac77fa15045..e2e5ccb9a0b5d 100644 --- a/library/core/src/future/async_drop.rs +++ b/library/core/src/future/async_drop.rs @@ -107,7 +107,8 @@ impl fmt::Debug for AsyncDropInPlace { impl Future for AsyncDropInPlace { type Output = (); - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { // SAFETY: This code simply forwards poll call to the inner future unsafe { Pin::new_unchecked(&mut self.get_unchecked_mut().0) }.poll(cx) diff --git a/library/core/src/hint.rs b/library/core/src/hint.rs index a69f0afdb0a59..c0cbc6bebbcd4 100644 --- a/library/core/src/hint.rs +++ b/library/core/src/hint.rs @@ -193,7 +193,8 @@ pub const unsafe fn unreachable_unchecked() -> ! { /// pointer already has the builtin assumption that it is nonnull. However, it illustrates the /// kind of changes the optimizer can make even when the behavior is less obviously related. #[track_caller] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[doc(alias = "assume")] #[stable(feature = "hint_assert_unchecked", since = "1.81.0")] #[rustc_const_stable(feature = "hint_assert_unchecked", since = "1.81.0")] @@ -263,7 +264,8 @@ pub const unsafe fn assert_unchecked(cond: bool) { /// ``` /// /// [`thread::yield_now`]: ../../std/thread/fn.yield_now.html -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "renamed_spin_loop", since = "1.49.0")] pub fn spin_loop() { #[cfg(target_arch = "x86")] @@ -508,7 +510,8 @@ pub const fn black_box(dummy: T) -> T { #[unstable(feature = "hint_must_use", issue = "94745")] #[rustc_const_unstable(feature = "hint_must_use", issue = "94745")] #[must_use] // <-- :) -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] pub const fn must_use(value: T) -> T { value } diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index 7870a62ea81cd..2aacf710714e0 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -2947,7 +2947,8 @@ pub const unsafe fn typed_swap(x: *mut T, y: *mut T) { /// primarily used by [`ub_checks::assert_unsafe_precondition`]. #[rustc_const_unstable(feature = "const_ub_checks", issue = "none")] #[unstable(feature = "core_intrinsics", issue = "none")] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_intrinsic] pub const fn ub_checks() -> bool { cfg!(ub_checks) @@ -3287,7 +3288,8 @@ pub const fn ptr_metadata + ?Sized, M>(_ptr: *cons #[stable(feature = "rust1", since = "1.0.0")] #[rustc_allowed_through_unstable_modules] #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_diagnostic_item = "ptr_copy_nonoverlapping"] pub const unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize) { @@ -3389,7 +3391,8 @@ pub const unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: us #[stable(feature = "rust1", since = "1.0.0")] #[rustc_allowed_through_unstable_modules] #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_diagnostic_item = "ptr_copy"] pub const unsafe fn copy(src: *const T, dst: *mut T, count: usize) { @@ -3469,7 +3472,8 @@ pub const unsafe fn copy(src: *const T, dst: *mut T, count: usize) { #[stable(feature = "rust1", since = "1.0.0")] #[rustc_allowed_through_unstable_modules] #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_diagnostic_item = "ptr_write_bytes"] pub const unsafe fn write_bytes(dst: *mut T, val: u8, count: usize) { diff --git a/library/core/src/mem/manually_drop.rs b/library/core/src/mem/manually_drop.rs index 3e47785ee488e..fc28ff727484d 100644 --- a/library/core/src/mem/manually_drop.rs +++ b/library/core/src/mem/manually_drop.rs @@ -70,7 +70,8 @@ impl ManuallyDrop { #[must_use = "if you don't need the wrapper, you can use `mem::forget` instead"] #[stable(feature = "manually_drop", since = "1.20.0")] #[rustc_const_stable(feature = "const_manually_drop", since = "1.32.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn new(value: T) -> ManuallyDrop { ManuallyDrop { value } } @@ -88,7 +89,8 @@ impl ManuallyDrop { /// ``` #[stable(feature = "manually_drop", since = "1.20.0")] #[rustc_const_stable(feature = "const_manually_drop", since = "1.32.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn into_inner(slot: ManuallyDrop) -> T { slot.value } @@ -154,7 +156,8 @@ impl ManuallyDrop { #[stable(feature = "manually_drop", since = "1.20.0")] impl Deref for ManuallyDrop { type Target = T; - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn deref(&self) -> &T { &self.value } @@ -162,7 +165,8 @@ impl Deref for ManuallyDrop { #[stable(feature = "manually_drop", since = "1.20.0")] impl DerefMut for ManuallyDrop { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn deref_mut(&mut self) -> &mut T { &mut self.value } diff --git a/library/core/src/mem/maybe_uninit.rs b/library/core/src/mem/maybe_uninit.rs index c67796ad3db83..d03af8f0d3937 100644 --- a/library/core/src/mem/maybe_uninit.rs +++ b/library/core/src/mem/maybe_uninit.rs @@ -245,7 +245,8 @@ pub union MaybeUninit { #[stable(feature = "maybe_uninit", since = "1.36.0")] impl Clone for MaybeUninit { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn clone(&self) -> Self { // Not calling `T::clone()`, we cannot know if we are initialized enough for that. *self @@ -280,7 +281,8 @@ impl MaybeUninit { #[stable(feature = "maybe_uninit", since = "1.36.0")] #[rustc_const_stable(feature = "const_maybe_uninit", since = "1.36.0")] #[must_use = "use `forget` to avoid running Drop code"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn new(val: T) -> MaybeUninit { MaybeUninit { value: ManuallyDrop::new(val) } } @@ -302,7 +304,8 @@ impl MaybeUninit { #[stable(feature = "maybe_uninit", since = "1.36.0")] #[rustc_const_stable(feature = "const_maybe_uninit", since = "1.36.0")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[rustc_diagnostic_item = "maybe_uninit_uninit"] pub const fn uninit() -> MaybeUninit { MaybeUninit { uninit: () } @@ -340,7 +343,8 @@ impl MaybeUninit { #[unstable(feature = "maybe_uninit_uninit_array", issue = "96097")] #[rustc_const_unstable(feature = "const_maybe_uninit_uninit_array", issue = "96097")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn uninit_array() -> [Self; N] { [const { MaybeUninit::uninit() }; N] } @@ -487,7 +491,8 @@ impl MaybeUninit { /// ``` #[stable(feature = "maybe_uninit_write", since = "1.55.0")] #[rustc_const_unstable(feature = "const_maybe_uninit_write", issue = "63567")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn write(&mut self, val: T) -> &mut T { *self = MaybeUninit::new(val); // SAFETY: We just initialized this value. @@ -529,7 +534,8 @@ impl MaybeUninit { /// until they are, it is advisable to avoid them.) #[stable(feature = "maybe_uninit", since = "1.36.0")] #[rustc_const_stable(feature = "const_maybe_uninit_as_ptr", since = "1.59.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn as_ptr(&self) -> *const T { // `MaybeUninit` and `ManuallyDrop` are both `repr(transparent)` so we can cast the pointer. self as *const _ as *const T @@ -574,7 +580,8 @@ impl MaybeUninit { since = "CURRENT_RUSTC_VERSION" )] #[cfg_attr(bootstrap, rustc_allow_const_fn_unstable(const_mut_refs))] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn as_mut_ptr(&mut self) -> *mut T { // `MaybeUninit` and `ManuallyDrop` are both `repr(transparent)` so we can cast the pointer. self as *mut _ as *mut T @@ -627,7 +634,8 @@ impl MaybeUninit { /// ``` #[stable(feature = "maybe_uninit", since = "1.36.0")] #[rustc_const_stable(feature = "const_maybe_uninit_assume_init_by_value", since = "1.59.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[rustc_diagnostic_item = "assume_init"] #[track_caller] pub const unsafe fn assume_init(self) -> T { @@ -698,7 +706,8 @@ impl MaybeUninit { /// ``` #[stable(feature = "maybe_uninit_extra", since = "1.60.0")] #[rustc_const_stable(feature = "const_maybe_uninit_assume_init_read", since = "1.75.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const unsafe fn assume_init_read(&self) -> T { // SAFETY: the caller must guarantee that `self` is initialized. @@ -797,7 +806,8 @@ impl MaybeUninit { /// ``` #[stable(feature = "maybe_uninit_ref", since = "1.55.0")] #[rustc_const_stable(feature = "const_maybe_uninit_assume_init_ref", since = "1.59.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const unsafe fn assume_init_ref(&self) -> &T { // SAFETY: the caller must guarantee that `self` is initialized. // This also means that `self` must be a `value` variant. @@ -914,7 +924,8 @@ impl MaybeUninit { /// ``` #[stable(feature = "maybe_uninit_ref", since = "1.55.0")] #[rustc_const_unstable(feature = "const_maybe_uninit_assume_init", issue = "none")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const unsafe fn assume_init_mut(&mut self) -> &mut T { // SAFETY: the caller must guarantee that `self` is initialized. // This also means that `self` must be a `value` variant. @@ -951,7 +962,8 @@ impl MaybeUninit { /// ``` #[unstable(feature = "maybe_uninit_array_assume_init", issue = "96097")] #[rustc_const_unstable(feature = "const_maybe_uninit_array_assume_init", issue = "96097")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const unsafe fn array_assume_init(array: [Self; N]) -> [T; N] { // SAFETY: @@ -978,7 +990,8 @@ impl MaybeUninit { /// [`assume_init_ref`]: MaybeUninit::assume_init_ref #[unstable(feature = "maybe_uninit_slice", issue = "63569")] #[rustc_const_unstable(feature = "maybe_uninit_slice", issue = "63569")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const unsafe fn slice_assume_init_ref(slice: &[Self]) -> &[T] { // SAFETY: casting `slice` to a `*const [T]` is safe since the caller guarantees that // `slice` is initialized, and `MaybeUninit` is guaranteed to have the same layout as `T`. @@ -1000,7 +1013,8 @@ impl MaybeUninit { /// [`assume_init_mut`]: MaybeUninit::assume_init_mut #[unstable(feature = "maybe_uninit_slice", issue = "63569")] #[rustc_const_unstable(feature = "const_maybe_uninit_assume_init", issue = "none")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const unsafe fn slice_assume_init_mut(slice: &mut [Self]) -> &mut [T] { // SAFETY: similar to safety notes for `slice_get_ref`, but we have a // mutable reference which is also guaranteed to be valid for writes. @@ -1010,7 +1024,8 @@ impl MaybeUninit { /// Gets a pointer to the first element of the array. #[unstable(feature = "maybe_uninit_slice", issue = "63569")] #[rustc_const_unstable(feature = "maybe_uninit_slice", issue = "63569")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn slice_as_ptr(this: &[MaybeUninit]) -> *const T { this.as_ptr() as *const T } @@ -1018,7 +1033,8 @@ impl MaybeUninit { /// Gets a mutable pointer to the first element of the array. #[unstable(feature = "maybe_uninit_slice", issue = "63569")] #[rustc_const_unstable(feature = "maybe_uninit_slice", issue = "63569")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn slice_as_mut_ptr(this: &mut [MaybeUninit]) -> *mut T { this.as_mut_ptr() as *mut T } diff --git a/library/core/src/mem/mod.rs b/library/core/src/mem/mod.rs index 414262fcf5ab1..47c31e2b7bcbb 100644 --- a/library/core/src/mem/mod.rs +++ b/library/core/src/mem/mod.rs @@ -299,7 +299,8 @@ pub fn forget_unsized(t: T) { /// [`Box`]: ../../std/boxed/struct.Box.html /// [`Option<&T>`]: crate::option::Option /// -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] @@ -459,7 +460,8 @@ pub fn min_align_of_val(val: &T) -> usize { /// /// assert_eq!(4, mem::align_of::()); /// ``` -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] @@ -644,7 +646,8 @@ pub const fn needs_drop() -> bool { /// let _x: &i32 = unsafe { mem::zeroed() }; // Undefined behavior! /// let _y: fn() = unsafe { mem::zeroed() }; // And again! /// ``` -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[allow(deprecated_in_future)] @@ -683,7 +686,8 @@ pub const unsafe fn zeroed() -> T { /// [uninit]: MaybeUninit::uninit /// [assume_init]: MaybeUninit::assume_init /// [inv]: MaybeUninit#initialization-invariant -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[deprecated(since = "1.39.0", note = "use `mem::MaybeUninit` instead")] #[stable(feature = "rust1", since = "1.0.0")] @@ -1194,7 +1198,8 @@ pub const fn discriminant(v: &T) -> Discriminant { /// assert_eq!(mem::variant_count::>(), 2); /// assert_eq!(mem::variant_count::>(), 2); /// ``` -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[unstable(feature = "variant_count", issue = "73662")] #[rustc_const_unstable(feature = "variant_count", issue = "73662")] diff --git a/library/core/src/num/dec2flt/common.rs b/library/core/src/num/dec2flt/common.rs index 4dadf406ae8c7..0df8da974a28b 100644 --- a/library/core/src/num/dec2flt/common.rs +++ b/library/core/src/num/dec2flt/common.rs @@ -18,14 +18,16 @@ pub(crate) trait ByteSlice { } impl ByteSlice for [u8] { - #[inline(always)] // inlining this is crucial to remove bound checks + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] // inlining this is crucial to remove bound checks fn read_u64(&self) -> u64 { let mut tmp = [0; 8]; tmp.copy_from_slice(&self[..8]); u64::from_le_bytes(tmp) } - #[inline(always)] // inlining this is crucial to remove bound checks + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] // inlining this is crucial to remove bound checks fn write_u64(&mut self, value: u64) { self[..8].copy_from_slice(&value.to_le_bytes()) } diff --git a/library/core/src/num/dec2flt/mod.rs b/library/core/src/num/dec2flt/mod.rs index 87bfd0d256634..5071c56d42373 100644 --- a/library/core/src/num/dec2flt/mod.rs +++ b/library/core/src/num/dec2flt/mod.rs @@ -147,7 +147,7 @@ macro_rules! from_str_float_impl { /// by `src` (following the same rules for rounding as for the /// results of primitive operations). // We add the `#[inline(never)]` attribute, since its content will - // be filled with that of `dec2flt`, which has #[inline(always)]. + // be filled with that of `dec2flt`, which has #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]. // Since `dec2flt` is generic, a normal inline attribute on this function // with `dec2flt` having no attributes results in heavily repeated // generation of `dec2flt`, despite the fact only a maximum of 2 @@ -227,7 +227,8 @@ fn biased_fp_to_float(x: BiasedFp) -> T { } /// Converts a decimal string into a floating point number. -#[inline(always)] // Will be inlined into a function with `#[inline(never)]`, see above +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] // Will be inlined into a function with `#[inline(never)]`, see above pub fn dec2flt(s: &str) -> Result { let mut s = s.as_bytes(); let c = if let Some(&c) = s.first() { diff --git a/library/core/src/num/int_macros.rs b/library/core/src/num/int_macros.rs index 7241b3ff6a3b7..199ef70ae0a3c 100644 --- a/library/core/src/num/int_macros.rs +++ b/library/core/src/num/int_macros.rs @@ -78,7 +78,7 @@ macro_rules! int_impl { #[doc(alias = "popcnt")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn count_ones(self) -> u32 { (self as $UnsignedT).count_ones() } /// Returns the number of zeros in the binary representation of `self`. @@ -94,7 +94,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn count_zeros(self) -> u32 { (!self).count_ones() } @@ -118,7 +118,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn leading_zeros(self) -> u32 { (self as $UnsignedT).leading_zeros() } @@ -138,7 +138,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn trailing_zeros(self) -> u32 { (self as $UnsignedT).trailing_zeros() } @@ -158,7 +158,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "leading_trailing_ones", since = "1.46.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn leading_ones(self) -> u32 { (self as $UnsignedT).leading_ones() } @@ -178,7 +178,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "leading_trailing_ones", since = "1.46.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn trailing_ones(self) -> u32 { (self as $UnsignedT).trailing_ones() } @@ -202,7 +202,7 @@ macro_rules! int_impl { #[unstable(feature = "integer_sign_cast", issue = "125882")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn cast_unsigned(self) -> $UnsignedT { self as $UnsignedT } @@ -226,7 +226,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn rotate_left(self, n: u32) -> Self { (self as $UnsignedT).rotate_left(n) as Self } @@ -251,7 +251,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn rotate_right(self, n: u32) -> Self { (self as $UnsignedT).rotate_right(n) as Self } @@ -273,7 +273,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn swap_bytes(self) -> Self { (self as $UnsignedT).swap_bytes() as Self } @@ -296,7 +296,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "reverse_bits", since = "1.37.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn reverse_bits(self) -> Self { (self as $UnsignedT).reverse_bits() as Self } @@ -509,7 +509,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "unchecked_math", since = "1.79.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_add(self, rhs: Self) -> Self { assert_unsafe_precondition!( @@ -661,7 +661,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "unchecked_math", since = "1.79.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_sub(self, rhs: Self) -> Self { assert_unsafe_precondition!( @@ -813,7 +813,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "unchecked_math", since = "1.79.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_mul(self, rhs: Self) -> Self { assert_unsafe_precondition!( @@ -1162,7 +1162,7 @@ macro_rules! int_impl { #[must_use = "this returns the result of the operation, \ without modifying the original"] #[rustc_const_unstable(feature = "unchecked_neg", issue = "85122")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_neg(self) -> Self { assert_unsafe_precondition!( @@ -1295,7 +1295,7 @@ macro_rules! int_impl { #[must_use = "this returns the result of the operation, \ without modifying the original"] #[rustc_const_unstable(feature = "unchecked_shifts", issue = "85122")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_shl(self, rhs: u32) -> Self { assert_unsafe_precondition!( @@ -1421,7 +1421,7 @@ macro_rules! int_impl { #[must_use = "this returns the result of the operation, \ without modifying the original"] #[rustc_const_unstable(feature = "unchecked_shifts", issue = "85122")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_shr(self, rhs: u32) -> Self { assert_unsafe_precondition!( @@ -1688,7 +1688,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn saturating_add(self, rhs: Self) -> Self { intrinsics::saturating_add(self, rhs) } @@ -1734,7 +1734,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn saturating_sub(self, rhs: Self) -> Self { intrinsics::saturating_sub(self, rhs) } @@ -1782,7 +1782,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn saturating_neg(self) -> Self { intrinsics::saturating_sub(0, self) } @@ -1911,7 +1911,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_add(self, rhs: Self) -> Self { intrinsics::wrapping_add(self, rhs) } @@ -1931,7 +1931,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_add_unsigned(self, rhs: $UnsignedT) -> Self { self.wrapping_add(rhs as Self) } @@ -1951,7 +1951,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_sub(self, rhs: Self) -> Self { intrinsics::wrapping_sub(self, rhs) } @@ -1971,7 +1971,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_sub_unsigned(self, rhs: $UnsignedT) -> Self { self.wrapping_sub(rhs as Self) } @@ -1991,7 +1991,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_mul(self, rhs: Self) -> Self { intrinsics::wrapping_mul(self, rhs) } @@ -2126,7 +2126,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_neg(self) -> Self { (0 as $SelfT).wrapping_sub(self) } @@ -2151,7 +2151,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_allow_const_fn_unstable(unchecked_shifts)] pub const fn wrapping_shl(self, rhs: u32) -> Self { // SAFETY: the masking by the bitsize of the type ensures that we do not shift @@ -2181,7 +2181,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_allow_const_fn_unstable(unchecked_shifts)] pub const fn wrapping_shr(self, rhs: u32) -> Self { // SAFETY: the masking by the bitsize of the type ensures that we do not shift @@ -2319,7 +2319,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn overflowing_add(self, rhs: Self) -> (Self, bool) { let (a, b) = intrinsics::add_with_overflow(self as $ActualT, rhs as $ActualT); (a as Self, b) @@ -2426,7 +2426,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn overflowing_sub(self, rhs: Self) -> (Self, bool) { let (a, b) = intrinsics::sub_with_overflow(self as $ActualT, rhs as $ActualT); (a as Self, b) @@ -2534,7 +2534,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn overflowing_mul(self, rhs: Self) -> (Self, bool) { let (a, b) = intrinsics::mul_with_overflow(self as $ActualT, rhs as $ActualT); (a as Self, b) @@ -3478,7 +3478,7 @@ macro_rules! int_impl { #[rustc_const_stable(feature = "const_int_sign", since = "1.47.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn signum(self) -> Self { // Picking the right way to phrase this is complicated // () @@ -3505,7 +3505,7 @@ macro_rules! int_impl { #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn is_positive(self) -> bool { self > 0 } /// Returns `true` if `self` is negative and `false` if the number is zero or @@ -3522,7 +3522,7 @@ macro_rules! int_impl { #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn is_negative(self) -> bool { self < 0 } /// Returns the memory representation of this integer as a byte array in @@ -3709,7 +3709,7 @@ macro_rules! int_impl { /// /// Returns the smallest value that can be represented by this integer type. #[stable(feature = "rust1", since = "1.0.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_promotable] #[rustc_const_stable(feature = "const_min_value", since = "1.32.0")] #[deprecated(since = "TBD", note = "replaced by the `MIN` associated constant on this type")] @@ -3723,7 +3723,7 @@ macro_rules! int_impl { /// /// Returns the largest value that can be represented by this integer type. #[stable(feature = "rust1", since = "1.0.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_promotable] #[rustc_const_stable(feature = "const_max_value", since = "1.32.0")] #[deprecated(since = "TBD", note = "replaced by the `MAX` associated constant on this type")] diff --git a/library/core/src/num/mod.rs b/library/core/src/num/mod.rs index 15de4fa15c481..2fb68192841cd 100644 --- a/library/core/src/num/mod.rs +++ b/library/core/src/num/mod.rs @@ -1393,7 +1393,8 @@ from_str_radix_int_impl! { isize i8 i16 i32 i64 i128 usize u8 u16 u32 u64 u128 } /// Note that if the radix is known to the compiler, it is just the check of digits.len that /// is done at runtime. #[doc(hidden)] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[unstable(issue = "none", feature = "std_internals")] #[rustc_const_stable(feature = "const_int_from_str", since = "1.82.0")] pub const fn can_not_overflow(radix: u32, is_signed_ty: bool, digits: &[u8]) -> bool { diff --git a/library/core/src/num/nonzero.rs b/library/core/src/num/nonzero.rs index e5c9a7e086ac9..af6bdc3573934 100644 --- a/library/core/src/num/nonzero.rs +++ b/library/core/src/num/nonzero.rs @@ -597,7 +597,7 @@ macro_rules! nonzero_integer { #[doc(alias = "popcnt")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn count_ones(self) -> NonZero { // SAFETY: // `self` is non-zero, which means it has at least one bit set, which means @@ -630,7 +630,7 @@ macro_rules! nonzero_integer { #[unstable(feature = "nonzero_bitwise", issue = "128281")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn rotate_left(self, n: u32) -> Self { let result = self.get().rotate_left(n); // SAFETY: Rotating bits preserves the property int > 0. @@ -663,7 +663,7 @@ macro_rules! nonzero_integer { #[unstable(feature = "nonzero_bitwise", issue = "128281")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn rotate_right(self, n: u32) -> Self { let result = self.get().rotate_right(n); // SAFETY: Rotating bits preserves the property int > 0. @@ -692,7 +692,7 @@ macro_rules! nonzero_integer { #[unstable(feature = "nonzero_bitwise", issue = "128281")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn swap_bytes(self) -> Self { let result = self.get().swap_bytes(); // SAFETY: Shuffling bytes preserves the property int > 0. @@ -722,7 +722,7 @@ macro_rules! nonzero_integer { #[unstable(feature = "nonzero_bitwise", issue = "128281")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn reverse_bits(self) -> Self { let result = self.get().reverse_bits(); // SAFETY: Reversing bits preserves the property int > 0. @@ -757,7 +757,7 @@ macro_rules! nonzero_integer { /// ``` #[unstable(feature = "nonzero_bitwise", issue = "128281")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn from_be(x: Self) -> Self { let result = $Int::from_be(x.get()); // SAFETY: Shuffling bytes preserves the property int > 0. @@ -792,7 +792,7 @@ macro_rules! nonzero_integer { /// ``` #[unstable(feature = "nonzero_bitwise", issue = "128281")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn from_le(x: Self) -> Self { let result = $Int::from_le(x.get()); // SAFETY: Shuffling bytes preserves the property int > 0. @@ -827,7 +827,7 @@ macro_rules! nonzero_integer { #[unstable(feature = "nonzero_bitwise", issue = "128281")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn to_be(self) -> Self { let result = self.get().to_be(); // SAFETY: Shuffling bytes preserves the property int > 0. @@ -862,7 +862,7 @@ macro_rules! nonzero_integer { #[unstable(feature = "nonzero_bitwise", issue = "128281")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn to_le(self) -> Self { let result = self.get().to_le(); // SAFETY: Shuffling bytes preserves the property int > 0. diff --git a/library/core/src/num/uint_macros.rs b/library/core/src/num/uint_macros.rs index d9036abecc592..828bca02642f1 100644 --- a/library/core/src/num/uint_macros.rs +++ b/library/core/src/num/uint_macros.rs @@ -80,7 +80,7 @@ macro_rules! uint_impl { #[doc(alias = "popcnt")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn count_ones(self) -> u32 { return intrinsics::ctpop(self); } @@ -102,7 +102,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn count_zeros(self) -> u32 { (!self).count_ones() } @@ -131,7 +131,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn leading_zeros(self) -> u32 { return intrinsics::ctlz(self as $ActualT); } @@ -157,7 +157,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn trailing_zeros(self) -> u32 { return intrinsics::cttz(self); } @@ -182,7 +182,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "leading_trailing_ones", since = "1.46.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn leading_ones(self) -> u32 { (!self).leading_zeros() } @@ -208,7 +208,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "leading_trailing_ones", since = "1.46.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn trailing_ones(self) -> u32 { (!self).trailing_zeros() } @@ -232,7 +232,7 @@ macro_rules! uint_impl { #[unstable(feature = "integer_sign_cast", issue = "125882")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn cast_signed(self) -> $SignedT { self as $SignedT } @@ -256,7 +256,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn rotate_left(self, n: u32) -> Self { return intrinsics::rotate_left(self, n); } @@ -281,7 +281,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn rotate_right(self, n: u32) -> Self { return intrinsics::rotate_right(self, n); } @@ -302,7 +302,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn swap_bytes(self) -> Self { intrinsics::bswap(self as $ActualT) as Self } @@ -325,7 +325,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "reverse_bits", since = "1.37.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn reverse_bits(self) -> Self { intrinsics::bitreverse(self as $ActualT) as Self } @@ -351,7 +351,7 @@ macro_rules! uint_impl { #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn from_be(x: Self) -> Self { #[cfg(target_endian = "big")] { @@ -384,7 +384,7 @@ macro_rules! uint_impl { #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn from_le(x: Self) -> Self { #[cfg(target_endian = "little")] { @@ -418,7 +418,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn to_be(self) -> Self { // or not to be? #[cfg(target_endian = "big")] { @@ -452,7 +452,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn to_le(self) -> Self { #[cfg(target_endian = "little")] { @@ -556,7 +556,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "unchecked_math", since = "1.79.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_add(self, rhs: Self) -> Self { assert_unsafe_precondition!( @@ -748,7 +748,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "unchecked_math", since = "1.79.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_sub(self, rhs: Self) -> Self { assert_unsafe_precondition!( @@ -905,7 +905,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "unchecked_math", since = "1.79.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_mul(self, rhs: Self) -> Self { assert_unsafe_precondition!( @@ -978,7 +978,7 @@ macro_rules! uint_impl { #[rustc_const_unstable(feature = "const_strict_overflow_ops", issue = "118260")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn strict_div(self, rhs: Self) -> Self { self / rhs @@ -1038,7 +1038,7 @@ macro_rules! uint_impl { #[rustc_const_unstable(feature = "const_strict_overflow_ops", issue = "118260")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn strict_div_euclid(self, rhs: Self) -> Self { self / rhs @@ -1100,7 +1100,7 @@ macro_rules! uint_impl { #[rustc_const_unstable(feature = "const_strict_overflow_ops", issue = "118260")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn strict_rem(self, rhs: Self) -> Self { self % rhs @@ -1162,7 +1162,7 @@ macro_rules! uint_impl { #[rustc_const_unstable(feature = "const_strict_overflow_ops", issue = "118260")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn strict_rem_euclid(self, rhs: Self) -> Self { self % rhs @@ -1484,7 +1484,7 @@ macro_rules! uint_impl { #[must_use = "this returns the result of the operation, \ without modifying the original"] #[rustc_const_unstable(feature = "unchecked_shifts", issue = "85122")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_shl(self, rhs: u32) -> Self { assert_unsafe_precondition!( @@ -1610,7 +1610,7 @@ macro_rules! uint_impl { #[must_use = "this returns the result of the operation, \ without modifying the original"] #[rustc_const_unstable(feature = "unchecked_shifts", issue = "85122")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn unchecked_shr(self, rhs: u32) -> Self { assert_unsafe_precondition!( @@ -1756,7 +1756,7 @@ macro_rules! uint_impl { #[must_use = "this returns the result of the operation, \ without modifying the original"] #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn saturating_add(self, rhs: Self) -> Self { intrinsics::saturating_add(self, rhs) } @@ -1804,7 +1804,7 @@ macro_rules! uint_impl { #[must_use = "this returns the result of the operation, \ without modifying the original"] #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn saturating_sub(self, rhs: Self) -> Self { intrinsics::saturating_sub(self, rhs) } @@ -1896,7 +1896,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_add(self, rhs: Self) -> Self { intrinsics::wrapping_add(self, rhs) } @@ -1937,7 +1937,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_sub(self, rhs: Self) -> Self { intrinsics::wrapping_sub(self, rhs) } @@ -1960,7 +1960,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_mul(self, rhs: Self) -> Self { intrinsics::wrapping_mul(self, rhs) } @@ -1986,7 +1986,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn wrapping_div(self, rhs: Self) -> Self { self / rhs @@ -2015,7 +2015,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn wrapping_div_euclid(self, rhs: Self) -> Self { self / rhs @@ -2043,7 +2043,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn wrapping_rem(self, rhs: Self) -> Self { self % rhs @@ -2073,7 +2073,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn wrapping_rem_euclid(self, rhs: Self) -> Self { self % rhs @@ -2103,7 +2103,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_neg(self) -> Self { (0 as $SelfT).wrapping_sub(self) } @@ -2131,7 +2131,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_allow_const_fn_unstable(unchecked_shifts)] pub const fn wrapping_shl(self, rhs: u32) -> Self { // SAFETY: the masking by the bitsize of the type ensures that we do not shift @@ -2164,7 +2164,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_allow_const_fn_unstable(unchecked_shifts)] pub const fn wrapping_shr(self, rhs: u32) -> Self { // SAFETY: the masking by the bitsize of the type ensures that we do not shift @@ -2248,7 +2248,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn overflowing_add(self, rhs: Self) -> (Self, bool) { let (a, b) = intrinsics::add_with_overflow(self as $ActualT, rhs as $ActualT); (a as Self, b) @@ -2347,7 +2347,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn overflowing_sub(self, rhs: Self) -> (Self, bool) { let (a, b) = intrinsics::sub_with_overflow(self as $ActualT, rhs as $ActualT); (a as Self, b) @@ -2446,7 +2446,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn overflowing_mul(self, rhs: Self) -> (Self, bool) { let (a, b) = intrinsics::mul_with_overflow(self as $ActualT, rhs as $ActualT); (a as Self, b) @@ -2470,7 +2470,7 @@ macro_rules! uint_impl { /// ``` #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_div(2), (2, false));")] /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "wrapping", since = "1.7.0")] #[rustc_const_stable(feature = "const_overflowing_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ @@ -2501,7 +2501,7 @@ macro_rules! uint_impl { /// ``` #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_div_euclid(2), (2, false));")] /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "euclidean_division", since = "1.38.0")] #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ @@ -2529,7 +2529,7 @@ macro_rules! uint_impl { /// ``` #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_rem(2), (1, false));")] /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "wrapping", since = "1.7.0")] #[rustc_const_stable(feature = "const_overflowing_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ @@ -2560,7 +2560,7 @@ macro_rules! uint_impl { /// ``` #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_rem_euclid(2), (1, false));")] /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "euclidean_division", since = "1.38.0")] #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ @@ -2585,7 +2585,7 @@ macro_rules! uint_impl { #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".overflowing_neg(), (0, false));")] #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".overflowing_neg(), (-2i32 as ", stringify!($SelfT), ", true));")] /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "wrapping", since = "1.7.0")] #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ @@ -2615,7 +2615,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn overflowing_shl(self, rhs: u32) -> (Self, bool) { (self.wrapping_shl(rhs), rhs >= Self::BITS) } @@ -2640,7 +2640,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn overflowing_shr(self, rhs: u32) -> (Self, bool) { (self.wrapping_shr(rhs), rhs >= Self::BITS) } @@ -2803,7 +2803,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn div_euclid(self, rhs: Self) -> Self { self / rhs @@ -2832,7 +2832,7 @@ macro_rules! uint_impl { #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn rem_euclid(self, rhs: Self) -> Self { self % rhs @@ -2857,7 +2857,7 @@ macro_rules! uint_impl { #[unstable(feature = "int_roundings", issue = "88581")] #[must_use = "this returns the result of the operation, \ without modifying the original"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] pub const fn div_floor(self, rhs: Self) -> Self { self / rhs @@ -2994,7 +2994,7 @@ macro_rules! uint_impl { #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_is_power_of_two", since = "1.32.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] pub const fn is_power_of_two(self) -> bool { self.count_ones() == 1 } @@ -3279,7 +3279,7 @@ macro_rules! uint_impl { /// Returns the smallest value that can be represented by this integer type. #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_const_stable(feature = "const_max_value", since = "1.32.0")] #[deprecated(since = "TBD", note = "replaced by the `MIN` associated constant on this type")] #[rustc_diagnostic_item = concat!(stringify!($SelfT), "_legacy_fn_min_value")] @@ -3291,7 +3291,7 @@ macro_rules! uint_impl { /// Returns the largest value that can be represented by this integer type. #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_const_stable(feature = "const_max_value", since = "1.32.0")] #[deprecated(since = "TBD", note = "replaced by the `MAX` associated constant on this type")] #[rustc_diagnostic_item = concat!(stringify!($SelfT), "_legacy_fn_max_value")] diff --git a/library/core/src/option.rs b/library/core/src/option.rs index 5ba13969605fc..f6aa15303c1c8 100644 --- a/library/core/src/option.rs +++ b/library/core/src/option.rs @@ -958,7 +958,8 @@ impl Option { /// let x: Option<&str> = None; /// assert_eq!(x.unwrap(), "air"); // fails /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] #[stable(feature = "rust1", since = "1.0.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "option_unwrap")] diff --git a/library/core/src/pin.rs b/library/core/src/pin.rs index 9c13662e08e8f..18878386ca7c8 100644 --- a/library/core/src/pin.rs +++ b/library/core/src/pin.rs @@ -1185,7 +1185,8 @@ impl> Pin { /// // pinning guarantees are actually upheld. /// let mut pinned: Pin<&mut u8> = Pin::new(&mut val); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[rustc_const_unstable(feature = "const_pin", issue = "76654")] #[stable(feature = "pin", since = "1.33.0")] pub const fn new(pointer: Ptr) -> Pin { @@ -1213,7 +1214,8 @@ impl> Pin { /// let r = Pin::into_inner(pinned); /// assert_eq!(*r, 5); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[rustc_const_unstable(feature = "const_pin", issue = "76654")] #[stable(feature = "pin_into_inner", since = "1.39.0")] pub const fn into_inner(pin: Pin) -> Ptr { @@ -1350,7 +1352,8 @@ impl Pin { /// [`mem::swap`]: crate::mem::swap /// [`pin` module docs]: self #[lang = "new_unchecked"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[rustc_const_unstable(feature = "const_pin", issue = "76654")] #[stable(feature = "pin", since = "1.33.0")] pub const unsafe fn new_unchecked(pointer: Ptr) -> Pin { @@ -1365,7 +1368,8 @@ impl Pin { /// "Malicious" implementations of `Pointer::Deref` are likewise /// ruled out by the contract of `Pin::new_unchecked`. #[stable(feature = "pin", since = "1.33.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn as_ref(&self) -> Pin<&Ptr::Target> { // SAFETY: see documentation on this function unsafe { Pin::new_unchecked(&*self.__pointer) } @@ -1409,7 +1413,8 @@ impl Pin { /// } /// ``` #[stable(feature = "pin", since = "1.33.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn as_mut(&mut self) -> Pin<&mut Ptr::Target> { // SAFETY: see documentation on this function unsafe { Pin::new_unchecked(&mut *self.__pointer) } @@ -1424,7 +1429,8 @@ impl Pin { /// `Pin::new_unchecked`. #[unstable(feature = "pin_deref_mut", issue = "86918")] #[must_use = "`self` will be dropped if the result is not used"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn as_deref_mut(self: Pin<&mut Pin>) -> Pin<&mut Ptr::Target> { // SAFETY: What we're asserting here is that going from // @@ -1474,7 +1480,8 @@ impl Pin { /// /// [subtle-details]: self#subtle-details-and-the-drop-guarantee #[stable(feature = "pin", since = "1.33.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn set(&mut self, value: Ptr::Target) where Ptr::Target: Sized, @@ -1502,7 +1509,8 @@ impl Pin { /// /// If the underlying data is [`Unpin`], [`Pin::into_inner`] should be used /// instead. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[rustc_const_unstable(feature = "const_pin", issue = "76654")] #[stable(feature = "pin_into_inner", since = "1.39.0")] pub const unsafe fn into_inner_unchecked(pin: Pin) -> Ptr { @@ -1557,7 +1565,8 @@ impl<'a, T: ?Sized> Pin<&'a T> { /// with the same lifetime as the reference it wraps. /// /// ["pinning projections"]: self#projections-and-structural-pinning - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[rustc_const_unstable(feature = "const_pin", issue = "76654")] #[stable(feature = "pin", since = "1.33.0")] @@ -1568,7 +1577,8 @@ impl<'a, T: ?Sized> Pin<&'a T> { impl<'a, T: ?Sized> Pin<&'a mut T> { /// Converts this `Pin<&mut T>` into a `Pin<&T>` with the same lifetime. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[must_use = "`self` will be dropped if the result is not used"] #[rustc_const_unstable(feature = "const_pin", issue = "76654")] #[stable(feature = "pin", since = "1.33.0")] @@ -1585,7 +1595,8 @@ impl<'a, T: ?Sized> Pin<&'a mut T> { /// that lives for as long as the borrow of the `Pin`, not the lifetime of /// the `Pin` itself. This method allows turning the `Pin` into a reference /// with the same lifetime as the original `Pin`. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "pin", since = "1.33.0")] #[rustc_const_unstable(feature = "const_pin", issue = "76654")] @@ -1606,7 +1617,8 @@ impl<'a, T: ?Sized> Pin<&'a mut T> { /// /// If the underlying data is `Unpin`, `Pin::get_mut` should be used /// instead. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "pin", since = "1.33.0")] #[rustc_const_unstable(feature = "const_pin", issue = "76654")] diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index 3b45d46b31d5e..9cfee00855382 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -58,7 +58,8 @@ impl *const T { #[stable(feature = "ptr_cast", since = "1.38.0")] #[rustc_const_stable(feature = "const_ptr_cast", since = "1.38.0")] #[rustc_diagnostic_item = "const_ptr_cast"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn cast(self) -> *const U { self as _ } @@ -109,7 +110,8 @@ impl *const T { #[stable(feature = "ptr_const_cast", since = "1.65.0")] #[rustc_const_stable(feature = "ptr_const_cast", since = "1.65.0")] #[rustc_diagnostic_item = "ptr_cast_mut"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn cast_mut(self) -> *mut T { self as _ } @@ -138,7 +140,8 @@ impl *const T { /// might change in the future (including possibly weakening this so it becomes wholly /// equivalent to `self as usize`). See the [module documentation][crate::ptr] for details. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[unstable(feature = "strict_provenance", issue = "95228")] pub fn addr(self) -> usize { // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. @@ -173,7 +176,8 @@ impl *const T { /// /// [`with_exposed_provenance`]: with_exposed_provenance #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[unstable(feature = "exposed_provenance", issue = "95228")] pub fn expose_provenance(self) -> usize { // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. @@ -388,7 +392,8 @@ impl *const T { #[stable(feature = "rust1", since = "1.0.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn offset(self, count: isize) -> *const T where @@ -409,7 +414,8 @@ impl *const T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -472,7 +478,8 @@ impl *const T { #[stable(feature = "ptr_wrapping_offset", since = "1.16.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_offset(self, count: isize) -> *const T where T: Sized, @@ -492,7 +499,8 @@ impl *const T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -533,7 +541,8 @@ impl *const T { /// ``` #[unstable(feature = "ptr_mask", issue = "98290")] #[must_use = "returns a new pointer rather than modifying its argument"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn mask(self, mask: usize) -> *const T { intrinsics::ptr_mask(self.cast::<()>(), mask).with_metadata_of(self) } @@ -642,7 +651,8 @@ impl *const T { /// /// For non-`Sized` pointees this operation considers only the data pointers, /// ignoring the metadata. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -849,7 +859,8 @@ impl *const T { #[stable(feature = "pointer_methods", since = "1.26.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn add(self, count: usize) -> Self where @@ -870,7 +881,8 @@ impl *const T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -924,7 +936,8 @@ impl *const T { #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] #[rustc_allow_const_fn_unstable(unchecked_neg)] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn sub(self, count: usize) -> Self where @@ -953,7 +966,8 @@ impl *const T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -1017,7 +1031,8 @@ impl *const T { #[stable(feature = "pointer_methods", since = "1.26.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_add(self, count: usize) -> Self where T: Sized, @@ -1036,7 +1051,8 @@ impl *const T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -1098,7 +1114,8 @@ impl *const T { #[stable(feature = "pointer_methods", since = "1.26.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_sub(self, count: usize) -> Self where T: Sized, @@ -1117,7 +1134,8 @@ impl *const T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -1569,7 +1587,8 @@ impl *const [T] { /// let slice: *const [i8] = ptr::slice_from_raw_parts(ptr::null(), 3); /// assert!(!slice.is_empty()); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "slice_ptr_len", since = "1.79.0")] #[rustc_const_stable(feature = "const_slice_ptr_len", since = "1.79.0")] pub const fn is_empty(self) -> bool { diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 08d06cad55d06..f66c6eb4476d3 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -594,7 +594,8 @@ pub unsafe fn drop_in_place(to_drop: *mut T) { /// assert!(p.is_null()); /// assert_eq!(p as usize, 0); // this pointer has the address 0 /// ``` -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] @@ -620,7 +621,8 @@ pub const fn null() -> *const T { /// assert!(p.is_null()); /// assert_eq!(p as usize, 0); // this pointer has the address 0 /// ``` -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] @@ -645,7 +647,8 @@ pub const fn null_mut() -> *mut T { /// /// This API and its claimed semantics are part of the Strict Provenance experiment, /// see the [module documentation][crate::ptr] for details. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[rustc_const_stable(feature = "stable_things_using_strict_provenance", since = "1.61.0")] #[unstable(feature = "strict_provenance", issue = "95228")] @@ -667,7 +670,8 @@ pub const fn without_provenance(addr: usize) -> *const T { /// a `T`, which means this must not be used as a "not yet initialized" /// sentinel value. Types that lazily allocate must track initialization by /// some other means. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[rustc_const_stable(feature = "stable_things_using_strict_provenance", since = "1.61.0")] #[unstable(feature = "strict_provenance", issue = "95228")] @@ -689,7 +693,8 @@ pub const fn dangling() -> *const T { /// /// This API and its claimed semantics are part of the Strict Provenance experiment, /// see the [module documentation][crate::ptr] for details. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[rustc_const_stable(feature = "stable_things_using_strict_provenance", since = "1.61.0")] #[unstable(feature = "strict_provenance", issue = "95228")] @@ -711,7 +716,8 @@ pub const fn without_provenance_mut(addr: usize) -> *mut T { /// a `T`, which means this must not be used as a "not yet initialized" /// sentinel value. Types that lazily allocate must track initialization by /// some other means. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[rustc_const_stable(feature = "stable_things_using_strict_provenance", since = "1.61.0")] #[unstable(feature = "strict_provenance", issue = "95228")] @@ -755,7 +761,8 @@ pub const fn dangling_mut() -> *mut T { /// It is unclear whether this function can be given a satisfying unambiguous specification. This /// API and its claimed semantics are part of [Exposed Provenance][self#exposed-provenance]. #[must_use] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[unstable(feature = "exposed_provenance", issue = "95228")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[allow(fuzzy_provenance_casts)] // this *is* the explicit provenance API one should use instead @@ -795,7 +802,8 @@ where /// It is unclear whether this function can be given a satisfying unambiguous specification. This /// API and its claimed semantics are part of [Exposed Provenance][self#exposed-provenance]. #[must_use] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[unstable(feature = "exposed_provenance", issue = "95228")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[allow(fuzzy_provenance_casts)] // this *is* the explicit provenance API one should use instead @@ -854,7 +862,8 @@ where /// let p = ptr::from_ref(&x); /// unsafe { p.read() }; /// ``` -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[stable(feature = "ptr_from_ref", since = "1.76.0")] #[rustc_const_stable(feature = "ptr_from_ref", since = "1.76.0")] @@ -905,7 +914,8 @@ pub const fn from_ref(r: &T) -> *const T { /// let p = ptr::from_mut(&mut x); /// unsafe { p.write(T::default()) }; /// ``` -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use] #[stable(feature = "ptr_from_ref", since = "1.76.0")] #[rustc_const_stable(feature = "ptr_from_ref", since = "1.76.0")] @@ -2136,7 +2146,8 @@ pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usiz /// assert!(!std::ptr::eq(&a[0..2], &a[1..3])); /// ``` #[stable(feature = "ptr_eq", since = "1.17.0")] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use = "pointer comparison produces a value"] #[rustc_diagnostic_item = "ptr_eq"] #[allow(ambiguous_wide_pointer_comparisons)] // it's actually clear here @@ -2162,7 +2173,8 @@ pub fn eq(a: *const T, b: *const T) -> bool { /// assert!(!ptr::eq::(whole, first)); /// ``` #[stable(feature = "ptr_addr_eq", since = "1.76.0")] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use = "pointer comparison produces a value"] pub fn addr_eq(p: *const T, q: *const U) -> bool { (p as *const ()) == (q as *const ()) @@ -2189,7 +2201,8 @@ pub fn addr_eq(p: *const T, q: *const U) -> bool { /// assert!(!ptr::fn_addr_eq(a as fn(), b as fn())); /// ``` #[unstable(feature = "ptr_fn_addr_eq", issue = "129322")] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[must_use = "function pointer comparison produces a value"] pub fn fn_addr_eq(f: T, g: U) -> bool { f.addr() == g.addr() diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 5fa3b9bf61f7f..27a9a57eb5430 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -40,7 +40,8 @@ impl *mut T { #[stable(feature = "ptr_cast", since = "1.38.0")] #[rustc_const_stable(feature = "const_ptr_cast", since = "1.38.0")] #[rustc_diagnostic_item = "ptr_cast"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn cast(self) -> *mut U { self as _ } @@ -97,7 +98,8 @@ impl *mut T { #[stable(feature = "ptr_const_cast", since = "1.65.0")] #[rustc_const_stable(feature = "ptr_const_cast", since = "1.65.0")] #[rustc_diagnostic_item = "ptr_cast_const"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn cast_const(self) -> *const T { self as _ } @@ -127,7 +129,8 @@ impl *mut T { /// might change in the future (including possibly weakening this so it becomes wholly /// equivalent to `self as usize`). See the [module documentation][crate::ptr] for details. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[unstable(feature = "strict_provenance", issue = "95228")] pub fn addr(self) -> usize { // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. @@ -161,7 +164,8 @@ impl *mut T { /// API and its claimed semantics are part of [Exposed Provenance][super#exposed-provenance]. /// /// [`with_exposed_provenance_mut`]: with_exposed_provenance_mut - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[unstable(feature = "exposed_provenance", issue = "95228")] pub fn expose_provenance(self) -> usize { // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. @@ -386,7 +390,8 @@ impl *mut T { #[stable(feature = "rust1", since = "1.0.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn offset(self, count: isize) -> *mut T where @@ -409,7 +414,8 @@ impl *mut T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -469,7 +475,8 @@ impl *mut T { #[stable(feature = "ptr_wrapping_offset", since = "1.16.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_offset(self, count: isize) -> *mut T where T: Sized, @@ -489,7 +496,8 @@ impl *mut T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -533,7 +541,8 @@ impl *mut T { /// ``` #[unstable(feature = "ptr_mask", issue = "98290")] #[must_use = "returns a new pointer rather than modifying its argument"] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub fn mask(self, mask: usize) -> *mut T { intrinsics::ptr_mask(self.cast::<()>(), mask).cast_mut().with_metadata_of(self) } @@ -786,7 +795,8 @@ impl *mut T { /// ``` #[stable(feature = "ptr_offset_from", since = "1.47.0")] #[rustc_const_stable(feature = "const_ptr_offset_from", since = "1.65.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn offset_from(self, origin: *const T) -> isize where @@ -805,7 +815,8 @@ impl *mut T { /// /// For non-`Sized` pointees this operation considers only the data pointers, /// ignoring the metadata. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -930,7 +941,8 @@ impl *mut T { #[stable(feature = "pointer_methods", since = "1.26.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn add(self, count: usize) -> Self where @@ -951,7 +963,8 @@ impl *mut T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -1005,7 +1018,8 @@ impl *mut T { #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] #[rustc_allow_const_fn_unstable(unchecked_neg)] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn sub(self, count: usize) -> Self where @@ -1034,7 +1048,8 @@ impl *mut T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -1096,7 +1111,8 @@ impl *mut T { #[stable(feature = "pointer_methods", since = "1.26.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_add(self, count: usize) -> Self where T: Sized, @@ -1115,7 +1131,8 @@ impl *mut T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -1175,7 +1192,8 @@ impl *mut T { #[stable(feature = "pointer_methods", since = "1.26.0")] #[must_use = "returns a new pointer rather than modifying its argument"] #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn wrapping_sub(self, count: usize) -> Self where T: Sized, @@ -1194,7 +1212,8 @@ impl *mut T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "pointer_byte_offsets", since = "1.75.0")] #[rustc_const_stable(feature = "const_pointer_byte_offsets", since = "1.75.0")] #[rustc_allow_const_fn_unstable(set_ptr_value)] @@ -1210,7 +1229,8 @@ impl *mut T { /// [`ptr::read`]: crate::ptr::read() #[stable(feature = "pointer_methods", since = "1.26.0")] #[rustc_const_stable(feature = "const_ptr_read", since = "1.71.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn read(self) -> T where @@ -1231,7 +1251,8 @@ impl *mut T { /// /// [`ptr::read_volatile`]: crate::ptr::read_volatile() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn read_volatile(self) -> T where @@ -1251,7 +1272,8 @@ impl *mut T { /// [`ptr::read_unaligned`]: crate::ptr::read_unaligned() #[stable(feature = "pointer_methods", since = "1.26.0")] #[rustc_const_stable(feature = "const_ptr_read", since = "1.71.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn read_unaligned(self) -> T where @@ -1271,7 +1293,8 @@ impl *mut T { /// [`ptr::copy`]: crate::ptr::copy() #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn copy_to(self, dest: *mut T, count: usize) where @@ -1291,7 +1314,8 @@ impl *mut T { /// [`ptr::copy_nonoverlapping`]: crate::ptr::copy_nonoverlapping() #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn copy_to_nonoverlapping(self, dest: *mut T, count: usize) where @@ -1311,7 +1335,8 @@ impl *mut T { /// [`ptr::copy`]: crate::ptr::copy() #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn copy_from(self, src: *const T, count: usize) where @@ -1331,7 +1356,8 @@ impl *mut T { /// [`ptr::copy_nonoverlapping`]: crate::ptr::copy_nonoverlapping() #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn copy_from_nonoverlapping(self, src: *const T, count: usize) where @@ -1347,7 +1373,8 @@ impl *mut T { /// /// [`ptr::drop_in_place`]: crate::ptr::drop_in_place() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub unsafe fn drop_in_place(self) { // SAFETY: the caller must uphold the safety contract for `drop_in_place`. unsafe { drop_in_place(self) } @@ -1361,7 +1388,8 @@ impl *mut T { /// [`ptr::write`]: crate::ptr::write() #[stable(feature = "pointer_methods", since = "1.26.0")] #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn write(self, val: T) where @@ -1380,7 +1408,8 @@ impl *mut T { #[doc(alias = "memset")] #[stable(feature = "pointer_methods", since = "1.26.0")] #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn write_bytes(self, val: u8, count: usize) where @@ -1401,7 +1430,8 @@ impl *mut T { /// /// [`ptr::write_volatile`]: crate::ptr::write_volatile() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn write_volatile(self, val: T) where @@ -1421,7 +1451,8 @@ impl *mut T { /// [`ptr::write_unaligned`]: crate::ptr::write_unaligned() #[stable(feature = "pointer_methods", since = "1.26.0")] #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn write_unaligned(self, val: T) where @@ -1438,7 +1469,8 @@ impl *mut T { /// /// [`ptr::replace`]: crate::ptr::replace() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub unsafe fn replace(self, src: T) -> T where T: Sized, @@ -1456,7 +1488,8 @@ impl *mut T { /// [`ptr::swap`]: crate::ptr::swap() #[stable(feature = "pointer_methods", since = "1.26.0")] #[rustc_const_unstable(feature = "const_swap", issue = "83163")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const unsafe fn swap(self, with: *mut T) where T: Sized, @@ -1801,7 +1834,8 @@ impl *mut [T] { /// let slice: *mut [i8] = ptr::slice_from_raw_parts_mut(ptr::null_mut(), 3); /// assert_eq!(slice.len(), 3); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "slice_ptr_len", since = "1.79.0")] #[rustc_const_stable(feature = "const_slice_ptr_len", since = "1.79.0")] #[rustc_allow_const_fn_unstable(ptr_metadata)] @@ -1819,7 +1853,8 @@ impl *mut [T] { /// let slice: *mut [i8] = ptr::slice_from_raw_parts_mut(ptr::null_mut(), 3); /// assert!(!slice.is_empty()); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "slice_ptr_len", since = "1.79.0")] #[rustc_const_stable(feature = "const_slice_ptr_len", since = "1.79.0")] pub const fn is_empty(self) -> bool { @@ -1866,7 +1901,8 @@ impl *mut [T] { /// assert_eq!(&*right, [3, 0, 5, 6]); /// } /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] #[unstable(feature = "raw_slice_split", issue = "95595")] pub unsafe fn split_at_mut(self, mid: usize) -> (*mut [T], *mut [T]) { @@ -1910,7 +1946,8 @@ impl *mut [T] { /// } /// assert_eq!(v, [1, 2, 3, 4, 5, 6]); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[unstable(feature = "raw_slice_split", issue = "95595")] pub unsafe fn split_at_mut_unchecked(self, mid: usize) -> (*mut [T], *mut [T]) { let len = self.len(); @@ -1937,7 +1974,8 @@ impl *mut [T] { /// let slice: *mut [i8] = ptr::slice_from_raw_parts_mut(ptr::null_mut(), 3); /// assert_eq!(slice.as_mut_ptr(), ptr::null_mut()); /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[unstable(feature = "slice_ptr_get", issue = "74265")] #[rustc_const_unstable(feature = "slice_ptr_get", issue = "74265")] pub const fn as_mut_ptr(self) -> *mut T { @@ -1965,7 +2003,8 @@ impl *mut [T] { /// } /// ``` #[unstable(feature = "slice_ptr_get", issue = "74265")] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub unsafe fn get_unchecked_mut(self, index: I) -> *mut I::Output where I: SliceIndex<[T]>, @@ -2125,7 +2164,8 @@ impl *mut [T; N] { // Equality for pointers #[stable(feature = "rust1", since = "1.0.0")] impl PartialEq for *mut T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[allow(ambiguous_wide_pointer_comparisons)] fn eq(&self, other: &*mut T) -> bool { *self == *other @@ -2152,31 +2192,36 @@ impl Ord for *mut T { #[stable(feature = "rust1", since = "1.0.0")] impl PartialOrd for *mut T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[allow(ambiguous_wide_pointer_comparisons)] fn partial_cmp(&self, other: &*mut T) -> Option { Some(self.cmp(other)) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[allow(ambiguous_wide_pointer_comparisons)] fn lt(&self, other: &*mut T) -> bool { *self < *other } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[allow(ambiguous_wide_pointer_comparisons)] fn le(&self, other: &*mut T) -> bool { *self <= *other } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[allow(ambiguous_wide_pointer_comparisons)] fn gt(&self, other: &*mut T) -> bool { *self > *other } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[allow(ambiguous_wide_pointer_comparisons)] fn ge(&self, other: &*mut T) -> bool { *self >= *other diff --git a/library/core/src/ptr/non_null.rs b/library/core/src/ptr/non_null.rs index 673acc2972fe4..de07e8df4f969 100644 --- a/library/core/src/ptr/non_null.rs +++ b/library/core/src/ptr/non_null.rs @@ -324,7 +324,8 @@ impl NonNull { #[rustc_const_stable(feature = "const_nonnull_as_ptr", since = "1.32.0")] #[rustc_never_returns_null_ptr] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn as_ptr(self) -> *mut T { self.pointer as *mut T } @@ -358,7 +359,8 @@ impl NonNull { #[stable(feature = "nonnull", since = "1.25.0")] #[rustc_const_stable(feature = "const_nonnull_as_ref", since = "1.73.0")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const unsafe fn as_ref<'a>(&self) -> &'a T { // SAFETY: the caller must guarantee that `self` meets all the // requirements for a reference. @@ -396,7 +398,8 @@ impl NonNull { #[stable(feature = "nonnull", since = "1.25.0")] #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const unsafe fn as_mut<'a>(&mut self) -> &'a mut T { // SAFETY: the caller must guarantee that `self` meets all the // requirements for a mutable reference. @@ -462,7 +465,8 @@ impl NonNull { /// println!("{}", ptr.offset(2).read()); /// } /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[must_use = "returns a new pointer rather than modifying its argument"] #[stable(feature = "non_null_convenience", since = "1.80.0")] @@ -489,7 +493,8 @@ impl NonNull { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] #[rustc_const_stable(feature = "non_null_convenience", since = "1.80.0")] @@ -538,7 +543,8 @@ impl NonNull { /// println!("{}", ptr.add(2).read() as char); /// } /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[must_use = "returns a new pointer rather than modifying its argument"] #[stable(feature = "non_null_convenience", since = "1.80.0")] @@ -565,7 +571,8 @@ impl NonNull { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_allow_const_fn_unstable(set_ptr_value)] #[stable(feature = "non_null_convenience", since = "1.80.0")] @@ -616,7 +623,8 @@ impl NonNull { /// println!("{}", end.sub(2).read() as char); /// } /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[must_use = "returns a new pointer rather than modifying its argument"] #[stable(feature = "non_null_convenience", since = "1.80.0")] @@ -649,7 +657,8 @@ impl NonNull { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_allow_const_fn_unstable(set_ptr_value)] #[stable(feature = "non_null_convenience", since = "1.80.0")] @@ -770,7 +779,8 @@ impl NonNull { /// /// For non-`Sized` pointees this operation considers only the data pointers, /// ignoring the metadata. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] #[rustc_const_stable(feature = "non_null_convenience", since = "1.80.0")] @@ -921,7 +931,8 @@ impl NonNull { /// See [`ptr::copy`] for safety concerns and examples. /// /// [`ptr::copy`]: crate::ptr::copy() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] @@ -941,7 +952,8 @@ impl NonNull { /// See [`ptr::copy_nonoverlapping`] for safety concerns and examples. /// /// [`ptr::copy_nonoverlapping`]: crate::ptr::copy_nonoverlapping() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] @@ -961,7 +973,8 @@ impl NonNull { /// See [`ptr::copy`] for safety concerns and examples. /// /// [`ptr::copy`]: crate::ptr::copy() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] @@ -981,7 +994,8 @@ impl NonNull { /// See [`ptr::copy_nonoverlapping`] for safety concerns and examples. /// /// [`ptr::copy_nonoverlapping`]: crate::ptr::copy_nonoverlapping() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] @@ -998,7 +1012,8 @@ impl NonNull { /// See [`ptr::drop_in_place`] for safety concerns and examples. /// /// [`ptr::drop_in_place`]: crate::ptr::drop_in_place() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "non_null_convenience", since = "1.80.0")] pub unsafe fn drop_in_place(self) { // SAFETY: the caller must uphold the safety contract for `drop_in_place`. @@ -1011,7 +1026,8 @@ impl NonNull { /// See [`ptr::write`] for safety concerns and examples. /// /// [`ptr::write`]: crate::ptr::write() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] @@ -1029,7 +1045,8 @@ impl NonNull { /// See [`ptr::write_bytes`] for safety concerns and examples. /// /// [`ptr::write_bytes`]: crate::ptr::write_bytes() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[doc(alias = "memset")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] @@ -1052,7 +1069,8 @@ impl NonNull { /// See [`ptr::write_volatile`] for safety concerns and examples. /// /// [`ptr::write_volatile`]: crate::ptr::write_volatile() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] pub unsafe fn write_volatile(self, val: T) @@ -1071,7 +1089,8 @@ impl NonNull { /// See [`ptr::write_unaligned`] for safety concerns and examples. /// /// [`ptr::write_unaligned`]: crate::ptr::write_unaligned() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[stable(feature = "non_null_convenience", since = "1.80.0")] #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] @@ -1089,7 +1108,8 @@ impl NonNull { /// See [`ptr::replace`] for safety concerns and examples. /// /// [`ptr::replace`]: crate::ptr::replace() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "non_null_convenience", since = "1.80.0")] pub unsafe fn replace(self, src: T) -> T where @@ -1106,7 +1126,8 @@ impl NonNull { /// See [`ptr::swap`] for safety concerns and examples. /// /// [`ptr::swap`]: crate::ptr::swap() - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[stable(feature = "non_null_convenience", since = "1.80.0")] #[rustc_const_unstable(feature = "const_swap", issue = "83163")] pub const unsafe fn swap(self, with: NonNull) @@ -1669,7 +1690,8 @@ impl NonNull<[T]> { #[stable(feature = "nonnull", since = "1.25.0")] impl Clone for NonNull { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn clone(&self) -> Self { *self } diff --git a/library/core/src/result.rs b/library/core/src/result.rs index 9edd58259ba0f..b7c54b3fb6836 100644 --- a/library/core/src/result.rs +++ b/library/core/src/result.rs @@ -1091,7 +1091,8 @@ impl Result { /// let x: Result = Err("emergency failure"); /// x.unwrap(); // panics with `emergency failure` /// ``` - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[track_caller] #[stable(feature = "rust1", since = "1.0.0")] pub fn unwrap(self) -> T diff --git a/library/core/src/slice/index.rs b/library/core/src/slice/index.rs index bc8571c8503e9..2828f3be206a1 100644 --- a/library/core/src/slice/index.rs +++ b/library/core/src/slice/index.rs @@ -11,7 +11,8 @@ where { type Output = I::Output; - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn index(&self, index: I) -> &I::Output { index.index(self) } @@ -22,7 +23,8 @@ impl ops::IndexMut for [T] where I: SliceIndex<[T]>, { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn index_mut(&mut self, index: I) -> &mut I::Output { index.index_mut(self) } @@ -110,21 +112,24 @@ const fn slice_end_index_overflow_fail() -> ! { // Both the safe and unsafe public methods share these helpers, // which use intrinsics directly to get *no* extra checks. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] const unsafe fn get_noubcheck(ptr: *const [T], index: usize) -> *const T { let ptr = ptr as *const T; // SAFETY: The caller already checked these preconditions unsafe { crate::intrinsics::offset(ptr, index) } } -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] const unsafe fn get_mut_noubcheck(ptr: *mut [T], index: usize) -> *mut T { let ptr = ptr as *mut T; // SAFETY: The caller already checked these preconditions unsafe { crate::intrinsics::offset(ptr, index) } } -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] const unsafe fn get_offset_len_noubcheck( ptr: *const [T], offset: usize, @@ -135,7 +140,8 @@ const unsafe fn get_offset_len_noubcheck( crate::intrinsics::aggregate_raw_ptr(ptr, len) } -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] const unsafe fn get_offset_len_mut_noubcheck( ptr: *mut [T], offset: usize, @@ -455,7 +461,8 @@ unsafe impl SliceIndex<[T]> for ops::Range { } } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn index(self, slice: &[T]) -> &[T] { // Using checked_sub is a safe way to get `SubUnchecked` in MIR let Some(new_len) = usize::checked_sub(self.end, self.start) else { @@ -507,7 +514,8 @@ unsafe impl SliceIndex<[T]> for range::Range { unsafe { ops::Range::from(self).get_unchecked_mut(slice) } } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn index(self, slice: &[T]) -> &[T] { ops::Range::from(self).index(slice) } @@ -545,7 +553,8 @@ unsafe impl SliceIndex<[T]> for ops::RangeTo { unsafe { (0..self.end).get_unchecked_mut(slice) } } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn index(self, slice: &[T]) -> &[T] { (0..self.end).index(slice) } diff --git a/library/core/src/slice/iter/macros.rs b/library/core/src/slice/iter/macros.rs index c2a3819464410..013f2c51a6e6c 100644 --- a/library/core/src/slice/iter/macros.rs +++ b/library/core/src/slice/iter/macros.rs @@ -84,7 +84,7 @@ macro_rules! iterator { } // Helper function for creating a slice from the iterator. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn make_slice(&self) -> &'a [T] { // SAFETY: the iterator was created from a slice with pointer // `self.ptr` and length `len!(self)`. This guarantees that all @@ -95,7 +95,7 @@ macro_rules! iterator { // Helper function for moving the start of the iterator forwards by `offset` elements, // returning the old start. // Unsafe because the offset must not exceed `self.len()`. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] unsafe fn post_inc_start(&mut self, offset: usize) -> NonNull { let old = self.ptr; @@ -114,7 +114,7 @@ macro_rules! iterator { // Helper function for moving the end of the iterator backwards by `offset` elements, // returning the new end. // Unsafe because the offset must not exceed `self.len()`. - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] unsafe fn pre_dec_end(&mut self, offset: usize) -> NonNull { if_zst!(mut self, // SAFETY: By our precondition, `offset` can be at most the @@ -137,12 +137,12 @@ macro_rules! iterator { #[stable(feature = "rust1", since = "1.0.0")] impl ExactSizeIterator for $name<'_, T> { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn len(&self) -> usize { len!(self) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))] fn is_empty(&self) -> bool { is_empty!(self) } diff --git a/library/core/src/slice/mod.rs b/library/core/src/slice/mod.rs index cdcca0eee888d..83bc68cea5860 100644 --- a/library/core/src/slice/mod.rs +++ b/library/core/src/slice/mod.rs @@ -736,7 +736,8 @@ impl [T] { #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_slice_as_ptr", since = "1.32.0")] #[rustc_never_returns_null_ptr] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[must_use] pub const fn as_ptr(&self) -> *const T { self as *const [T] as *const T @@ -767,7 +768,8 @@ impl [T] { #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] #[rustc_allow_const_fn_unstable(const_mut_refs)] #[rustc_never_returns_null_ptr] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[must_use] pub const fn as_mut_ptr(&mut self) -> *mut T { self as *mut [T] as *mut T diff --git a/library/core/src/slice/sort/shared/mod.rs b/library/core/src/slice/sort/shared/mod.rs index ad1171bfc6a0a..e4d2e767c71a7 100644 --- a/library/core/src/slice/sort/shared/mod.rs +++ b/library/core/src/slice/sort/shared/mod.rs @@ -14,7 +14,8 @@ impl FreezeMarker for T {} /// /// Returns the length of the run, and a bool that is false when the run /// is ascending, and true if the run strictly descending. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] pub(crate) fn find_existing_run bool>( v: &[T], is_less: &mut F, diff --git a/library/core/src/slice/sort/shared/pivot.rs b/library/core/src/slice/sort/shared/pivot.rs index 255a1eb6c88a8..36d6a547dcd2f 100644 --- a/library/core/src/slice/sort/shared/pivot.rs +++ b/library/core/src/slice/sort/shared/pivot.rs @@ -69,7 +69,8 @@ unsafe fn median3_rec bool>( /// Calculates the median of 3 elements. /// /// SAFETY: a, b, c must be valid initialized elements. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] fn median3 bool>(a: &T, b: &T, c: &T, is_less: &mut F) -> *const T { // Compiler tends to make this branchless when sensible, and avoids the // third comparison when not. diff --git a/library/core/src/slice/sort/shared/smallsort.rs b/library/core/src/slice/sort/shared/smallsort.rs index fae628a7c1474..1f48b4b075ea2 100644 --- a/library/core/src/slice/sort/shared/smallsort.rs +++ b/library/core/src/slice/sort/shared/smallsort.rs @@ -28,13 +28,15 @@ pub(crate) trait StableSmallSortTypeImpl: Sized { } impl StableSmallSortTypeImpl for T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] default fn small_sort_threshold() -> usize { // Optimal number of comparisons, and good perf. SMALL_SORT_FALLBACK_THRESHOLD } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] default fn small_sort bool>( v: &mut [T], _scratch: &mut [MaybeUninit], @@ -47,12 +49,14 @@ impl StableSmallSortTypeImpl for T { } impl StableSmallSortTypeImpl for T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn small_sort_threshold() -> usize { SMALL_SORT_GENERAL_THRESHOLD } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn small_sort bool>( v: &mut [T], scratch: &mut [MaybeUninit], @@ -73,12 +77,14 @@ pub(crate) trait UnstableSmallSortTypeImpl: Sized { } impl UnstableSmallSortTypeImpl for T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] default fn small_sort_threshold() -> usize { SMALL_SORT_FALLBACK_THRESHOLD } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] default fn small_sort(v: &mut [T], is_less: &mut F) where F: FnMut(&T, &T) -> bool, @@ -88,12 +94,14 @@ impl UnstableSmallSortTypeImpl for T { } impl UnstableSmallSortTypeImpl for T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn small_sort_threshold() -> usize { ::small_sort_threshold() } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn small_sort(v: &mut [T], is_less: &mut F) where F: FnMut(&T, &T) -> bool, @@ -111,7 +119,8 @@ pub(crate) trait UnstableSmallSortFreezeTypeImpl: Sized + FreezeMarker { } impl UnstableSmallSortFreezeTypeImpl for T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] default fn small_sort_threshold() -> usize { if (mem::size_of::() * SMALL_SORT_GENERAL_SCRATCH_LEN) <= MAX_STACK_ARRAY_SIZE { SMALL_SORT_GENERAL_THRESHOLD @@ -120,7 +129,8 @@ impl UnstableSmallSortFreezeTypeImpl for T { } } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] default fn small_sort(v: &mut [T], is_less: &mut F) where F: FnMut(&T, &T) -> bool, @@ -140,7 +150,8 @@ trait CopyMarker {} impl CopyMarker for T {} impl UnstableSmallSortFreezeTypeImpl for T { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn small_sort_threshold() -> usize { if has_efficient_in_place_swap::() && (mem::size_of::() * SMALL_SORT_NETWORK_SCRATCH_LEN) <= MAX_STACK_ARRAY_SIZE @@ -153,7 +164,8 @@ impl UnstableSmallSortFreezeTypeImpl for T { } } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn small_sort(v: &mut [T], is_less: &mut F) where F: FnMut(&T, &T) -> bool, @@ -651,7 +663,8 @@ pub unsafe fn sort4_stable bool>( ptr::copy_nonoverlapping(max, dst.add(3), 1); } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn select(cond: bool, if_true: *const T, if_false: *const T) -> *const T { if cond { if_true } else { if_false } } @@ -679,7 +692,8 @@ unsafe fn sort8_stable bool>( } } -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] unsafe fn merge_up bool>( mut left_src: *const T, mut right_src: *const T, @@ -712,7 +726,8 @@ unsafe fn merge_up bool>( (left_src, right_src, dst) } -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] unsafe fn merge_down bool>( mut left_src: *const T, mut right_src: *const T, diff --git a/library/core/src/slice/sort/stable/drift.rs b/library/core/src/slice/sort/stable/drift.rs index 644e75a4581e9..3249fc2382c9e 100644 --- a/library/core/src/slice/sort/stable/drift.rs +++ b/library/core/src/slice/sort/stable/drift.rs @@ -152,7 +152,8 @@ pub fn sort bool>( // x < 2^63 + 2n // So as long as n < 2^62 we find that x < 2^64, meaning our operations do not // overflow. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] fn merge_tree_scale_factor(n: usize) -> u64 { if usize::BITS > u64::BITS { panic!("Platform not supported"); @@ -163,7 +164,8 @@ fn merge_tree_scale_factor(n: usize) -> u64 { // Note: merge_tree_depth output is < 64 when left < right as f*x and f*y must // differ in some bit, and is <= 64 always. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] fn merge_tree_depth(left: usize, mid: usize, right: usize, scale_factor: u64) -> u8 { let x = left as u64 + mid as u64; let y = mid as u64 + right as u64; @@ -187,7 +189,8 @@ fn sqrt_approx(n: usize) -> usize { } // Lazy logical runs as in Glidesort. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] fn logical_merge bool>( v: &mut [T], scratch: &mut [MaybeUninit], @@ -276,22 +279,26 @@ fn stable_quicksort bool>( struct DriftsortRun(usize); impl DriftsortRun { - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn new_sorted(length: usize) -> Self { Self((length << 1) | 1) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn new_unsorted(length: usize) -> Self { Self(length << 1) } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn sorted(self) -> bool { self.0 & 1 == 1 } - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] fn len(self) -> usize { self.0 >> 1 } diff --git a/library/core/src/slice/sort/stable/mod.rs b/library/core/src/slice/sort/stable/mod.rs index a383b0f589ccf..9474dc3d8b9cc 100644 --- a/library/core/src/slice/sort/stable/mod.rs +++ b/library/core/src/slice/sort/stable/mod.rs @@ -16,7 +16,8 @@ pub(crate) mod quicksort; /// /// Upholds all safety properties outlined here: /// -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] pub fn sort bool, BufT: BufGuard>(v: &mut [T], is_less: &mut F) { // Arrays of zero-sized types are always all-equal, and thus sorted. if T::IS_ZST { diff --git a/library/core/src/slice/sort/unstable/mod.rs b/library/core/src/slice/sort/unstable/mod.rs index 932e01f4401e5..b5f15c6bcde02 100644 --- a/library/core/src/slice/sort/unstable/mod.rs +++ b/library/core/src/slice/sort/unstable/mod.rs @@ -14,7 +14,8 @@ pub(crate) mod quicksort; /// /// Upholds all safety properties outlined here: /// -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] pub fn sort bool>(v: &mut [T], is_less: &mut F) { // Arrays of zero-sized types are always all-equal, and thus sorted. if T::IS_ZST { diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs index 712bf011c273e..0e1df1ef6ebae 100644 --- a/library/core/src/str/mod.rs +++ b/library/core/src/str/mod.rs @@ -295,7 +295,8 @@ impl str { #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "str_as_bytes", since = "1.39.0")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] #[allow(unused_attributes)] pub const fn as_bytes(&self) -> &[u8] { // SAFETY: const sound because we transmute two types with the same layout @@ -341,7 +342,8 @@ impl str { #[stable(feature = "str_mut_extras", since = "1.20.0")] #[rustc_const_unstable(feature = "const_str_as_mut", issue = "130086")] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const unsafe fn as_bytes_mut(&mut self) -> &mut [u8] { // SAFETY: the cast from `&str` to `&[u8]` is safe since `str` // has the same layout as `&[u8]` (only std can make this guarantee). @@ -371,7 +373,8 @@ impl str { #[rustc_const_stable(feature = "rustc_str_as_ptr", since = "1.32.0")] #[rustc_never_returns_null_ptr] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn as_ptr(&self) -> *const u8 { self as *const str as *const u8 } @@ -388,7 +391,8 @@ impl str { #[rustc_const_unstable(feature = "const_str_as_mut", issue = "130086")] #[rustc_never_returns_null_ptr] #[must_use] - #[inline(always)] + #[cfg_attr(bootstrap, inline(always))] + #[cfg_attr(not(bootstrap), inline(usually))] pub const fn as_mut_ptr(&mut self) -> *mut u8 { self as *mut str as *mut u8 } diff --git a/library/core/src/str/validations.rs b/library/core/src/str/validations.rs index cca8ff74dda8b..452572aa41cb3 100644 --- a/library/core/src/str/validations.rs +++ b/library/core/src/str/validations.rs @@ -121,7 +121,8 @@ const fn contains_nonascii(x: usize) -> bool { /// Walks through `v` checking that it's a valid UTF-8 sequence, /// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`. -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] #[rustc_const_unstable(feature = "str_internals", issue = "none")] pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { let mut index = 0; diff --git a/library/core/src/unicode/unicode_data.rs b/library/core/src/unicode/unicode_data.rs index db2e3ddd754f5..0196cd75dc313 100644 --- a/library/core/src/unicode/unicode_data.rs +++ b/library/core/src/unicode/unicode_data.rs @@ -1,7 +1,8 @@ ///! This file is generated by src/tools/unicode-table-generator; do not edit manually! #[rustc_const_unstable(feature = "const_unicode_case_lookup", issue = "101400")] -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] const fn bitset_search< const N: usize, const CHUNK_SIZE: usize, @@ -56,7 +57,8 @@ fn decode_length(short_offset_run_header: u32) -> usize { (short_offset_run_header >> 21) as usize } -#[inline(always)] +#[cfg_attr(bootstrap, inline(always))] +#[cfg_attr(not(bootstrap), inline(usually))] fn skip_search( needle: u32, short_offset_runs: &[u32; SOR],