mir-opt: Replace clone on primitives with copy

compiler/rustc_middle/src/mir/mod.rs

@@ -1915,6 +1915,27 @@ impl<'tcx> Place<'tcx> {
             (base, proj)
         })
     }
+
+    /// Generates a new place by appending `more_projections` to the existing ones
+    /// and interning the result.
+    pub fn project_deeper(self, more_projections: &[PlaceElem<'tcx>], tcx: TyCtxt<'tcx>) -> Self {
+        if more_projections.is_empty() {
+            return self;
+        }
+
+        let mut v: Vec<PlaceElem<'tcx>>;
+
+        let new_projections = if self.projection.is_empty() {
+            more_projections
+        } else {
+            v = Vec::with_capacity(self.projection.len() + more_projections.len());
+            v.extend(self.projection);
+            v.extend(more_projections);
+            &v
+        };
+
+        Place { local: self.local, projection: tcx.intern_place_elems(new_projections) }
+    }
 }
 
 impl From<Local> for Place<'_> {
@@ -2187,6 +2208,15 @@ impl<'tcx> Operand<'tcx> {
             Operand::Copy(_) | Operand::Move(_) => None,
         }
     }
+
+    /// Gets the `ty::FnDef` from an operand if it's a constant function item.
+    ///
+    /// While this is unlikely in general, it's the normal case of what you'll
+    /// find as the `func` in a [`TerminatorKind::Call`].
+    pub fn const_fn_def(&self) -> Option<(DefId, SubstsRef<'tcx>)> {
+        let const_ty = self.constant()?.literal.ty();
+        if let ty::FnDef(def_id, substs) = *const_ty.kind() { Some((def_id, substs)) } else { None }
+    }
 }
 
 ///////////////////////////////////////////////////////////////////////////

compiler/rustc_middle/src/ty/sty.rs

@@ -2371,6 +2371,57 @@ impl<'tcx> Ty<'tcx> {
             }
         }
     }
+
+    /// Fast path helper for primitives which are always `Copy` and which
+    /// have a side-effect-free `Clone` impl.
+    ///
+    /// Returning true means the type is known to be pure and `Copy+Clone`.
+    /// Returning `false` means nothing -- could be `Copy`, might not be.
+    ///
+    /// This is mostly useful for optimizations, as there are the types
+    /// on which we can replace cloning with dereferencing.
+    pub fn is_trivially_pure_clone_copy(self) -> bool {
+        match self.kind() {
+            ty::Bool | ty::Char | ty::Never => true,
+
+            // These aren't even `Clone`
+            ty::Str | ty::Slice(..) | ty::Foreign(..) | ty::Dynamic(..) => false,
+
+            ty::Int(..) | ty::Uint(..) | ty::Float(..) => true,
+
+            // The voldemort ZSTs are fine.
+            ty::FnDef(..) => true,
+
+            ty::Array(element_ty, _len) => element_ty.is_trivially_pure_clone_copy(),
+
+            // A 100-tuple isn't "trivial", so doing this only for reasonable sizes.
+            ty::Tuple(field_tys) => {
+                field_tys.len() <= 3 && field_tys.iter().all(Self::is_trivially_pure_clone_copy)
+            }
+
+            // Sometimes traits aren't implemented for every ABI or arity,
+            // because we can't be generic over everything yet.
+            ty::FnPtr(..) => false,
+
+            // Definitely absolutely not copy.
+            ty::Ref(_, _, hir::Mutability::Mut) => false,
+
+            // Thin pointers & thin shared references are pure-clone-copy, but for
+            // anything with custom metadata it might be more complicated.
+            ty::Ref(_, _, hir::Mutability::Not) | ty::RawPtr(..) => false,
+
+            ty::Generator(..) | ty::GeneratorWitness(..) => false,
+
+            // Might be, but not "trivial" so just giving the safe answer.
+            ty::Adt(..) | ty::Closure(..) | ty::Opaque(..) => false,
+
+            ty::Projection(..) | ty::Param(..) | ty::Infer(..) | ty::Error(..) => false,
+
+            ty::Bound(..) | ty::Placeholder(..) => {
+                bug!("`is_trivially_pure_clone_copy` applied to unexpected type: {:?}", self);
+            }
+        }
+    }
 }
 
 /// Extra information about why we ended up with a particular variance.

compiler/rustc_middle/src/ty/util.rs

@@ -704,7 +704,7 @@ impl<'tcx> Ty<'tcx> {
         tcx_at: TyCtxtAt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> bool {
-        tcx_at.is_copy_raw(param_env.and(self))
+        self.is_trivially_pure_clone_copy() || tcx_at.is_copy_raw(param_env.and(self))
     }
 
     /// Checks whether values of this type `T` have a size known at

compiler/rustc_mir_transform/src/instcombine.rs

@@ -4,7 +4,7 @@ use crate::MirPass;
 use rustc_hir::Mutability;
 use rustc_middle::mir::{
     BinOp, Body, Constant, LocalDecls, Operand, Place, ProjectionElem, Rvalue, SourceInfo,
-    StatementKind, UnOp,
+    Statement, StatementKind, Terminator, TerminatorKind, UnOp,
 };
 use rustc_middle::ty::{self, TyCtxt};
 
@@ -29,6 +29,11 @@ impl<'tcx> MirPass<'tcx> for InstCombine {
                     _ => {}
                 }
             }
+
+            ctx.combine_primitive_clone(
+                &mut block.terminator.as_mut().unwrap(),
+                &mut block.statements,
+            );
         }
     }
 }
@@ -130,4 +135,70 @@ impl<'tcx> InstCombineContext<'tcx, '_> {
             }
         }
     }
+
+    fn combine_primitive_clone(
+        &self,
+        terminator: &mut Terminator<'tcx>,
+        statements: &mut Vec<Statement<'tcx>>,
+    ) {
+        let TerminatorKind::Call { func, args, destination, .. } = &mut terminator.kind
+        else { return };
+
+        // It's definitely not a clone if there are multiple arguments
+        if args.len() != 1 {
+            return;
+        }
+
+        let Some((destination_place, destination_block)) = *destination
+        else { return };
+
+        // Only bother looking more if it's easy to know what we're calling
+        let Some((fn_def_id, fn_substs)) = func.const_fn_def()
+        else { return };
+
+        // Clone needs one subst, so we can cheaply rule out other stuff
+        if fn_substs.len() != 1 {
+            return;
+        }
+
+        // These types are easily available from locals, so check that before
+        // doing DefId lookups to figure out what we're actually calling.
+        let arg_ty = args[0].ty(self.local_decls, self.tcx);
+
+        let ty::Ref(_region, inner_ty, Mutability::Not) = *arg_ty.kind()
+        else { return };
+
+        if !inner_ty.is_trivially_pure_clone_copy() {
+            return;
+        }
+
+        let trait_def_id = self.tcx.trait_of_item(fn_def_id);
+        if trait_def_id.is_none() || trait_def_id != self.tcx.lang_items().clone_trait() {
+            return;
+        }
+
+        if !self.tcx.consider_optimizing(|| {
+            format!(
+                "InstCombine - Call: {:?} SourceInfo: {:?}",
+                (fn_def_id, fn_substs),
+                terminator.source_info
+            )
+        }) {
+            return;
+        }
+
+        let Some(arg_place) = args.pop().unwrap().place()
+        else { return };
+
+        statements.push(Statement {
+            source_info: terminator.source_info,
+            kind: StatementKind::Assign(box (
+                destination_place,
+                Rvalue::Use(Operand::Copy(
+                    arg_place.project_deeper(&[ProjectionElem::Deref], self.tcx),
+                )),
+            )),
+        });
+        terminator.kind = TerminatorKind::Goto { target: destination_block };
+    }
 }

src/test/assembly/sparc-struct-abi.rs

@@ -13,6 +13,7 @@
 pub trait Sized {}
 #[lang = "copy"]
 pub trait Copy {}
+impl Copy for f32 {}
 
 #[repr(C)]
 pub struct Franta {

src/test/assembly/target-feature-multiple.rs

@@ -23,6 +23,7 @@
 trait Sized {}
 #[lang = "copy"]
 trait Copy {}
+impl Copy for u32 {}
 
 // Use of these requires target features to be enabled
 extern "unadjusted" {

src/test/codegen/abi-sysv64.rs

@@ -13,6 +13,7 @@
 trait Sized {}
 #[lang = "copy"]
 trait Copy {}
+impl Copy for i64 {}
 
 // CHECK: define x86_64_sysvcc i64 @has_sysv64_abi
 #[no_mangle]

src/test/codegen/abi-x86-interrupt.rs

@@ -13,6 +13,7 @@
 trait Sized {}
 #[lang = "copy"]
 trait Copy {}
+impl Copy for i64 {}
 
 // CHECK: define x86_intrcc i64 @has_x86_interrupt_abi
 #[no_mangle]

src/test/codegen/frame-pointer.rs

@@ -17,7 +17,7 @@
 trait Sized { }
 #[lang="copy"]
 trait Copy { }
-
+impl Copy for u32 {}
 
 
 // CHECK: define i32 @peach{{.*}}[[PEACH_ATTRS:\#[0-9]+]] {

src/test/codegen/inline-hint.rs

@@ -6,7 +6,7 @@
 
 pub fn f() {
     let a = A;
-    let b = (0i32, 1i32, 2i32, 3i32);
+    let b = (0i32, 1i32, 2i32, 3 as *const i32);
     let c = || {};
 
     a(String::new(), String::new());
@@ -21,7 +21,7 @@ struct A(String, String);
 // CHECK-NOT:  inlinehint
 // CHECK-SAME: {{$}}
 
-// CHECK:      ; <(i32, i32, i32, i32) as core::clone::Clone>::clone
+// CHECK:      ; <(i32, i32, i32, *const i{{16|32|64}}) as core::clone::Clone>::clone
 // CHECK-NEXT: ; Function Attrs: inlinehint
 
 // CHECK:      ; inline_hint::f::{closure#0}

src/test/codegen/riscv-abi/riscv64-lp64-lp64f-lp64d-abi.rs

@@ -10,6 +10,14 @@
 trait Sized {}
 #[lang = "copy"]
 trait Copy {}
+impl Copy for bool {}
+impl Copy for i8 {}
+impl Copy for u8 {}
+impl Copy for i32 {}
+impl Copy for i64 {}
+impl Copy for u64 {}
+impl Copy for f32 {}
+impl Copy for f64 {}
 
 // CHECK: define void @f_void()
 #[no_mangle]

src/test/mir-opt/combine_clone_of_primitives.rs

@@ -0,0 +1,20 @@
+// compile-flags: -C opt-level=0 -Z inline_mir=no
+// ignore-wasm32 compiled with panic=abort by default
+
+// EMIT_MIR combine_clone_of_primitives.{impl#0}-clone.InstCombine.diff
+
+#[derive(Clone)]
+struct MyThing<T> {
+    v: T,
+    i: u64,
+    a: [f32; 3],
+}
+
+fn main() {
+    let x = MyThing::<i16> { v: 2, i: 3, a: [0.0; 3] };
+    let y = x.clone();
+
+    assert_eq!(y.v, 2);
+    assert_eq!(y.i, 3);
+    assert_eq!(y.a, [0.0; 3]);
+}

src/test/mir-opt/combine_clone_of_primitives.{impl#0}-clone.InstCombine.diff

@@ -0,0 +1,80 @@
+- // MIR for `<impl at $DIR/combine_clone_of_primitives.rs:6:10: 6:15>::clone` before InstCombine
++ // MIR for `<impl at $DIR/combine_clone_of_primitives.rs:6:10: 6:15>::clone` after InstCombine
+
+  fn <impl at $DIR/combine_clone_of_primitives.rs:6:10: 6:15>::clone(_1: &MyThing<T>) -> MyThing<T> {
+      debug self => _1;                    // in scope 0 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15
+      let mut _0: MyThing<T>;              // return place in scope 0 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15
+      let _2: &T;                          // in scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+      let _3: &u64;                        // in scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+      let _4: &[f32; 3];                   // in scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+      let mut _5: T;                       // in scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+      let mut _6: &T;                      // in scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+      let _7: &T;                          // in scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+      let mut _8: u64;                     // in scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+      let mut _9: &u64;                    // in scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+      let _10: &u64;                       // in scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+      let mut _11: [f32; 3];               // in scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+      let mut _12: &[f32; 3];              // in scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+      let _13: &[f32; 3];                  // in scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+      scope 1 {
+          debug __self_0_0 => _2;          // in scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+          debug __self_0_1 => _3;          // in scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+          debug __self_0_2 => _4;          // in scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+      }
+
+      bb0: {
+          _2 = &((*_1).0: T);              // scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+          _3 = &((*_1).1: u64);            // scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+          _4 = &((*_1).2: [f32; 3]);       // scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+-         _7 = &(*_2);                     // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+-         _6 = &(*_7);                     // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
++         _7 = _2;                         // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
++         _6 = _7;                         // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+          _5 = <T as Clone>::clone(move _6) -> bb1; // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+                                           // mir::Constant
+                                           // + span: $DIR/combine_clone_of_primitives.rs:8:5: 8:9
+                                           // + literal: Const { ty: for<'r> fn(&'r T) -> T {<T as Clone>::clone}, val: Value(Scalar(<ZST>)) }
+      }
+
+      bb1: {
+-         _10 = &(*_3);                    // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+-         _9 = &(*_10);                    // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+-         _8 = <u64 as Clone>::clone(move _9) -> [return: bb2, unwind: bb4]; // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+-                                          // mir::Constant
+-                                          // + span: $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+-                                          // + literal: Const { ty: for<'r> fn(&'r u64) -> u64 {<u64 as Clone>::clone}, val: Value(Scalar(<ZST>)) }
++         _10 = _3;                        // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
++         _9 = _10;                        // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
++         _8 = (*_9);                      // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
++         goto -> bb2;                     // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11
+      }
+
+      bb2: {
+-         _13 = &(*_4);                    // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+-         _12 = &(*_13);                   // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+-         _11 = <[f32; 3] as Clone>::clone(move _12) -> [return: bb3, unwind: bb4]; // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+-                                          // mir::Constant
+-                                          // + span: $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+-                                          // + literal: Const { ty: for<'r> fn(&'r [f32; 3]) -> [f32; 3] {<[f32; 3] as Clone>::clone}, val: Value(Scalar(<ZST>)) }
++         _13 = _4;                        // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
++         _12 = _13;                       // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
++         _11 = (*_12);                    // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
++         goto -> bb3;                     // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16
+      }
+
+      bb3: {
+          (_0.0: T) = move _5;             // scope 1 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15
+          (_0.1: u64) = move _8;           // scope 1 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15
+          (_0.2: [f32; 3]) = move _11;     // scope 1 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15
+          return;                          // scope 0 at $DIR/combine_clone_of_primitives.rs:6:15: 6:15
+      }
+
+      bb4 (cleanup): {
+          drop(_5) -> bb5;                 // scope 1 at $DIR/combine_clone_of_primitives.rs:6:14: 6:15
+      }
+
+      bb5 (cleanup): {
+          resume;                          // scope 0 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15
+      }
+  }
+

src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-registers.rs

@@ -7,6 +7,7 @@
 pub trait Sized { }
 #[lang="copy"]
 pub trait Copy { }
+impl Copy for u32 {}
 
 extern "rust-intrinsic" {
     pub fn transmute<T, U>(e: T) -> U;

src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-stack.rs

@@ -7,6 +7,7 @@
 pub trait Sized { }
 #[lang="copy"]
 pub trait Copy { }
+impl Copy for u32 {}
 
 extern "rust-intrinsic" {
     pub fn transmute<T, U>(e: T) -> U;