diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 80e7c5bd9edbc..c1463196c5916 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -368,6 +368,29 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { source_info.span, ); } + AssertKind::OccupiedNiche { + ref found, + ref start, + ref end, + ref type_name, + ref offset, + ref niche_ty, + } => { + let found = codegen_operand(fx, found).load_scalar(fx); + let start = codegen_operand(fx, start).load_scalar(fx); + let end = codegen_operand(fx, end).load_scalar(fx); + let type_name = fx.anonymous_str(type_name); + let offset = codegen_operand(fx, offset).load_scalar(fx); + let niche_ty = fx.anonymous_str(niche_ty); + let location = fx.get_caller_location(source_info).load_scalar(fx); + + codegen_panic_inner( + fx, + rustc_hir::LangItem::PanicOccupiedNiche, + &[found, start, end, type_name, offset, niche_ty, location], + source_info.span, + ) + } _ => { let msg_str = msg.description(); codegen_panic(fx, msg_str, source_info); diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index caade768795f6..571bdd94ad78c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -624,6 +624,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // and `#[track_caller]` adds an implicit third argument. (LangItem::PanicMisalignedPointerDereference, vec![required, found, location]) } + AssertKind::OccupiedNiche { ref found, ref start, ref end } => { + let found = self.codegen_operand(bx, found).immediate(); + let start = self.codegen_operand(bx, start).immediate(); + let end = self.codegen_operand(bx, end).immediate(); + (LangItem::PanicOccupiedNiche, vec![found, start, end, location]) + } _ => { let msg = bx.const_str(msg.description()); // It's `pub fn panic(expr: &str)`, with the wide reference being passed diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 0d0ebe6f390b0..4ee3f91805542 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -542,6 +542,11 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, found: eval_to_int(found)?, } } + OccupiedNiche { ref found, ref start, ref end } => OccupiedNiche { + found: eval_to_int(found)?, + start: eval_to_int(start)?, + end: eval_to_int(end)?, + }, }; Err(ConstEvalErrKind::AssertFailure(err).into()) } diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs index 8fc0205d6c772..4b399adec9f00 100644 --- a/compiler/rustc_const_eval/src/interpret/cast.rs +++ b/compiler/rustc_const_eval/src/interpret/cast.rs @@ -6,6 +6,7 @@ use rustc_middle::mir::interpret::{InterpResult, PointerArithmetic, Scalar}; use rustc_middle::mir::CastKind; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::layout::{IntegerExt, LayoutOf, TyAndLayout}; +use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{self, FloatTy, Ty, TypeAndMut}; use rustc_target::abi::Integer; use rustc_type_ir::TyKind::*; @@ -147,8 +148,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { if src.layout.size != dest.layout.size { let src_bytes = src.layout.size.bytes(); let dest_bytes = dest.layout.size.bytes(); - let src_ty = format!("{}", src.layout.ty); - let dest_ty = format!("{}", dest.layout.ty); + let src_ty = with_no_trimmed_paths!(format!("{}", src.layout.ty)); + let dest_ty = with_no_trimmed_paths!(format!("{}", dest.layout.ty)); throw_ub_custom!( fluent::const_eval_invalid_transmute, src_bytes = src_bytes, diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index c8ac95e29949b..cfab84c71396b 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -234,6 +234,7 @@ language_item_table! { ConstPanicFmt, sym::const_panic_fmt, const_panic_fmt, Target::Fn, GenericRequirement::None; PanicBoundsCheck, sym::panic_bounds_check, panic_bounds_check_fn, Target::Fn, GenericRequirement::Exact(0); PanicMisalignedPointerDereference, sym::panic_misaligned_pointer_dereference, panic_misaligned_pointer_dereference_fn, Target::Fn, GenericRequirement::Exact(0); + PanicOccupiedNiche, sym::panic_occupied_niche, panic_occupied_niche_fn, Target::Fn, GenericRequirement::Exact(0); PanicInfo, sym::panic_info, panic_info, Target::Struct, GenericRequirement::None; PanicLocation, sym::panic_location, panic_location, Target::Struct, GenericRequirement::None; PanicImpl, sym::panic_impl, panic_impl, Target::Fn, GenericRequirement::None; diff --git a/compiler/rustc_middle/messages.ftl b/compiler/rustc_middle/messages.ftl index 37ff5bcf1e2da..07ff2288e8d5f 100644 --- a/compiler/rustc_middle/messages.ftl +++ b/compiler/rustc_middle/messages.ftl @@ -15,6 +15,9 @@ middle_assert_divide_by_zero = middle_assert_misaligned_ptr_deref = misaligned pointer dereference: address must be a multiple of {$required} but is {$found} +middle_assert_occupied_niche = + occupied niche: {$found} must be in {$start}..={$end} + middle_assert_op_overflow = attempt to compute `{$left} {$op} {$right}`, which would overflow diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index b89c7bc512e1e..e2288d2e6b042 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -886,6 +886,7 @@ pub enum AssertKind<O> { ResumedAfterReturn(CoroutineKind), ResumedAfterPanic(CoroutineKind), MisalignedPointerDereference { required: O, found: O }, + OccupiedNiche { found: O, start: O, end: O }, } #[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)] diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs index affa83fa34858..a6b51c13be3af 100644 --- a/compiler/rustc_middle/src/mir/terminator.rs +++ b/compiler/rustc_middle/src/mir/terminator.rs @@ -150,7 +150,7 @@ impl<O> AssertKind<O> { ResumedAfterReturn(CoroutineKind::Async(_)) => "`async fn` resumed after completion", ResumedAfterPanic(CoroutineKind::Coroutine) => "coroutine resumed after panicking", ResumedAfterPanic(CoroutineKind::Async(_)) => "`async fn` resumed after panicking", - BoundsCheck { .. } | MisalignedPointerDereference { .. } => { + BoundsCheck { .. } | MisalignedPointerDereference { .. } | OccupiedNiche { .. } => { bug!("Unexpected AssertKind") } } @@ -213,6 +213,13 @@ impl<O> AssertKind<O> { "\"misaligned pointer dereference: address must be a multiple of {{}} but is {{}}\", {required:?}, {found:?}" ) } + OccupiedNiche { found, start, end } => { + write!( + f, + "\"occupied niche: {{}} must be in {{}}..={{}}\" {:?} {:?} {:?}", + found, start, end + ) + } _ => write!(f, "\"{}\"", self.description()), } } @@ -243,8 +250,8 @@ impl<O> AssertKind<O> { ResumedAfterPanic(CoroutineKind::Coroutine) => { middle_assert_coroutine_resume_after_panic } - MisalignedPointerDereference { .. } => middle_assert_misaligned_ptr_deref, + OccupiedNiche { .. } => middle_assert_occupied_niche, } } @@ -281,6 +288,11 @@ impl<O> AssertKind<O> { add!("required", format!("{required:#?}")); add!("found", format!("{found:#?}")); } + OccupiedNiche { found, start, end } => { + add!("found", format!("{found:?}")); + add!("start", format!("{start:?}")); + add!("end", format!("{end:?}")); + } } } } diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 0f0ca3a1420e9..c4806c24ce96e 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -625,6 +625,11 @@ macro_rules! make_mir_visitor { self.visit_operand(required, location); self.visit_operand(found, location); } + OccupiedNiche { found, start, end } => { + self.visit_operand(found, location); + self.visit_operand(start, location); + self.visit_operand(end, location); + } } } diff --git a/compiler/rustc_mir_transform/src/check_niches.rs b/compiler/rustc_mir_transform/src/check_niches.rs new file mode 100644 index 0000000000000..ac777a92b55b8 --- /dev/null +++ b/compiler/rustc_mir_transform/src/check_niches.rs @@ -0,0 +1,434 @@ +use crate::MirPass; +use rustc_hir::def::DefKind; +use rustc_hir::lang_items::LangItem; +use rustc_hir::Unsafety; +use rustc_index::IndexVec; +use rustc_middle::mir::interpret::Scalar; +use rustc_middle::mir::visit::NonMutatingUseContext; +use rustc_middle::mir::visit::PlaceContext; +use rustc_middle::mir::visit::Visitor; +use rustc_middle::mir::*; +use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::ty::{ParamEnv, ParamEnvAnd, Ty, TyCtxt}; +use rustc_session::Session; +use rustc_target::abi::{Integer, Niche, Primitive, Size}; + +pub struct CheckNiches; + +impl<'tcx> MirPass<'tcx> for CheckNiches { + fn is_enabled(&self, sess: &Session) -> bool { + sess.opts.debug_assertions + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + // This pass emits new panics. If for whatever reason we do not have a panic + // implementation, running this pass may cause otherwise-valid code to not compile. + if tcx.lang_items().get(LangItem::PanicImpl).is_none() { + return; + } + + let def_id = body.source.def_id(); + if tcx.type_of(def_id).instantiate_identity().is_coroutine() { + return; + } + + // This pass will in general insert an enormous amount of checks, so we need some way to + // trim them down a bit. + // Our tactic here is to only emit checks if we are compiling an `unsafe fn` or a function + // that contains an unsafe block. Yes this means that we can fail to check some operations. + // If you have a better strategy that doesn't impose 2x compile time overhead, please + // share. + let is_safe_code = + tcx.unsafety_check_result(def_id.expect_local()).used_unsafe_blocks.is_empty(); + let is_unsafe_fn = match tcx.def_kind(def_id) { + DefKind::Closure => false, + _ => tcx.fn_sig(def_id).skip_binder().unsafety() == Unsafety::Unsafe, + }; + if is_safe_code && !is_unsafe_fn { + return; + } + + with_no_trimmed_paths!(debug!("Inserting niche checks for {:?}", body.source)); + + let basic_blocks = body.basic_blocks.as_mut(); + let param_env = tcx.param_env_reveal_all_normalized(def_id); + let local_decls: &mut IndexVec<_, _> = &mut body.local_decls; + + // This pass inserts new blocks. Each insertion changes the Location for all + // statements/blocks after. Iterating or visiting the MIR in order would require updating + // our current location after every insertion. By iterating backwards, we dodge this issue: + // The only Locations that an insertion changes have already been handled. + for block in (0..basic_blocks.len()).rev() { + let block = block.into(); + for statement_index in (0..basic_blocks[block].statements.len()).rev() { + let location = Location { block, statement_index }; + let statement = &basic_blocks[block].statements[statement_index]; + let source_info = statement.source_info; + + let mut finder = NicheFinder { tcx, local_decls, param_env, places: Vec::new() }; + finder.visit_statement(statement, location); + + for (place, ty, niche) in core::mem::take(&mut finder.places) { + with_no_trimmed_paths!(debug!("Inserting niche check for {:?}", ty)); + let (block_data, new_block) = split_block(basic_blocks, location); + finder.insert_niche_check(block_data, new_block, place, niche, source_info); + } + } + } + } +} + +struct NicheFinder<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + local_decls: &'a mut IndexVec<Local, LocalDecl<'tcx>>, + param_env: ParamEnv<'tcx>, + places: Vec<(Operand<'tcx>, Ty<'tcx>, NicheKind)>, +} + +impl<'a, 'tcx> Visitor<'tcx> for NicheFinder<'a, 'tcx> { + fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { + if let Rvalue::Cast(CastKind::Transmute, op, ty) = rvalue { + if let Some(niche) = self.get_niche(*ty) { + self.places.push((op.clone(), *ty, niche)); + } + } + + self.super_rvalue(rvalue, location); + } + + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + match context { + PlaceContext::NonMutatingUse( + NonMutatingUseContext::Inspect + | NonMutatingUseContext::Copy + | NonMutatingUseContext::Move, + ) => {} + _ => { + return; + } + } + + let ty = place.ty(self.local_decls, self.tcx).ty; + let Some(niche) = self.get_niche(ty) else { + return; + }; + + self.places.push((Operand::Copy(*place), ty, niche)); + + self.super_place(place, context, location); + } +} + +impl<'a, 'tcx> NicheFinder<'a, 'tcx> { + fn insert_niche_check( + &mut self, + block_data: &mut BasicBlockData<'tcx>, + new_block: BasicBlock, + operand: Operand<'tcx>, + niche: NicheKind, + source_info: SourceInfo, + ) { + let value_in_niche = self + .local_decls + .push(LocalDecl::with_source_info(niche.niche().ty(self.tcx), source_info)) + .into(); + + match niche { + NicheKind::Full(niche) => { + // The niche occupies the entire source Operand, so we can just transmute + // directly to the niche primitive. + let rvalue = Rvalue::Cast(CastKind::Transmute, operand, niche.ty(self.tcx)); + block_data.statements.push(Statement { + source_info, + kind: StatementKind::Assign(Box::new((value_in_niche, rvalue))), + }); + } + NicheKind::Partial(niche, size) => { + let mu = Ty::new_maybe_uninit(self.tcx, niche.ty(self.tcx)); + + // Transmute the niche-containing type to a [MaybeUninit; N] + let array_len = size.bytes() / niche.size(self.tcx).bytes(); + let mu_array_ty = Ty::new_array(self.tcx, mu, array_len); + let mu_array = self + .local_decls + .push(LocalDecl::with_source_info(mu_array_ty, source_info)) + .into(); + let rvalue = Rvalue::Cast(CastKind::Transmute, operand, mu_array_ty); + block_data.statements.push(Statement { + source_info, + kind: StatementKind::Assign(Box::new((mu_array, rvalue))), + }); + + // Convert the niche byte offset into an array index + assert_eq!(niche.offset.bytes() % niche.size(self.tcx).bytes(), 0); + let offset = niche.offset.bytes() / niche.size(self.tcx).bytes(); + + let niche_as_mu = mu_array.project_deeper( + &[ProjectionElem::ConstantIndex { + offset, + min_length: array_len, + from_end: false, + }], + self.tcx, + ); + + // Transmute the MaybeUninit<T> to the niche primitive + let rvalue = Rvalue::Cast( + CastKind::Transmute, + Operand::Copy(niche_as_mu), + niche.ty(self.tcx), + ); + block_data.statements.push(Statement { + source_info, + kind: StatementKind::Assign(Box::new((value_in_niche, rvalue))), + }); + } + } + + let is_in_range = self + .local_decls + .push(LocalDecl::with_source_info(self.tcx.types.bool, source_info)) + .into(); + + NicheCheckBuilder { + tcx: self.tcx, + local_decls: self.local_decls, + block_data, + new_block, + niche: niche.niche(), + value_in_niche, + is_in_range, + source_info, + } + .insert_niche_check(); + } + + fn get_niche(&self, ty: Ty<'tcx>) -> Option<NicheKind> { + // If we can't get the layout of this, just skip the check. + let query = ParamEnvAnd { value: ty, param_env: self.param_env }; + let Ok(layout) = self.tcx.layout_of(query) else { + return None; + }; + + let niche = layout.largest_niche?; + + if niche.size(self.tcx) == layout.size { + Some(NicheKind::Full(niche)) + } else { + Some(NicheKind::Partial(niche, layout.size)) + } + } +} + +#[derive(Clone, Copy)] +enum NicheKind { + Full(Niche), + // We need the full Size of the type in order to do the transmute-to-MU approach + Partial(Niche, Size), +} + +impl NicheKind { + fn niche(self) -> Niche { + use NicheKind::*; + match self { + Full(niche) => niche, + Partial(niche, _size) => niche, + } + } +} + +fn split_block<'a, 'tcx>( + basic_blocks: &'a mut IndexVec<BasicBlock, BasicBlockData<'tcx>>, + location: Location, +) -> (&'a mut BasicBlockData<'tcx>, BasicBlock) { + let block_data = &mut basic_blocks[location.block]; + + // Drain every statement after this one and move the current terminator to a new basic block + let new_block = BasicBlockData { + statements: block_data.statements.drain(location.statement_index..).collect(), + terminator: block_data.terminator.take(), + is_cleanup: block_data.is_cleanup, + }; + + let new_block = basic_blocks.push(new_block); + let block_data = &mut basic_blocks[location.block]; + + (block_data, new_block) +} + +struct NicheCheckBuilder<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + block_data: &'a mut BasicBlockData<'tcx>, + local_decls: &'a mut IndexVec<Local, LocalDecl<'tcx>>, + new_block: BasicBlock, + niche: Niche, + value_in_niche: Place<'tcx>, + is_in_range: Place<'tcx>, + source_info: SourceInfo, +} + +impl<'a, 'tcx> NicheCheckBuilder<'a, 'tcx> { + fn insert_niche_check(&mut self) { + let niche = self.niche; + let tcx = self.tcx; + + let size = self.niche.size(self.tcx); + + if niche.valid_range.start == 0 { + // The niche starts at 0, so we can just check if it is Le the end + self.check_end_only(); + } else if niche.valid_range.end == (u128::MAX >> (128 - size.bits())) { + // The niche ends at the max, so we can just check if it is Ge the start + self.check_start_only(); + } else { + self.general_case(); + } + + let start = Operand::Constant(Box::new(ConstOperand { + span: self.source_info.span, + user_ty: None, + const_: Const::Val( + ConstValue::Scalar(Scalar::from_uint( + niche.valid_range.start as u128, + Size::from_bits(128), + )), + tcx.types.u128, + ), + })); + let end = Operand::Constant(Box::new(ConstOperand { + span: self.source_info.span, + user_ty: None, + const_: Const::Val( + ConstValue::Scalar(Scalar::from_uint( + niche.valid_range.end as u128, + Size::from_bits(128), + )), + tcx.types.u128, + ), + })); + + let u128_in_niche = self + .local_decls + .push(LocalDecl::with_source_info(tcx.types.u128, self.source_info)) + .into(); + let rvalue = + Rvalue::Cast(CastKind::IntToInt, Operand::Copy(self.value_in_niche), tcx.types.u128); + self.block_data.statements.push(Statement { + source_info: self.source_info, + kind: StatementKind::Assign(Box::new((u128_in_niche, rvalue))), + }); + + self.block_data.terminator = Some(Terminator { + source_info: self.source_info, + kind: TerminatorKind::Assert { + cond: Operand::Copy(self.is_in_range), + expected: true, + target: self.new_block, + msg: Box::new(AssertKind::OccupiedNiche { + found: Operand::Copy(u128_in_niche), + start, + end, + }), + // This calls panic_occupied_niche, which is #[rustc_nounwind]. + // We never want to insert an unwind into unsafe code, because unwinding could + // make a failing UB check turn into much worse UB when we start unwinding. + unwind: UnwindAction::Unreachable, + }, + }); + } + + fn niche_const(&self, val: u128) -> Operand<'tcx> { + Operand::Constant(Box::new(ConstOperand { + span: self.source_info.span, + user_ty: None, + const_: Const::Val( + ConstValue::Scalar(Scalar::from_uint(val, self.niche.size(self.tcx))), + self.niche.ty(self.tcx), + ), + })) + } + + fn add_assignment(&mut self, place: Place<'tcx>, rvalue: Rvalue<'tcx>) { + self.block_data.statements.push(Statement { + source_info: self.source_info, + kind: StatementKind::Assign(Box::new((place, rvalue))), + }); + } + + fn check_start_only(&mut self) { + let start = self.niche_const(self.niche.valid_range.start); + + let rvalue = + Rvalue::BinaryOp(BinOp::Ge, Box::new((Operand::Copy(self.value_in_niche), start))); + self.add_assignment(self.is_in_range, rvalue); + } + + fn check_end_only(&mut self) { + let end = self.niche_const(self.niche.valid_range.end); + + let rvalue = + Rvalue::BinaryOp(BinOp::Le, Box::new((Operand::Copy(self.value_in_niche), end))); + self.add_assignment(self.is_in_range, rvalue); + } + + fn general_case(&mut self) { + let mut max = self.niche.valid_range.end.wrapping_sub(self.niche.valid_range.start); + let size = self.niche.size(self.tcx); + if size.bits() < 128 { + let mask = (1 << size.bits()) - 1; + max &= mask; + } + + let start = self.niche_const(self.niche.valid_range.start); + let max_adjusted_allowed_value = self.niche_const(max); + + let biased = self + .local_decls + .push(LocalDecl::with_source_info(self.niche.ty(self.tcx), self.source_info)) + .into(); + let rvalue = + Rvalue::BinaryOp(BinOp::Sub, Box::new((Operand::Copy(self.value_in_niche), start))); + self.add_assignment(biased, rvalue); + + let rvalue = Rvalue::BinaryOp( + BinOp::Le, + Box::new((Operand::Copy(biased), max_adjusted_allowed_value)), + ); + self.add_assignment(self.is_in_range, rvalue); + } +} + +trait NicheExt { + fn ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx>; + fn size<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Size; +} + +impl NicheExt for Niche { + fn ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> { + let types = &tcx.types; + match self.value { + Primitive::Int(Integer::I8, _) => types.u8, + Primitive::Int(Integer::I16, _) => types.u16, + Primitive::Int(Integer::I32, _) => types.u32, + Primitive::Int(Integer::I64, _) => types.u64, + Primitive::Int(Integer::I128, _) => types.u128, + Primitive::Pointer(_) => types.usize, + Primitive::F32 => types.u32, + Primitive::F64 => types.u64, + } + } + + fn size<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Size { + let bits = match self.value { + Primitive::Int(Integer::I8, _) => 8, + Primitive::Int(Integer::I16, _) => 16, + Primitive::Int(Integer::I32, _) => 32, + Primitive::Int(Integer::I64, _) => 64, + Primitive::Int(Integer::I128, _) => 128, + Primitive::Pointer(_) => tcx.sess.target.pointer_width as usize, + Primitive::F32 => 32, + Primitive::F64 => 64, + }; + Size::from_bits(bits) + } +} diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 68b8911824c2e..40b9b01d75b8a 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -51,6 +51,7 @@ mod add_call_guards; mod add_moves_for_packed_drops; mod add_retag; mod check_const_item_mutation; +mod check_niches; mod check_packed_ref; pub mod check_unsafety; mod remove_place_mention; @@ -320,6 +321,7 @@ fn mir_promoted( tcx: TyCtxt<'_>, def: LocalDefId, ) -> (&Steal<Body<'_>>, &Steal<IndexVec<Promoted, Body<'_>>>) { + tcx.ensure_with_value().unsafety_check_result(def); // Ensure that we compute the `mir_const_qualif` for constants at // this point, before we steal the mir-const result. // Also this means promotion can rely on all const checks having been done. @@ -566,6 +568,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { body, &[ &check_alignment::CheckAlignment, + &check_niches::CheckNiches, &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first &unreachable_prop::UnreachablePropagation, &uninhabited_enum_branching::UninhabitedEnumBranching, diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs index eb8689130174d..c826bcc459d48 100644 --- a/compiler/rustc_smir/src/rustc_smir/mod.rs +++ b/compiler/rustc_smir/src/rustc_smir/mod.rs @@ -799,6 +799,13 @@ impl<'tcx> Stable<'tcx> for mir::AssertMessage<'tcx> { found: found.stable(tables), } } + AssertKind::OccupiedNiche { found, start, end } => { + stable_mir::mir::AssertMessage::OccupiedNiche { + found: found.stable(tables), + start: start.stable(tables), + end: end.stable(tables), + } + } } } } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 88d9dab2ba560..cb6f1bc4034c9 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1149,6 +1149,7 @@ symbols! { panic_location, panic_misaligned_pointer_dereference, panic_nounwind, + panic_occupied_niche, panic_runtime, panic_str, panic_unwind, diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs index 50c37e8f91033..e65b7cacdf6c8 100644 --- a/compiler/stable_mir/src/mir/body.rs +++ b/compiler/stable_mir/src/mir/body.rs @@ -149,6 +149,7 @@ pub enum AssertMessage { ResumedAfterReturn(CoroutineKind), ResumedAfterPanic(CoroutineKind), MisalignedPointerDereference { required: Operand, found: Operand }, + OccupiedNiche { found: Operand, start: Operand, end: Operand }, } #[derive(Clone, Debug)] diff --git a/library/core/src/panicking.rs b/library/core/src/panicking.rs index 39a5e8d9fe2ec..bc2902725efaa 100644 --- a/library/core/src/panicking.rs +++ b/library/core/src/panicking.rs @@ -208,6 +208,22 @@ fn panic_misaligned_pointer_dereference(required: usize, found: usize) -> ! { ) } +#[cold] +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] +#[track_caller] +#[cfg_attr(not(bootstrap), lang = "panic_occupied_niche")] // needed by codegen for panic on occupied niches +#[rustc_nounwind] +fn panic_occupied_niche(found: u128, min: u128, max: u128) -> ! { + if cfg!(feature = "panic_immediate_abort") { + super::intrinsics::abort() + } + + panic_nounwind_fmt( + format_args!("occupied niche: found {found:#x} but must be in {min:#x}..={max:#x}"), + /* force_no_backtrace */ false, + ) +} + /// Panic because we cannot unwind out of a function. /// /// This is a separate function to avoid the codesize impact of each crate containing the string to diff --git a/src/tools/miri/src/lib.rs b/src/tools/miri/src/lib.rs index b12aae6d4148c..44f5c99ba9128 100644 --- a/src/tools/miri/src/lib.rs +++ b/src/tools/miri/src/lib.rs @@ -139,5 +139,5 @@ pub const MIRI_DEFAULT_ARGS: &[&str] = &[ "-Zmir-emit-retag", "-Zmir-keep-place-mention", "-Zmir-opt-level=0", - "-Zmir-enable-passes=-CheckAlignment", + "-Zmir-enable-passes=-CheckAlignment,-CheckNiches", ]; diff --git a/tests/codegen/array-equality.rs b/tests/codegen/array-equality.rs index 1941452ea6159..0edd45ecf8d90 100644 --- a/tests/codegen/array-equality.rs +++ b/tests/codegen/array-equality.rs @@ -1,5 +1,6 @@ // compile-flags: -O -Z merge-functions=disabled // only-x86_64 +// ignore-debug: array comparison sometimes transmutes references, so we have niche checks in std #![crate_type = "lib"] diff --git a/tests/codegen/debug-vtable.rs b/tests/codegen/debug-vtable.rs index e52392b260bd4..8bcb28735ade0 100644 --- a/tests/codegen/debug-vtable.rs +++ b/tests/codegen/debug-vtable.rs @@ -6,7 +6,7 @@ // legacy mangling scheme rustc version and generic parameters are both hashed into a single part // of the name, thus randomizing item order with respect to rustc version. -// compile-flags: -Cdebuginfo=2 -Copt-level=0 -Csymbol-mangling-version=v0 +// compile-flags: -Cdebuginfo=2 -Copt-level=0 -Csymbol-mangling-version=v0 -Zmir-enable-passes=-CheckNiches // ignore-tidy-linelength // Make sure that vtables don't have the unnamed_addr attribute when debuginfo is enabled. diff --git a/tests/codegen/intrinsics/transmute-niched.rs b/tests/codegen/intrinsics/transmute-niched.rs index e9c8d803cb9e9..86acb47bc8266 100644 --- a/tests/codegen/intrinsics/transmute-niched.rs +++ b/tests/codegen/intrinsics/transmute-niched.rs @@ -1,6 +1,6 @@ // revisions: OPT DBG // [OPT] compile-flags: -C opt-level=3 -C no-prepopulate-passes -// [DBG] compile-flags: -C opt-level=0 -C no-prepopulate-passes +// [DBG] compile-flags: -C opt-level=0 -C no-prepopulate-passes -Zmir-enable-passes=-CheckNiches // only-64bit (so I don't need to worry about usize) #![crate_type = "lib"] diff --git a/tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs b/tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs index a3cd16e3dd5a5..e53ea03574319 100644 --- a/tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs +++ b/tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs @@ -1,7 +1,7 @@ // Verifies that pointer type membership tests for indirect calls are omitted. // // needs-sanitizer-cfi -// compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 +// compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 -Zmir-enable-passes=-CheckNiches #![crate_type="lib"] #![feature(no_sanitize)] diff --git a/tests/codegen/sanitizer/cfi-emit-type-checks.rs b/tests/codegen/sanitizer/cfi-emit-type-checks.rs index f0fe5de9f66c1..567597393f774 100644 --- a/tests/codegen/sanitizer/cfi-emit-type-checks.rs +++ b/tests/codegen/sanitizer/cfi-emit-type-checks.rs @@ -1,7 +1,7 @@ // Verifies that pointer type membership tests for indirect calls are emitted. // // needs-sanitizer-cfi -// compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 +// compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 -Zmir-enable-passes=-CheckNiches #![crate_type="lib"] diff --git a/tests/codegen/thread-local.rs b/tests/codegen/thread-local.rs index caf0366d2c144..383014d0c3132 100644 --- a/tests/codegen/thread-local.rs +++ b/tests/codegen/thread-local.rs @@ -1,4 +1,5 @@ // compile-flags: -O +// ignore-debug: niche checks in std interfere with the codegen we are looking for // aux-build:thread_local_aux.rs // ignore-windows FIXME(#84933) // ignore-wasm globals are used instead of thread locals diff --git a/tests/codegen/transmute-scalar.rs b/tests/codegen/transmute-scalar.rs index 39126b024a6e9..8b29a06f66969 100644 --- a/tests/codegen/transmute-scalar.rs +++ b/tests/codegen/transmute-scalar.rs @@ -1,4 +1,4 @@ -// compile-flags: -C opt-level=0 -C no-prepopulate-passes +// compile-flags: -C opt-level=0 -C no-prepopulate-passes -Zmir-enable-passes=-CheckNiches #![crate_type = "lib"] diff --git a/tests/mir-opt/remove_storage_markers.rs b/tests/mir-opt/remove_storage_markers.rs index 6666ff3b7263b..a1a4c15c2bd53 100644 --- a/tests/mir-opt/remove_storage_markers.rs +++ b/tests/mir-opt/remove_storage_markers.rs @@ -4,7 +4,7 @@ // Checks that storage markers are removed at opt-level=0. // -// compile-flags: -C opt-level=0 -Coverflow-checks=off +// compile-flags: -C opt-level=0 -Cdebug-assertions=off // EMIT_MIR remove_storage_markers.main.RemoveStorageMarkers.diff fn main() { diff --git a/tests/ui-fulldeps/stable-mir/crate-info.rs b/tests/ui-fulldeps/stable-mir/crate-info.rs index ed6b786f5e1de..692716e337fbc 100644 --- a/tests/ui-fulldeps/stable-mir/crate-info.rs +++ b/tests/ui-fulldeps/stable-mir/crate-info.rs @@ -186,6 +186,7 @@ fn main() { let args = vec![ "rustc".to_string(), "--crate-type=lib".to_string(), + "-Zmir-enable-passes=-CheckNiches".to_string(), "--crate-name".to_string(), CRATE_NAME.to_string(), path.to_string(), diff --git a/tests/ui/lint/large_assignments/box_rc_arc_allowed.rs b/tests/ui/lint/large_assignments/box_rc_arc_allowed.rs index 33113642023a2..99282829dbf29 100644 --- a/tests/ui/lint/large_assignments/box_rc_arc_allowed.rs +++ b/tests/ui/lint/large_assignments/box_rc_arc_allowed.rs @@ -5,7 +5,7 @@ // only-x86_64 // edition:2018 -// compile-flags: -Zmir-opt-level=0 +// compile-flags: -Zmir-opt-level=0 -Zmir-enable-passes=-CheckNiches use std::{sync::Arc, rc::Rc}; diff --git a/tests/ui/mir/check_niches/invalid_bool_transmutes.rs b/tests/ui/mir/check_niches/invalid_bool_transmutes.rs new file mode 100644 index 0000000000000..0916055b0cc00 --- /dev/null +++ b/tests/ui/mir/check_niches/invalid_bool_transmutes.rs @@ -0,0 +1,10 @@ +// run-fail +// ignore-wasm32-bare: No panic messages +// compile-flags: -C debug-assertions +// error-pattern: occupied niche: found 0x2 but must be in 0x0..=0x1 + +fn main() { + unsafe { + std::mem::transmute::<u8, bool>(2); + } +} diff --git a/tests/ui/mir/check_niches/invalid_enums.rs b/tests/ui/mir/check_niches/invalid_enums.rs new file mode 100644 index 0000000000000..a5117f8ba27a2 --- /dev/null +++ b/tests/ui/mir/check_niches/invalid_enums.rs @@ -0,0 +1,23 @@ +// run-fail +// ignore-wasm32-bare: No panic messages +// compile-flags: -C debug-assertions -Zmir-opt-level=0 + +#[repr(C)] +struct Thing { + x: usize, + y: Contents, + z: usize, +} + +#[repr(usize)] +enum Contents { + A = 8usize, + B = 9usize, + C = 10usize, +} + +fn main() { + unsafe { + let _thing = std::mem::transmute::<(usize, usize, usize), Thing>((0, 3, 0)); + } +} diff --git a/tests/ui/mir/check_niches/invalid_nonnull_transmute.rs b/tests/ui/mir/check_niches/invalid_nonnull_transmute.rs new file mode 100644 index 0000000000000..99ed52b81f8d4 --- /dev/null +++ b/tests/ui/mir/check_niches/invalid_nonnull_transmute.rs @@ -0,0 +1,10 @@ +// run-fail +// ignore-wasm32-bare: No panic messages +// compile-flags: -C debug-assertions -Zmir-opt-level=0 +// error-pattern: occupied niche: found 0x0 but must be in 0x1..=0xffffffff + +fn main() { + unsafe { + std::mem::transmute::<*const u8, std::ptr::NonNull<u8>>(std::ptr::null()); + } +} diff --git a/tests/ui/mir/check_niches/invalid_nonzero_argument.rs b/tests/ui/mir/check_niches/invalid_nonzero_argument.rs new file mode 100644 index 0000000000000..d0da59d0e41a3 --- /dev/null +++ b/tests/ui/mir/check_niches/invalid_nonzero_argument.rs @@ -0,0 +1,14 @@ +// run-fail +// ignore-wasm32-bare: No panic messages +// compile-flags: -C debug-assertions -Zmir-opt-level=0 +// error-pattern: occupied niche: found 0x0 but must be in 0x1..=0xff + +fn main() { + let mut bad = std::num::NonZeroU8::new(1u8).unwrap(); + unsafe { + std::ptr::write_bytes(&mut bad, 0u8, 1usize); + } + func(bad); +} + +fn func<T>(_t: T) {} diff --git a/tests/ui/mir/check_niches/valid_bool_transmutes.rs b/tests/ui/mir/check_niches/valid_bool_transmutes.rs new file mode 100644 index 0000000000000..adb67ff7517e9 --- /dev/null +++ b/tests/ui/mir/check_niches/valid_bool_transmutes.rs @@ -0,0 +1,12 @@ +// run-pass +// ignore-wasm32-bare: No panic messages +// compile-flags: -C debug-assertions + +fn main() { + unsafe { + std::mem::transmute::<u8, bool>(0); + } + unsafe { + std::mem::transmute::<u8, bool>(1); + } +} diff --git a/tests/ui/mir/check_niches/valid_nonzero_argument.rs b/tests/ui/mir/check_niches/valid_nonzero_argument.rs new file mode 100644 index 0000000000000..727d3e0d67c53 --- /dev/null +++ b/tests/ui/mir/check_niches/valid_nonzero_argument.rs @@ -0,0 +1,13 @@ +// run-pass +// compile-flags: -C debug-assertions -Zmir-opt-level=0 + +fn main() { + for val in i8::MIN..=i8::MAX { + if val != 0 { + let x = std::num::NonZeroI8::new(val).unwrap(); + if val != i8::MIN { + let _y = -x; + } + } + } +} diff --git a/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs b/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs index cad05917391be..7557e5ce34bb0 100644 --- a/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs +++ b/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs @@ -1,6 +1,6 @@ // run-pass // only-wasm32 -// compile-flags: -Zmir-opt-level=0 -C target-feature=+nontrapping-fptoint +// compile-flags: -Zmir-opt-level=0 -C target-feature=+nontrapping-fptoint -Zmir-enable-passes=-CheckNiches #![feature(test, stmt_expr_attributes)] #![deny(overflowing_literals)] diff --git a/tests/ui/numbers-arithmetic/saturating-float-casts.rs b/tests/ui/numbers-arithmetic/saturating-float-casts.rs index cc248a9bea087..d26273bc8345f 100644 --- a/tests/ui/numbers-arithmetic/saturating-float-casts.rs +++ b/tests/ui/numbers-arithmetic/saturating-float-casts.rs @@ -1,5 +1,5 @@ // run-pass -// compile-flags:-Zmir-opt-level=0 +// compile-flags:-Zmir-opt-level=0 -Zmir-enable-passes=-CheckNiches #![feature(test, stmt_expr_attributes)] #![deny(overflowing_literals)] diff --git a/tests/ui/print_type_sizes/niche-filling.rs b/tests/ui/print_type_sizes/niche-filling.rs index 5e620f248b65d..82afbd2495706 100644 --- a/tests/ui/print_type_sizes/niche-filling.rs +++ b/tests/ui/print_type_sizes/niche-filling.rs @@ -1,4 +1,4 @@ -// compile-flags: -Z print-type-sizes --crate-type=lib +// compile-flags: -Z print-type-sizes --crate-type=lib -Zmir-enable-passes=-CheckNiches // build-pass // ignore-pass // ^-- needed because `--pass check` does not emit the output needed. diff --git a/tests/ui/type-alias-enum-variants/self-in-enum-definition.rs b/tests/ui/type-alias-enum-variants/self-in-enum-definition.rs index 8dadd77fc16d5..9091b3ba6b8aa 100644 --- a/tests/ui/type-alias-enum-variants/self-in-enum-definition.rs +++ b/tests/ui/type-alias-enum-variants/self-in-enum-definition.rs @@ -1,3 +1,5 @@ +// compile-flags: -Cdebug-assertions=no + #[repr(u8)] enum Alpha { V1 = 41, diff --git a/tests/ui/type-alias-enum-variants/self-in-enum-definition.stderr b/tests/ui/type-alias-enum-variants/self-in-enum-definition.stderr index aa79b1a57c4c8..5c9d75fffdc50 100644 --- a/tests/ui/type-alias-enum-variants/self-in-enum-definition.stderr +++ b/tests/ui/type-alias-enum-variants/self-in-enum-definition.stderr @@ -1,63 +1,53 @@ error[E0391]: cycle detected when simplifying constant for the type system `Alpha::V3::{constant#0}` - --> $DIR/self-in-enum-definition.rs:5:10 + --> $DIR/self-in-enum-definition.rs:7:10 | LL | V3 = Self::V1 {} as u8 + 2, | ^^^^^^^^^^^^^^^^^^^^^ | note: ...which requires simplifying constant for the type system `Alpha::V3::{constant#0}`... - --> $DIR/self-in-enum-definition.rs:5:10 + --> $DIR/self-in-enum-definition.rs:7:10 | LL | V3 = Self::V1 {} as u8 + 2, | ^^^^^^^^^^^^^^^^^^^^^ note: ...which requires const-evaluating + checking `Alpha::V3::{constant#0}`... - --> $DIR/self-in-enum-definition.rs:5:10 + --> $DIR/self-in-enum-definition.rs:7:10 | LL | V3 = Self::V1 {} as u8 + 2, | ^^^^^^^^^^^^^^^^^^^^^ note: ...which requires caching mir of `Alpha::V3::{constant#0}` for CTFE... - --> $DIR/self-in-enum-definition.rs:5:10 + --> $DIR/self-in-enum-definition.rs:7:10 | LL | V3 = Self::V1 {} as u8 + 2, | ^^^^^^^^^^^^^^^^^^^^^ note: ...which requires elaborating drops for `Alpha::V3::{constant#0}`... - --> $DIR/self-in-enum-definition.rs:5:10 + --> $DIR/self-in-enum-definition.rs:7:10 | LL | V3 = Self::V1 {} as u8 + 2, | ^^^^^^^^^^^^^^^^^^^^^ note: ...which requires borrow-checking `Alpha::V3::{constant#0}`... - --> $DIR/self-in-enum-definition.rs:5:10 + --> $DIR/self-in-enum-definition.rs:7:10 | LL | V3 = Self::V1 {} as u8 + 2, | ^^^^^^^^^^^^^^^^^^^^^ note: ...which requires promoting constants in MIR for `Alpha::V3::{constant#0}`... - --> $DIR/self-in-enum-definition.rs:5:10 - | -LL | V3 = Self::V1 {} as u8 + 2, - | ^^^^^^^^^^^^^^^^^^^^^ -note: ...which requires const checking `Alpha::V3::{constant#0}`... - --> $DIR/self-in-enum-definition.rs:5:10 - | -LL | V3 = Self::V1 {} as u8 + 2, - | ^^^^^^^^^^^^^^^^^^^^^ -note: ...which requires preparing `Alpha::V3::{constant#0}` for borrow checking... - --> $DIR/self-in-enum-definition.rs:5:10 + --> $DIR/self-in-enum-definition.rs:7:10 | LL | V3 = Self::V1 {} as u8 + 2, | ^^^^^^^^^^^^^^^^^^^^^ note: ...which requires unsafety-checking `Alpha::V3::{constant#0}`... - --> $DIR/self-in-enum-definition.rs:5:10 + --> $DIR/self-in-enum-definition.rs:7:10 | LL | V3 = Self::V1 {} as u8 + 2, | ^^^^^^^^^^^^^^^^^^^^^ note: ...which requires building MIR for `Alpha::V3::{constant#0}`... - --> $DIR/self-in-enum-definition.rs:5:10 + --> $DIR/self-in-enum-definition.rs:7:10 | LL | V3 = Self::V1 {} as u8 + 2, | ^^^^^^^^^^^^^^^^^^^^^ = note: ...which requires computing layout of `Alpha`... = note: ...which again requires simplifying constant for the type system `Alpha::V3::{constant#0}`, completing the cycle note: cycle used when collecting item types in top-level module - --> $DIR/self-in-enum-definition.rs:1:1 + --> $DIR/self-in-enum-definition.rs:3:1 | LL | / #[repr(u8)] LL | | enum Alpha {