From 46b620754e59438b275de5e177028992b6f5af34 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Wed, 23 Sep 2026 04:02:28 +0100 Subject: [PATCH 01/10] Remove incorrect comments about Rvalue::Aggregate not being allowed --- compiler/rustc_middle/src/mir/syntax.rs | 4 ---- compiler/rustc_public/src/mir/body.rs | 3 --- 2 files changed, 7 deletions(-) diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 487c2c43da7d5..b0462a8c2bd28 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -127,7 +127,6 @@ pub enum RuntimePhase { /// disallowed: /// * [`TerminatorKind::Yield`] /// * [`TerminatorKind::CoroutineDrop`] - /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array` /// * [`Rvalue::CopyForDeref`] /// * [`PlaceElem::OpaqueCast`] /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp) @@ -1449,9 +1448,6 @@ pub enum Rvalue<'tcx> { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(Box>, IndexVec>), /// A CopyForDeref is equivalent to a read from a place at the diff --git a/compiler/rustc_public/src/mir/body.rs b/compiler/rustc_public/src/mir/body.rs index a65798aff1a00..f4e93bd05a20c 100644 --- a/compiler/rustc_public/src/mir/body.rs +++ b/compiler/rustc_public/src/mir/body.rs @@ -551,9 +551,6 @@ pub enum Rvalue { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(AggregateKind, Vec), /// * `Offset` has the same semantics as `<*const T>::offset`, except that the second From 0a266baf9e4ed0f185955614550b7da940caed92 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Wed, 23 Sep 2026 04:02:30 +0100 Subject: [PATCH 02/10] Add MIR move elimination pass --- compiler/rustc_index/src/interval.rs | 16 + compiler/rustc_mir_transform/src/dest_prop.rs | 4 +- compiler/rustc_mir_transform/src/lib.rs | 2 + .../src/move_elimination.rs | 912 ++++++++++++++++++ compiler/rustc_session/src/options.rs | 2 +- ...indirect_source_alias.MoveElimination.diff | 27 + ..._fixup.aggregate_swap.MoveElimination.diff | 84 ++ ...lias_fixup.array_swap.MoveElimination.diff | 55 ++ ...lias_fixup.enum_alias.MoveElimination.diff | 120 +++ ...ed_aggregate_aliasing.MoveElimination.diff | 90 ++ tests/mir-opt/move-elimination/alias_fixup.rs | 156 +++ ...lias_fixup.simd_alias.MoveElimination.diff | 78 ++ ....simple_partial_alias.MoveElimination.diff | 52 + ...basic.array_aggregate.MoveElimination.diff | 67 ++ .../basic.enum_aggregate.MoveElimination.diff | 57 ++ .../basic.nrvo_borrowed.MoveElimination.diff | 46 + ...basic.nrvo_unborrowed.MoveElimination.diff | 24 + tests/mir-opt/move-elimination/basic.rs | 80 ++ ...asic.struct_aggregate.MoveElimination.diff | 49 + .../dse.dse_guard.MoveElimination.diff | 92 ++ tests/mir-opt/move-elimination/dse.rs | 36 + ...stant_index_projected.MoveElimination.diff | 41 + ...x_local_not_projected.MoveElimination.diff | 16 + ...overlapping_lifetimes.MoveElimination.diff | 126 +++ ..._fields_not_projected.MoveElimination.diff | 48 + tests/mir-opt/move-elimination/exclusions.rs | 139 +++ ...l_tuple_not_projected.MoveElimination.diff | 72 ++ ...d_field_not_projected.MoveElimination.diff | 30 + ...ubslice_not_projected.MoveElimination.diff | 40 + ...wind.call_destination.MoveElimination.diff | 27 + tests/mir-opt/move-elimination/unwind.rs | 41 + 31 files changed, 2627 insertions(+), 2 deletions(-) create mode 100644 compiler/rustc_mir_transform/src/move_elimination.rs create mode 100644 tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.array_swap.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.enum_alias.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.rs create mode 100644 tests/mir-opt/move-elimination/alias_fixup.simd_alias.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.rs create mode 100644 tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/dse.rs create mode 100644 tests/mir-opt/move-elimination/exclusions.constant_index_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.rs create mode 100644 tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.subslice_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/unwind.rs diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index b7b1531e50857..d0dcb947c6d1c 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -374,6 +374,12 @@ impl SparseIntervalMatrix { self.rows.iter_enumerated() } + pub fn clear_row(&mut self, row: R) { + if let Some(row) = self.rows.get_mut(row) { + row.clear(); + } + } + fn ensure_row(&mut self, row: R) -> &mut IntervalSet { self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size)) } @@ -397,6 +403,16 @@ impl SparseIntervalMatrix { write_row.union(read_row) } + pub fn disjoint_rows(&self, a: R, b: R) -> bool + where + C: Step, + { + let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else { + return true; + }; + a.disjoint(b) + } + pub fn insert_all_into_row(&mut self, row: R) { self.ensure_row(row).insert_all(); } diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 7f9bba03ff345..5ea98a8fb92f5 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -155,7 +155,9 @@ pub(super) struct DestinationPropagation; impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { fn policy(&self, ctx: &crate::PassCtx<'_>) -> PassPolicy { - PassPolicy::optional(ctx.mir_opt_level() >= 2) + PassPolicy::optional( + ctx.mir_opt_level() >= 2 && !ctx.opts.unstable_opts.mir_move_elimination, + ) } #[tracing::instrument(level = "trace", skip(self, tcx, body))] diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 2a38b3b182184..d73d9a82f1a22 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -165,6 +165,7 @@ declare_passes! { mod lower_slice_len : LowerSliceLenCalls; mod match_branches : MatchBranchSimplification; mod mentioned_items : MentionedItems; + mod move_elimination : MoveElimination; mod multiple_return_terminators : MultipleReturnTerminators; mod post_drop_elaboration : CheckLiveDrops; mod prettify : ReorderBasicBlocks, ReorderLocals; @@ -770,6 +771,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' &dead_store_elimination::DeadStoreElimination::Final, &dest_prop::DestinationPropagation, &tail_copy_to_move::TailCopyToMove, + &move_elimination::MoveElimination, &simplify::SimplifyLocals::Final, &multiple_return_terminators::MultipleReturnTerminators, // Some cleanup necessary at least for LLVM and potentially other codegen backends. diff --git a/compiler/rustc_mir_transform/src/move_elimination.rs b/compiler/rustc_mir_transform/src/move_elimination.rs new file mode 100644 index 0000000000000..ac6ddc33578e4 --- /dev/null +++ b/compiler/rustc_mir_transform/src/move_elimination.rs @@ -0,0 +1,912 @@ +//! Eliminates copies and moves by unifying MIR places whose allocation ranges +//! are disjoint. +//! +//! See RFC 3943 for the local-lifetime semantics that make this optimization +//! possible. +//! +//! # Motivation +//! +//! MIR building can insert a lot of redundant copies, and Rust code in general +//! often tends to move values around a lot. The result is a lot of assignments +//! of the form `dest = {move} src;` in MIR. MIR building for constants in +//! particular tends to create additional locals that are only used inside a +//! single block to shuffle a value around unnecessarily. +//! +//! Additionally, Rust constructs nested aggregates by repeatedly moving values +//! into fields. For example, a function may build an inner value in a local, +//! move it into an outer aggregate, then move that aggregate into the caller's +//! destination. If these intermediate source and destination places have +//! different addresses, each layer needs an actual copy or move of the bytes. +//! +//! LLVM cannot remove these copies when both the source and destination +//! addresses are observed because merging the allocations would be an +//! observable change: the program could see that two addresses which were +//! previously distinct have become the same. This pass removes the copies +//! earlier, while MIR still has the information needed to prove that the two +//! allocation ranges do not overlap. +//! +//! # Optimization +//! +//! The basis of this optimization is place unification. If the source and +//! destination of an assignment have the same address, then the assignment is a +//! no-op. The same idea applies to aggregate construction: if a field operand +//! is already located at the corresponding field of the destination, then the +//! aggregate assignment does not need to copy those fields. +//! +//! The pass represents each unification as a mapping from a local to the place +//! that should replace it. Mappings are transitive, so `_3` can be resolved +//! through `_2.1` to `_1.0.1` if earlier mappings established those +//! relationships. +//! +//! The mapping is built by scanning the MIR for assignment statements. For +//! simple `Use` assignments, it tries to unify the source and destination +//! places. For `Aggregate` assignments, it tries to map each field operand to +//! the corresponding field in the assignment destination. Once all mappings +//! have been chosen, they are applied with one rewrite pass over the body. +//! +//! # Constraints +//! +//! Adding a mapping must preserve these conditions: +//! +//! * At least one side of the candidate pair must be a bare local. The pass can +//! map a local to a place with projections, but it cannot map between two +//! places that both already have projections. +//! +//! * Any projections in the mapped place must be stable everywhere the local is +//! used. `Deref` and `Index` projections are rejected because they may refer +//! to different memory at different points in the function. +//! +//! * The allocation ranges of the source and destination places must not +//! overlap. This is checked using `PreciseLiveness`, which computes the +//! points where each local must have a distinct allocation. The non-overlap +//! proof is required so that the operational semantics can allow both places +//! to have the same address. +//! +//! * Special-use locals such as arguments and the return place must keep their +//! roles. Temps may be mapped into an argument or return place, but two +//! special-use locals are not mapped into each other. +//! +//! * Some locals are used in contexts where projections cannot be added, such +//! as `Index` projections. These locals may only be replaced by another bare +//! local. +//! +//! # Storage lifetimes +//! +//! The original storage markers no longer describe the merged locals, so they +//! are removed. Locals without storage markers have storage for the full function. +//! +//! # Aliasing fixup +//! +//! MIR assignments currently require source and destination places not to +//! overlap for types that are not treated as scalars in codegen. After local +//! unification, some assignments may violate that invariant, so a final phase +//! rewrites them into a form codegen can handle. For each assignment: +//! +//! * Self-assignments, such as `_1 = _1`, are deleted. +//! +//! * Simple `Use` assignments whose source and destination overlap but are not +//! identical are routed through a temporary: the source is read into the +//! temporary first, and the temporary is then moved into the destination. +//! +//! * Aggregate assignments with any operand that aliases the destination are +//! decomposed into per-field assignments. Field self-assignments are dropped. +//! Other aliasing fields are read into temporaries first, then all +//! destination fields are written. For enum aggregates the discriminant is +//! set after fields are written. +//! +//! * Other rvalues, such as `Repeat` and `Cast`, are hoisted into a temporary +//! if any place they access aliases the destination. +//! +//! * Rvalues that operate only on scalar types, such as binary and unary ops, +//! `Discriminant`, `Ref`, and `RawPtr`, are left untouched because their +//! codegen does not rely on the no-aliasing assumption. + +use rustc_abi::{ExternAbi, FieldIdx, VariantIdx}; +use rustc_const_eval::util::most_packed_projection; +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, VisitPlacesWith, Visitor}; +use rustc_middle::mir::*; +use rustc_middle::ty::{Ty, TyCtxt}; +use rustc_mir_dataflow::impls::{SplitPointIndex, dump_liveness_matrix, liveness_matrix}; +use rustc_mir_dataflow::points::DenseLocationMap; +use tracing::{debug, trace}; + +use crate::PassPolicy; +use crate::patch::MirPatch; + +pub(super) struct MoveElimination; + +impl<'tcx> crate::MirPass<'tcx> for MoveElimination { + fn policy(&self, ctx: &crate::PassCtx<'_>) -> PassPolicy { + PassPolicy::optional( + ctx.mir_opt_level() >= 2 && ctx.opts.unstable_opts.mir_move_elimination, + ) + } + + #[tracing::instrument(level = "trace", skip(self, tcx, body))] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let def_id = body.source.def_id(); + trace!(?def_id); + + let points = DenseLocationMap::new(body); + let mut liveness_matrix = + liveness_matrix(tcx, body, &points, Some("MoveElimination.liveness")); + + dump_liveness_matrix(tcx, body, "MoveElimination.pre-liveness", &points, &liveness_matrix); + + let unprojectable_locals = UnprojectableLocals::find(body); + trace!(?unprojectable_locals); + + let rust_call_tuples = find_rust_call_tuples(tcx, body); + trace!(?rust_call_tuples); + + let remapped_locals = PlaceUnification::run( + tcx, + body, + &mut liveness_matrix, + unprojectable_locals, + rust_call_tuples, + ); + + apply_mappings(tcx, body, &remapped_locals); + + dump_liveness_matrix(tcx, body, "MoveElimination.post-liveness", &points, &liveness_matrix); + + apply_alias_fixup(tcx, body); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Unprojectable locals + +/// Set of locals which can only be replaced with another local, instead of +/// an arbitrary place. This is usually because it is used directly as a +/// `Local` outside of a place (e.g. `Index` projections). +#[derive(Debug)] +struct UnprojectableLocals { + locals: DenseBitSet, +} + +impl UnprojectableLocals { + fn find(body: &Body<'_>) -> DenseBitSet { + let mut out = Self { locals: DenseBitSet::new_empty(body.local_decls.len()) }; + + // Arguments and return places have fixed roles and cannot be replaced + // with projected locals. + out.locals.insert(RETURN_PLACE); + for arg in body.args_iter() { + out.locals.insert(arg); + } + + out.visit_body(body); + out.locals + } +} + +impl<'tcx> Visitor<'tcx> for UnprojectableLocals { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + // We can't add more projections before a first position Deref projection. + if place.is_indirect() { + trace!( + "unprojectable local {:?} due to use as deref base at {location:?}", + place.local + ); + self.locals.insert(place.local); + } + + // Only call visit_local for projections, not the base local. + self.visit_projection(place.as_ref(), context, location); + } + + fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) { + // Storage markers are removed. + if let PlaceContext::NonUse(NonUseContext::StorageLive | NonUseContext::StorageDead) = + context + { + return; + } + + // If this is reached, it means that this is a bare local used outside + // of a place, which means it cannot be replaced with a projection of + // another local. + trace!("unprojectable local {local:?} at {location:?} ({context:?})"); + self.locals.insert(local); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// "rust-call" tuple handling + +/// Search for tuple locals passed to calls using the "rust-call" ABI. +/// +/// For rust-call ABI calls, caller-side MIR passes the logical arguments as a +/// tuple operand. We want to avoid remapping other locals into fields of that +/// tuple, especially if one of those locals is borrowed. +/// +/// Since the tuple itself is never borrowed, it is trivial for LLVM alias +/// analysis to see that accesses to one argument do not affect the others, but +/// merging the arguments into tuple fields from the start can hide that +/// independence. +fn find_rust_call_tuples<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> DenseBitSet { + let mut rust_call_tuples = DenseBitSet::new_empty(body.local_decls.len()); + + for block in body.basic_blocks.iter() { + let terminator = block.terminator(); + let (func, args) = match &terminator.kind { + TerminatorKind::Call { func, args, .. } + | TerminatorKind::TailCall { func, args, .. } => (func, args), + _ => continue, + }; + + let sig = func.ty(&body.local_decls, tcx).fn_sig(tcx); + if sig.abi() != ExternAbi::RustCall { + continue; + } + + let arg_tuple = args.last().expect("rust-call ABI requires a tuple argument"); + let (Operand::Copy(place) | Operand::Move(place)) = arg_tuple.node else { + continue; + }; + if let Some(local) = place.as_local() { + rust_call_tuples.insert(local); + } + } + + rust_call_tuples +} + +//////////////////////////////////////////////////////////////////////////////// +// Local unification + +struct PlaceUnification<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + body: &'a Body<'tcx>, + liveness_matrix: &'a mut SparseIntervalMatrix, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, + remapped_locals: IndexVec>>, +} + +impl<'tcx> PlaceUnification<'_, 'tcx> { + fn run( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + liveness_matrix: &mut SparseIntervalMatrix, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, + ) -> IndexVec>> { + let mut visitor = PlaceUnification { + tcx, + body, + liveness_matrix, + unprojectable_locals, + rust_call_tuples, + remapped_locals: IndexVec::from_elem_n(None, body.local_decls.len()), + }; + visitor.visit_body(body); + + // Finalize the mappings by transitively resolving all locals to their + // new final place. + for local in visitor.remapped_locals.indices() { + if let Some(place) = visitor.remapped_locals[local] { + let place = visitor.resolve_place(place); + visitor.remapped_locals[local] = Some(place); + trace!("Remapped {local:?} to {place:?}"); + } + } + + visitor.remapped_locals + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn resolve_place(&self, mut place: Place<'tcx>) -> Place<'tcx> { + while let Some(new_place) = self.remapped_locals[place.local] { + place = new_place.project_deeper(place.projection, self.tcx); + } + place + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn can_unify_places(&self, a: Place<'tcx>, b: Place<'tcx>) -> Option<(Local, Place<'tcx>)> { + let a = self.resolve_place(a); + let b = self.resolve_place(b); + + if a.local == b.local { + if a.projection != b.projection { + trace!("cannot unify same local with different projections"); + } + return None; + } + + if self.rust_call_tuples.contains(a.local) || self.rust_call_tuples.contains(b.local) { + trace!("cannot unify {a:?} and {b:?} involving a rust-call tuple argument"); + return None; + } + + let (local, place) = match (a.as_local(), b.as_local()) { + (None, None) => { + trace!("cannot unify 2 places that both have projections"); + return None; + } + (None, Some(b)) => { + if self.unprojectable_locals.contains(b) { + trace!("cannot unify {b:?} which cannot be projected"); + return None; + } + (b, a) + } + (Some(a), None) => { + if self.unprojectable_locals.contains(a) { + trace!("cannot unify {a:?} which cannot be projected"); + return None; + } + (a, b) + } + (Some(a), Some(b)) => match (self.body.local_kind(a), self.body.local_kind(b)) { + ( + LocalKind::Arg | LocalKind::ReturnPointer, + LocalKind::Arg | LocalKind::ReturnPointer, + ) => { + trace!("cannot unify {a:?} and {b:?} which are both arguments or return place"); + return None; + } + (LocalKind::Arg | LocalKind::ReturnPointer, LocalKind::Temp) => (b, a.into()), + (LocalKind::Temp, _) => (a, b.into()), + }, + }; + + if most_packed_projection(self.tcx, &self.body.local_decls, place).is_some() { + trace!("cannot unify {place:?} which has packed field projections"); + return None; + } + + if !self.liveness_matrix.disjoint_rows(local, place.local) { + trace!("cannot unify {a:?} and {b:?} which have overlapping live ranges"); + return None; + } + + // FIXME(#112651): This can be removed afterwards. + let local_ty = self.body.local_decls[local].ty; + let place_ty = place.ty(&self.body.local_decls, self.tcx).ty; + if local_ty != place_ty { + trace!( + "cannot unify {a:?} and {b:?} which have different types due to subtyping ({local_ty:?} vs {place_ty:?})" + ); + return None; + } + + Some((local, place)) + } + + #[tracing::instrument(level = "trace", skip(self))] + fn remap_local(&mut self, local: Local, place: Place<'tcx>) { + debug_assert!(self.remapped_locals[local].is_none()); + self.remapped_locals[local] = Some(place); + + self.liveness_matrix.union_rows(local, place.local); + self.liveness_matrix.clear_row(local); + + // If the original local was unprojectable then this now also applies to + // the mapped local. + if self.unprojectable_locals.contains(local) { + debug_assert!(place.projection.is_empty()); + self.unprojectable_locals.insert(place.local); + } + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + location: Location, + ) { + // Attempt to unify each field operand with the corresponding field in + // the destination place. + let mut candidates = vec![]; + for (idx, operand) in operands.iter_enumerated() { + let (Operand::Copy(src) | Operand::Move(src)) = *operand else { + continue; + }; + let Some(src) = src.as_local() else { + continue; + }; + let dest = project_field(self.tcx, dest, idx, self.body.local_decls[src].ty); + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(dest, src.into()) { + candidates.push((local, place)); + } + } + + // Do the actual remapping *after* checking for live range overlaps. + // + // If we tried to do this immediately after `can_unify_places` for each + // operand then this would only allow unifying with one operand: once + // the first operand's live range is unified with the destination, all + // remaining inputs conflict with the combined live range. + // + // By deferring remapping until all inputs are checked, we can unify + // with all eligible inputs instead of just one. This is sound because + // they will each be mapped to a disjoint field of the destination + // place. + for (local, place) in candidates { + // Repeated operands can produce multiple candidates for the same + // local. Only keep the first mapping. + if self.remapped_locals[local].is_some() { + continue; + } + self.remap_local(local, place); + } + } +} + +/// Since we are replacing all uses of a local with another place, we need to +/// ensure that the projections on that place are stable no matter where it is +/// used in the body. Additional this local may be used in debuginfo, so ensure +/// that the projections are compatible with usage in debuginfo. +fn check_projections(place: Place<'_>) -> bool { + place.projection.iter().all(|elem| elem.is_stable_offset() && elem.can_use_in_debuginfo()) +} + +impl<'tcx> Visitor<'tcx> for PlaceUnification<'_, 'tcx> { + fn visit_assign(&mut self, dest: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { + if !check_projections(*dest) { + return; + } + match rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), _) => { + if !check_projections(*src) { + return; + } + + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(*src, *dest) { + self.remap_local(local, place); + } + } + Rvalue::Aggregate(aggregate_kind, operands) => match *aggregate_kind { + AggregateKind::Array(_) => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + location, + ), + AggregateKind::Tuple => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + location, + ), + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + location, + ) + } + AggregateKind::Adt(adt_did, var_idx, _, _, None) => { + let def = self.tcx.adt_def(adt_did); + if def.repr().simd() { + // MCP#838 banned projections into SIMD types. + return; + } + self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + location, + ) + } + _ => {} + }, + _ => {} + }; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Apply place mappings to the MIR body. + +fn apply_mappings<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + remapped_locals: &IndexVec>>, +) { + let mut rewriter = PlaceUpdater { tcx, remapped_locals }; + rewriter.visit_body_preserves_cfg(body); +} + +struct PlaceUpdater<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + remapped_locals: &'a IndexVec>>, +} + +impl<'tcx> MutVisitor<'tcx> for PlaceUpdater<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_local(&mut self, local: &mut Local, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[*local] { + trace!("replacing {local:?} with {new_place:?} at {location:?} ({context:?})"); + *local = new_place.as_local().expect("mapped place shouldn't have projections"); + } + } + + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[place.local] { + trace!("replacing {place:?} with {new_place:?} at {location:?} ({context:?})"); + *place = new_place.project_deeper(place.projection, self.tcx) + } + + // Only call visit_local for projections, not the base local. + if let Some(new_projection) = self.process_projection(&place.projection, location) { + place.projection = self.tcx().mk_place_elems(&new_projection); + } + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + match statement.kind { + // Remove storage lifetime markers because they no longer describe + // the merged locals. The later storage reconstruction pass rebuilds + // these markers for all locals, which can also shorten the storage + // lifetimes of locals that were not merged. + StatementKind::StorageDead(_) | StatementKind::StorageLive(_) => { + statement.make_nop(true); + return; + } + _ => {} + } + + self.super_statement(statement, location); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Aliasing assignment fixup +// +// MIR assignments currently do not allow source and destination to alias, so +// fix this in post-processing. + +fn apply_alias_fixup<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let mut patcher = MirPatch::new(body); + let mut fixup = AliasFixup { tcx, local_decls: &body.local_decls, patcher: &mut patcher }; + for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { + fixup.visit_basic_block_data(block, data); + } + patcher.apply(body); +} + +/// Returns whether 2 places alias, ignoring indirect places. +fn places_directly_alias<'tcx>( + tcx: TyCtxt<'tcx>, + local_decls: &IndexVec>, + a: Place<'tcx>, + b: Place<'tcx>, +) -> bool { + // This function doesn't handle indirect aliasing. + if a.local != b.local || a.is_indirect_first_projection() || b.is_indirect_first_projection() { + return false; + } + + for ((prefix, elem_a), (_, elem_b)) in a.iter_projections().zip(b.iter_projections()) { + // Continue until we find the first mismatching projection. + if elem_a == elem_b { + continue; + } + + match (elem_a, elem_b) { + // Disjoint fields don't alias except if they are union fields. + (PlaceElem::Field(_, _), PlaceElem::Field(_, _)) => { + let ty = prefix.ty(local_decls, tcx).ty; + return ty.is_union(); + } + + // Disjoint slice elements don't alias. + ( + PlaceElem::ConstantIndex { offset: offset_a, min_length: _, from_end: from_end_a }, + PlaceElem::ConstantIndex { offset: offset_b, min_length: _, from_end: from_end_b }, + ) if from_end_a == from_end_b && offset_a != offset_b => { + return false; + } + + // Conservatively assume the places may alias. + _ => return true, + } + } + + // If the projections are identical *or* one is a prefix of the other then + // the places alias. + true +} + +struct AliasFixup<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + local_decls: &'a IndexVec>, + patcher: &'a mut MirPatch<'tcx>, +} + +impl<'tcx> AliasFixup<'_, 'tcx> { + fn isolate_rvalue_to_local( + &mut self, + rvalue: Rvalue<'tcx>, + source_info: SourceInfo, + location: Location, + ) -> Place<'tcx> { + let ty = rvalue.ty(self.local_decls, self.tcx); + let temp = Place::from(self.patcher.new_temp(ty, source_info.span)); + trace!("isolating {rvalue:?} to {temp:?} due to conflict"); + self.patcher.add_statement(location, StatementKind::StorageLive(temp.local)); + self.patcher.add_assign(location, Place::from(temp), rvalue); + self.patcher.add_statement( + location.successor_within_block(), + StatementKind::StorageDead(temp.local), + ); + temp + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + enum_variant: Option, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + source_info: SourceInfo, + location: Location, + ) { + // Fast path: if no direct operand aliases the destination, we're done. + // + // We only look for direct aliases here, which is sufficient because we + // know the input MIR did not have any aliasing and we didn't introduce + // any indirect aliasing in this pass. + // + // If the destination place is indirect then it cannot be the start of + // a lifetime as per the RFC 3943 MIR semantics. This means that the + // lifetime of the underlying allocation must have started earlier, + // which overlaps the early point of the assignment statement. Therefore + // we couldn't have unified any source operand with this destination + // place. + // + // If the source place is indirect then a similar reasoning applies. The + // only exception is if there are multiple source places (e.g. + // aggregates). In that situation it's possible for an indirect source + // to overlap the destination if and only if there is also a direct + // source that overlaps it: + // + // _2 = &_1 + // _3 = (copy *_2, move _1) // _1 becomes _3.1 after unification + // + // We handle this here in 2 ways: if there is no direct alias, then + // we're fine. Otherwise, treat all indirect sources as potentially + // aliasing with the destination operand. + let has_direct_alias = operands.iter().any(|op| match op { + Operand::Copy(src) | Operand::Move(src) => { + places_directly_alias(self.tcx, self.local_decls, dest, *src) + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => false, + }); + if !has_direct_alias { + return; + } + + debug!("splitting aggregate assignment at {location:?}"); + + // Split into per-field assignments. + let mut assignments = vec![]; + for (idx, op) in operands.iter_enumerated() { + let field_ty = op.ty(self.local_decls, self.tcx); + let dest_field = project_field(self.tcx, dest, idx, field_ty); + + let emit_op = match op { + Operand::Copy(src) | Operand::Move(src) => { + if *src == dest_field { + // Skip identity assignments. + continue; + } else if src.is_indirect_first_projection() + || places_directly_alias(self.tcx, self.local_decls, dest, *src) + { + // Partial alias: hoist the source to a temp first so the + // per-field write no longer overlaps the dest. Indirect + // sources also need hoisting here because they may point + // at one of the direct aliasing operands. + Operand::Move(self.isolate_rvalue_to_local( + Rvalue::Use(op.clone(), WithRetag::No), + source_info, + location, + )) + } else { + op.clone() + } + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => op.clone(), + }; + assignments.push((dest_field, emit_op)); + } + + // Perform assignments *after* all aliasing fields have been read into + // temporary locals. + for (dest_field, emit_op) in assignments { + self.patcher.add_assign(location, dest_field, Rvalue::Use(emit_op, WithRetag::No)); + } + + // Delete the original aggregate assignment. + self.patcher.nop_statement(location); + + // For enum variants, set the discriminant after all field writes. + if let Some(variant_index) = enum_variant { + self.patcher.add_statement( + location, + StatementKind::SetDiscriminant { place: Box::new(dest), variant_index }, + ); + } + } +} + +impl<'tcx> MutVisitor<'tcx> for AliasFixup<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + // Fixup the MIR to remove aliasing assignments. + if let StatementKind::Assign((dest, rvalue)) = &mut statement.kind { + match *rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), with_retag) => { + if places_directly_alias(self.tcx, self.local_decls, *dest, src) { + if src == *dest { + debug!("{:?} turned into self-assignment, deleting", location); + statement.make_nop(true); + } else { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), with_retag); + } + } + } + Rvalue::Aggregate(AggregateKind::Array(_), ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate(AggregateKind::Tuple, ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate( + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)), + ref mut operands, + ) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + None, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ) + } + // MCP#838 banned projections into SIMD types, so use the + // general aliasing fallback below for those aggregates. + Rvalue::Aggregate( + AggregateKind::Adt(adt_did, var_idx, _, _, None), + ref mut operands, + ) if !self.tcx.adt_def(adt_did).repr().simd() => { + let def = self.tcx.adt_def(adt_did); + self.visit_aggregate_assign( + *dest, + def.is_enum().then_some(var_idx), + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + statement.source_info, + location, + ) + } + + // For other rvalues, don't try to split them into components + // and instead just introduce a temporary if there is any + // aliasing + Rvalue::Aggregate(..) + | Rvalue::Repeat(..) + | Rvalue::Cast(..) + | Rvalue::CopyForDeref(..) + | Rvalue::WrapUnsafeBinder(..) => { + let mut overlaps_dest = false; + VisitPlacesWith(|place, _ctxt| { + if places_directly_alias(self.tcx, self.local_decls, *dest, place) { + overlaps_dest = true; + } + }) + .visit_rvalue(rvalue, location); + if overlaps_dest { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), WithRetag::No); + } + } + + // These either cannot have aliasing, or allow it because they + // only operate on scalar backend types. + Rvalue::Use(Operand::Constant(..) | Operand::RuntimeChecks(..), _) + | Rvalue::Ref(..) + | Rvalue::ThreadLocalRef(..) + | Rvalue::BinaryOp(..) + | Rvalue::UnaryOp(..) + | Rvalue::Discriminant(..) + | Rvalue::RawPtr(..) + | Rvalue::Reborrow(..) => {} + } + } + } +} diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index e6a9d5e5ebd49..abefc1c9767db 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2690,7 +2690,7 @@ options! { "include extra comments in mir pretty printing, like line numbers and statement indices, \ details about types, etc. (boolean for all passes, 'nll' to enable in NLL MIR only, default: 'nll')"), mir_move_elimination: bool = (false, parse_bool, [TRACKED], - "enable the experimental MIR move elimination pass (default: no)"), + "enable the experimental MIR semantics for the move elimination optimization (default: no)"), mir_opt_bisect_limit: Option = (None, parse_opt_number, [TRACKED], "limit the number of MIR optimization pass executions (global across all bodies). \ Pass executions after this limit are skipped and reported. (default: no limit)"), diff --git a/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff new file mode 100644 index 0000000000000..1b4e6dbba1825 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff @@ -0,0 +1,27 @@ +- // MIR for `aggregate_indirect_source_alias` before MoveElimination ++ // MIR for `aggregate_indirect_source_alias` after MoveElimination + + fn aggregate_indirect_source_alias() -> (u8, u8) { + let mut _0: (u8, u8); + let mut _1: u8; + let mut _2: *const u8; + let mut _3: (u8, u8); ++ let mut _4: u8; + + bb0: { +- _1 = const 1_u8; +- _2 = &raw const _1; +- _3 = (copy (*_2), move _1); +- _0 = copy _3; ++ (_0.1: u8) = const 1_u8; ++ _2 = &raw const (_0.1: u8); ++ StorageLive(_4); ++ _4 = no_retag copy (*_2); ++ (_0.0: u8) = no_retag move _4; ++ nop; ++ StorageDead(_4); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff new file mode 100644 index 0000000000000..d958b6d346d35 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff @@ -0,0 +1,84 @@ +- // MIR for `aggregate_swap` before MoveElimination ++ // MIR for `aggregate_swap` after MoveElimination + + fn aggregate_swap(_1: u8, _2: u8) -> (u8, u8) { + debug x => _1; + debug y => _2; + let mut _0: (u8, u8); + let mut _3: (u8, u8); + let mut _4: u8; + let mut _5: u8; + let mut _8: u8; + let mut _9: u8; ++ let mut _10: u8; + scope 1 { +- debug pair => _3; ++ debug pair => _0; + let _6: u8; + scope 2 { +- debug a => _6; ++ debug a => _9; + let _7: u8; + scope 3 { +- debug b => _7; ++ debug b => (_0.1: u8); + } + } + } + + bb0: { +- StorageLive(_3); +- StorageLive(_4); +- _4 = copy _1; +- StorageLive(_5); +- _5 = copy _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- StorageLive(_6); +- _6 = copy (_3.0: u8); +- StorageLive(_7); +- _7 = copy (_3.1: u8); +- StorageLive(_8); +- _8 = copy _7; +- StorageLive(_9); +- _9 = copy _6; +- _3 = (move _8, move _9); +- StorageDead(_9); +- StorageDead(_8); +- _0 = copy _3; +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = (move _1, move _2); ++ nop; ++ nop; ++ nop; ++ _9 = copy (_0.0: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_10); ++ _10 = no_retag move (_0.1: u8); ++ (_0.0: u8) = no_retag move _10; ++ (_0.1: u8) = no_retag move _9; ++ nop; ++ StorageDead(_10); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.array_swap.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.array_swap.MoveElimination.diff new file mode 100644 index 0000000000000..f577fe6e46955 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.array_swap.MoveElimination.diff @@ -0,0 +1,55 @@ +- // MIR for `array_swap` before MoveElimination ++ // MIR for `array_swap` after MoveElimination + + fn array_swap(_1: [u8; 2]) -> [u8; 2] { + debug array => _1; + let mut _0: [u8; 2]; + let _2: u8; + let _3: u8; + let mut _4: u8; + let mut _5: u8; ++ let mut _6: u8; + scope 1 { +- debug a => _2; +- debug b => _3; ++ debug a => _5; ++ debug b => _1[1 of 2]; + } + + bb0: { +- StorageLive(_2); +- _2 = copy _1[0 of 2]; +- StorageLive(_3); +- _3 = copy _1[1 of 2]; +- StorageLive(_4); +- _4 = copy _3; +- StorageLive(_5); +- _5 = copy _2; +- _1 = [move _4, move _5]; +- StorageDead(_5); +- StorageDead(_4); ++ nop; ++ _5 = copy _1[0 of 2]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_6); ++ _6 = no_retag move _1[1 of 2]; ++ _1[0 of 1] = no_retag move _6; ++ _1[1 of 2] = no_retag move _5; ++ nop; ++ StorageDead(_6); ++ nop; ++ nop; + _0 = copy _1; +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.enum_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.enum_alias.MoveElimination.diff new file mode 100644 index 0000000000000..0997f614ac55e --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.enum_alias.MoveElimination.diff @@ -0,0 +1,120 @@ +- // MIR for `enum_alias` before MoveElimination ++ // MIR for `enum_alias` after MoveElimination + + fn enum_alias(_1: bool, _2: u8, _3: u8) -> PairEnum { + debug flag => _1; + debug x => _2; + debug y => _3; + let mut _0: PairEnum; + let _4: PairEnum; + let mut _5: u8; + let mut _6: u8; + let mut _7: bool; + let mut _8: isize; + let mut _11: u8; + let mut _12: u8; ++ let mut _13: u8; + scope 1 { +- debug input => _4; ++ debug input => _0; + let _9: u8; + let _10: u8; + scope 2 { +- debug a => _9; +- debug b => _10; ++ debug a => _12; ++ debug b => ((_0 as Pair).1: u8); + } + } + + bb0: { +- StorageLive(_4); +- StorageLive(_5); +- _5 = copy _2; +- StorageLive(_6); +- _6 = copy _3; +- _4 = PairEnum::Pair(move _5, move _6); +- StorageDead(_6); +- StorageDead(_5); +- StorageLive(_7); +- _7 = copy _1; +- switchInt(move _7) -> [0: bb2, otherwise: bb1]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = PairEnum::Pair(move _2, move _3); ++ nop; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb2, otherwise: bb1]; + } + + bb1: { +- _0 = move _4; ++ nop; + goto -> bb6; + } + + bb2: { +- _8 = discriminant(_4); ++ _8 = discriminant(_0); + switchInt(move _8) -> [0: bb4, 1: bb5, otherwise: bb3]; + } + + bb3: { + unreachable; + } + + bb4: { + _0 = PairEnum::Empty; + goto -> bb6; + } + + bb5: { +- StorageLive(_9); +- _9 = copy ((_4 as Pair).0: u8); +- StorageLive(_10); +- _10 = copy ((_4 as Pair).1: u8); +- StorageLive(_11); +- _11 = copy _10; +- StorageLive(_12); +- _12 = copy _9; +- _0 = PairEnum::Pair(move _11, move _12); +- StorageDead(_12); +- StorageDead(_11); +- StorageDead(_10); +- StorageDead(_9); ++ nop; ++ _12 = copy ((_0 as Pair).0: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_13); ++ _13 = no_retag move ((_0 as Pair).1: u8); ++ ((_0 as variant#1).0: u8) = no_retag move _13; ++ ((_0 as variant#1).1: u8) = no_retag move _12; ++ discriminant(_0) = 1; ++ nop; ++ StorageDead(_13); ++ nop; ++ nop; ++ nop; ++ nop; + goto -> bb6; + } + + bb6: { +- StorageDead(_7); +- StorageDead(_4); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff new file mode 100644 index 0000000000000..0b3f73be637a0 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff @@ -0,0 +1,90 @@ +- // MIR for `mixed_aggregate_aliasing` before MoveElimination ++ // MIR for `mixed_aggregate_aliasing` after MoveElimination + + fn mixed_aggregate_aliasing(_1: bool, _2: u8) -> Triple { + debug flag => _1; + debug z => _2; + let mut _0: Triple; + let _3: Triple; + let mut _5: bool; + let mut _6: u8; + let mut _7: u8; + let mut _8: u8; ++ let mut _9: u8; + scope 1 { +- debug input => _3; ++ debug input => _0; + let _4: Triple; + scope 2 { +- debug out => _4; ++ debug out => _0; + } + } + + bb0: { +- StorageLive(_3); +- _3 = opaque_triple() -> [return: bb1, unwind unreachable]; ++ nop; ++ _0 = opaque_triple() -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_4); +- StorageLive(_5); +- _5 = copy _1; +- switchInt(move _5) -> [0: bb3, otherwise: bb2]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb2]; + } + + bb2: { +- _4 = copy _3; ++ nop; + goto -> bb4; + } + + bb3: { +- StorageLive(_6); +- _6 = copy (_3.1: u8); +- StorageLive(_7); +- _7 = copy (_3.0: u8); +- StorageLive(_8); +- _8 = copy _2; +- _4 = Triple(move _6, move _7, move _8); +- StorageDead(_8); +- StorageDead(_7); +- StorageDead(_6); ++ nop; ++ _6 = copy (_0.1: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_9); ++ _9 = no_retag move (_0.0: u8); ++ (_0.0: u8) = no_retag move _6; ++ (_0.1: u8) = no_retag move _9; ++ (_0.2: u8) = no_retag move _2; ++ nop; ++ StorageDead(_9); ++ nop; ++ nop; ++ nop; + goto -> bb4; + } + + bb4: { +- StorageDead(_5); +- _0 = copy _4; +- StorageDead(_4); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.rs b/tests/mir-opt/move-elimination/alias_fixup.rs new file mode 100644 index 0000000000000..080b076a97498 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.rs @@ -0,0 +1,156 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![feature(core_intrinsics, custom_mir, repr_simd)] +#![allow(dead_code)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +#[derive(Copy, Clone)] +pub struct Triple(u8, u8, u8); + +pub union U { + a: [u8; 4], + b: [u8; 4], +} + +unsafe extern "C" { + safe fn opaque_triple() -> Triple; +} + +// EMIT_MIR alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff +pub fn mixed_aggregate_aliasing(flag: bool, z: u8) -> Triple { + // This checks an aggregate assignment on one branch after the other branch + // remaps the input into the return place: overlapping field reads are + // hoisted through temporaries before writing back into the return place. + // CHECK-LABEL: fn mixed_aggregate_aliasing( + // CHECK: debug z => _2; + // CHECK: debug input => _0; + // CHECK: debug out => _0; + // CHECK: [[field1:_.*]] = copy (_0.1: u8); + // CHECK: [[field0:_.*]] = no_retag move (_0.0: u8); + // CHECK: (_0.0: u8) = no_retag move [[field1]]; + // CHECK: (_0.1: u8) = no_retag move [[field0]]; + // CHECK: (_0.2: u8) = no_retag move _2; + let input = opaque_triple(); + let out = if flag { input } else { Triple(input.1, input.0, z) }; + out +} + +// EMIT_MIR alias_fixup.aggregate_swap.MoveElimination.diff +pub fn aggregate_swap(x: u8, y: u8) -> (u8, u8) { + // This checks that an aggregate swap-like assignment is safe after any + // remapping that makes source fields share storage with destination fields. + // CHECK-LABEL: fn aggregate_swap( + // CHECK: debug pair => _0; + // CHECK: [[saved:_.*]] = copy (_0.0: u8); + // CHECK: [[tmp:_.*]] = no_retag move (_0.1: u8); + // CHECK: (_0.0: u8) = no_retag move [[tmp]]; + // CHECK: (_0.1: u8) = no_retag move [[saved]]; + let mut pair = (x, y); + let a = pair.0; + let b = pair.1; + pair = (b, a); + pair +} + +// EMIT_MIR alias_fixup.simple_partial_alias.MoveElimination.diff +pub fn simple_partial_alias(x: [u8; 4]) -> U { + // This checks a non-aggregate assignment involving two same-typed union + // fields, which are conservatively treated as aliasing. + // CHECK-LABEL: fn simple_partial_alias( + // CHECK: debug u => _0; + // CHECK: _0 = U { a: move _1 }; + // CHECK: [[tmp:_.*]] = copy (_0.0: [u8; 4]); + // CHECK: (_0.1: [u8; 4]) = move [[tmp]]; + let mut u = U { a: x }; + let tmp = unsafe { u.a }; + u.b = tmp; + u +} + +// EMIT_MIR alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn aggregate_indirect_source_alias() -> (u8, u8) { + // This checks that, when an aggregate has a direct operand aliasing the + // destination, indirect operands are also hoisted before field writes. + // CHECK-LABEL: fn aggregate_indirect_source_alias( + // CHECK: [[p:_.*]] = &raw const (_0.1: u8); + // CHECK: [[tmp:_.*]] = no_retag copy (*[[p]]); + // CHECK: (_0.0: u8) = no_retag move [[tmp]]; + mir! { + let a: u8; + let p: *const u8; + let out: (u8, u8); + + { + a = 1_u8; + p = &raw const a; + out = (*p, Move(a)); + RET = out; + Return() + } + } +} + +// Aliasing array elements must all be read before any element is overwritten. +// EMIT_MIR alias_fixup.array_swap.MoveElimination.diff +pub fn array_swap(mut array: [u8; 2]) -> [u8; 2] { + // CHECK-LABEL: fn array_swap( + // CHECK: debug array => _1; + // CHECK: [[FIRST:_[0-9]+]] = copy _1[0 of 2]; + // CHECK: [[SECOND:_[0-9]+]] = no_retag move _1[1 of 2]; + // CHECK: _1[0 of 1] = no_retag move [[SECOND]]; + // CHECK: _1[1 of 2] = no_retag move [[FIRST]]; + let [a, b] = array; + array = [b, a]; + array +} + +pub enum PairEnum { + Empty, + Pair(u8, u8), +} + +// Enum alias fixup must read both fields before writing the swapped payload, +// then set the discriminant after the field writes. +// EMIT_MIR alias_fixup.enum_alias.MoveElimination.diff +pub fn enum_alias(flag: bool, x: u8, y: u8) -> PairEnum { + // CHECK-LABEL: fn enum_alias( + // CHECK: debug input => _0; + // CHECK: [[FIRST:_[0-9]+]] = copy ((_0 as Pair).0: u8); + // CHECK: [[SECOND:_[0-9]+]] = no_retag move ((_0 as Pair).1: u8); + // CHECK: ((_0 as variant#1).0: u8) = no_retag move [[SECOND]]; + // CHECK: ((_0 as variant#1).1: u8) = no_retag move [[FIRST]]; + // CHECK-NEXT: discriminant(_0) = 1; + let input = PairEnum::Pair(x, y); + if flag { + input + } else { + match input { + PairEnum::Pair(a, b) => PairEnum::Pair(b, a), // swap + PairEnum::Empty => PairEnum::Empty, + } + } +} + +#[repr(simd)] +#[derive(Copy, Clone)] +pub struct Vector([u32; 4]); + +pub union VectorUnion { + vector: Vector, + array: [u32; 4], +} + +// SIMD aggregates cannot be replaced with direct access to the inner field. +// EMIT_MIR alias_fixup.simd_alias.MoveElimination.diff +pub unsafe fn simd_alias(flag: bool) -> VectorUnion { + // CHECK-LABEL: fn simd_alias( + // CHECK: debug input => _0; + // CHECK: [[TEMP:_[0-9]+]] = Vector(move (_0.1: [u32; 4])); + // CHECK-NEXT: (_0.0: Vector) = no_retag move [[TEMP]]; + let input = VectorUnion { vector: Vector([0; 4]) }; + if flag { VectorUnion { vector: Vector(input.array) } } else { input } +} diff --git a/tests/mir-opt/move-elimination/alias_fixup.simd_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.simd_alias.MoveElimination.diff new file mode 100644 index 0000000000000..7c2b4ebfb7fca --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.simd_alias.MoveElimination.diff @@ -0,0 +1,78 @@ +- // MIR for `simd_alias` before MoveElimination ++ // MIR for `simd_alias` after MoveElimination + + fn simd_alias(_1: bool) -> VectorUnion { + debug flag => _1; + let mut _0: VectorUnion; + let _2: VectorUnion; + let mut _3: Vector; + let mut _4: [u32; 4]; + let mut _5: bool; + let mut _6: Vector; + let mut _7: [u32; 4]; ++ let mut _8: Vector; + scope 1 { +- debug input => _2; ++ debug input => _0; + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- StorageLive(_4); ++ nop; ++ nop; ++ nop; + _4 = [const 0_u32; 4]; +- _3 = Vector(move _4); +- StorageDead(_4); +- _2 = VectorUnion { vector: move _3 }; +- StorageDead(_3); +- StorageLive(_5); +- _5 = copy _1; +- switchInt(move _5) -> [0: bb2, otherwise: bb1]; ++ (_0.0: Vector) = Vector(move _4); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb2, otherwise: bb1]; + } + + bb1: { +- StorageLive(_6); +- StorageLive(_7); +- _7 = copy (_2.1: [u32; 4]); +- _6 = Vector(move _7); +- StorageDead(_7); +- _0 = VectorUnion { vector: move _6 }; +- StorageDead(_6); ++ nop; ++ nop; ++ nop; ++ StorageLive(_8); ++ _8 = Vector(move (_0.1: [u32; 4])); ++ (_0.0: Vector) = no_retag move _8; ++ StorageDead(_8); ++ nop; ++ nop; ++ nop; + goto -> bb3; + } + + bb2: { +- _0 = move _2; ++ nop; + goto -> bb3; + } + + bb3: { +- StorageDead(_5); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff new file mode 100644 index 0000000000000..e548e921f3142 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff @@ -0,0 +1,52 @@ +- // MIR for `simple_partial_alias` before MoveElimination ++ // MIR for `simple_partial_alias` after MoveElimination + + fn simple_partial_alias(_1: [u8; 4]) -> U { + debug x => _1; + let mut _0: U; + let mut _2: U; + let mut _3: [u8; 4]; + let mut _5: [u8; 4]; + scope 1 { +- debug u => _2; ++ debug u => _0; + let _4: [u8; 4]; + scope 2 { +- debug tmp => _4; ++ debug tmp => _5; + } + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = U { a: move _3 }; +- StorageDead(_3); +- StorageLive(_4); +- _4 = copy (_2.0: [u8; 4]); +- StorageLive(_5); +- _5 = copy _4; +- (_2.1: [u8; 4]) = move _5; +- StorageDead(_5); +- _0 = move _2; +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ _0 = U { a: move _1 }; ++ nop; ++ nop; ++ _5 = copy (_0.0: [u8; 4]); ++ nop; ++ nop; ++ (_0.1: [u8; 4]) = move _5; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..01938d10da6c5 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff @@ -0,0 +1,67 @@ +- // MIR for `array_aggregate` before MoveElimination ++ // MIR for `array_aggregate` after MoveElimination + + fn array_aggregate() -> [[u8; 8]; 3] { + let mut _0: [[u8; 8]; 3]; + let _1: [u8; 8]; + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + let mut _6: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _0[0 of 1]; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _0[1 of 2]; + let _3: [u8; 8]; + scope 3 { +- debug c => _3; ++ debug c => _0[2 of 3]; + } + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = [const 3_u8; 8]; +- StorageLive(_4); +- _4 = move _1; +- StorageLive(_5); +- _5 = move _2; +- StorageLive(_6); +- _6 = move _3; +- _0 = [move _4, move _5, move _6]; +- StorageDead(_6); +- StorageDead(_5); +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ _0[0 of 1] = [const 1_u8; 8]; ++ nop; ++ _0[1 of 2] = [const 2_u8; 8]; ++ nop; ++ _0[2 of 3] = [const 3_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..5f40d8a3c82fd --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff @@ -0,0 +1,57 @@ +- // MIR for `enum_aggregate` before MoveElimination ++ // MIR for `enum_aggregate` after MoveElimination + + fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + let mut _0: std::result::Result<([u8; 8], [u8; 8]), ()>; + let _1: [u8; 8]; + let mut _3: ([u8; 8], [u8; 8]); + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- StorageLive(_4); +- _4 = move _1; +- StorageLive(_5); +- _5 = move _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- _0 = Result::<([u8; 8], [u8; 8]), ()>::Ok(move _3); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ discriminant(_0) = 0; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff new file mode 100644 index 0000000000000..6762e06029772 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff @@ -0,0 +1,46 @@ +- // MIR for `nrvo_borrowed` before MoveElimination ++ // MIR for `nrvo_borrowed` after MoveElimination + + fn nrvo_borrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let mut _1: [u8; 8]; + let _2: (); + let mut _3: &mut [u8; 8]; + let mut _4: &mut [u8; 8]; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- StorageLive(_3); +- StorageLive(_4); +- _4 = &mut _1; ++ nop; ++ _0 = [const 1_u8; 8]; ++ nop; ++ nop; ++ nop; ++ _4 = &mut _0; + _3 = &mut (*_4); + _2 = init(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); +- StorageDead(_4); +- StorageDead(_2); +- _0 = move _1; +- StorageDead(_1); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff new file mode 100644 index 0000000000000..7a44d4b598211 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff @@ -0,0 +1,24 @@ +- // MIR for `nrvo_unborrowed` before MoveElimination ++ // MIR for `nrvo_unborrowed` after MoveElimination + + fn nrvo_unborrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let _1: [u8; 8]; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- _0 = move _1; +- StorageDead(_1); ++ nop; ++ _0 = [const 1_u8; 8]; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.rs b/tests/mir-opt/move-elimination/basic.rs new file mode 100644 index 0000000000000..96f1776bebef1 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.rs @@ -0,0 +1,80 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+TailCopyToMove + +struct Pair { + a: [u8; 8], + b: [u8; 8], +} + +fn init(_: &mut [u8; 8]) {} + +// EMIT_MIR basic.nrvo_unborrowed.MoveElimination.diff +pub fn nrvo_unborrowed() -> [u8; 8] { + // This checks the simplest NRVO-style case: the local should be merged with + // the return place even though it has `Copy` type. + // CHECK-LABEL: fn nrvo_unborrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = [const 1_u8; 8] + let buf = [1; 8]; + buf +} + +// EMIT_MIR basic.nrvo_borrowed.MoveElimination.diff +pub fn nrvo_borrowed() -> [u8; 8] { + // This checks that taking a temporary mutable borrow does not prevent + // merging a `Copy` local once the borrow has ended. + // CHECK-LABEL: fn nrvo_borrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = [const 1_u8; 8] + // CHECK: init(move {{_.*}}) + // CHECK-NOT: _0 = move + let mut buf = [1; 8]; + init(&mut buf); + buf +} + +// EMIT_MIR basic.struct_aggregate.MoveElimination.diff +pub fn struct_aggregate() -> Pair { + // This checks aggregate field remapping: the field locals can live directly + // in the return place's fields. + // CHECK-LABEL: fn struct_aggregate( + // CHECK: debug a => (_0.0: [u8; 8]); + // CHECK: debug b => (_0.1: [u8; 8]); + // CHECK: (_0.0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (_0.1: [u8; 8]) = [const 2_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + Pair { a, b } +} + +// EMIT_MIR basic.enum_aggregate.MoveElimination.diff +pub fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + // This checks aggregate field remapping for enums: the payload fields can + // be written directly and then the discriminant is set for the variant. + // CHECK-LABEL: fn enum_aggregate( + // CHECK: debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + // CHECK: debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; + // CHECK: discriminant(_0) = 0; + let a = [1; 8]; + let b = [2; 8]; + Result::Ok((a, b)) +} + +// EMIT_MIR basic.array_aggregate.MoveElimination.diff +pub fn array_aggregate() -> [[u8; 8]; 3] { + // This checks aggregate remapping for arrays, which uses ConstantIndex + // projections rather than field projections. + // CHECK-LABEL: fn array_aggregate( + // CHECK: debug a => _0[0 of 1]; + // CHECK: debug b => _0[1 of 2]; + // CHECK: debug c => _0[2 of 3]; + // CHECK: _0[0 of 1] = [const 1_u8; 8]; + // CHECK: _0[1 of 2] = [const 2_u8; 8]; + // CHECK: _0[2 of 3] = [const 3_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + let c = [3; 8]; + [a, b, c] +} diff --git a/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..b03c4d2f3cff9 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff @@ -0,0 +1,49 @@ +- // MIR for `struct_aggregate` before MoveElimination ++ // MIR for `struct_aggregate` after MoveElimination + + fn struct_aggregate() -> Pair { + let mut _0: Pair; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (_0.0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (_0.1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = move _1; +- StorageLive(_4); +- _4 = move _2; +- _0 = Pair { a: move _3, b: move _4 }; +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (_0.0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (_0.1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff b/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff new file mode 100644 index 0000000000000..2f3019364f343 --- /dev/null +++ b/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff @@ -0,0 +1,92 @@ +- // MIR for `dse_guard` before MoveElimination ++ // MIR for `dse_guard` after MoveElimination + + fn dse_guard() -> () { + let mut _0: (); + let mut _1: Fields; + let mut _3: Fields; + let mut _4: Fields; + let _5: (); + let mut _6: *const Fields; + let mut _7: Fields; + let _8: (); + let mut _9: *const Fields; + scope 1 { +- debug a => _1; ++ debug a => _7; + let mut _2: Fields; + scope 2 { + debug b => _2; + } + } + + bb0: { +- StorageLive(_1); +- StorageLive(_2); +- StorageLive(_3); +- _3 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- _2 = move _3; +- StorageDead(_3); +- StorageLive(_4); +- _4 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ _7 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- _1 = move _4; +- StorageDead(_4); +- StorageLive(_5); +- StorageLive(_6); +- _6 = &raw const _1; ++ nop; ++ nop; ++ nop; ++ nop; ++ _6 = &raw const _7; + _5 = observe(move _6) -> [return: bb3, unwind unreachable]; + } + + bb3: { +- StorageDead(_6); +- StorageDead(_5); +- StorageLive(_7); +- _7 = move _1; ++ nop; ++ nop; ++ nop; ++ nop; + _2 = move _7; +- StorageDead(_7); +- StorageLive(_8); +- StorageLive(_9); ++ nop; ++ nop; ++ nop; + _9 = &raw const _2; + _8 = observe(move _9) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- StorageDead(_9); +- StorageDead(_8); ++ nop; ++ nop; + _0 = const (); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/dse.rs b/tests/mir-opt/move-elimination/dse.rs new file mode 100644 index 0000000000000..77685fb533fdd --- /dev/null +++ b/tests/mir-opt/move-elimination/dse.rs @@ -0,0 +1,36 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+DeadStoreElimination-initial + +pub struct Fields { + data: [u8; 8], + tag: u8, +} + +unsafe extern "C" { + safe fn observe(_: *const Fields); + safe fn make_fields(_: u8) -> Fields; +} + +// EMIT_MIR dse.dse_guard.MoveElimination.diff +pub fn dse_guard() { + // This guards the RFC soundness hazard: DSE must not remove the first write + // to `b`, because that write keeps `b`'s address-observed lifetime + // overlapping with `a` and prevents the later move from being eliminated. + // CHECK-LABEL: fn dse_guard( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: [[b]] = make_fields(const 0_u8) + // CHECK: [[a]] = make_fields(const 1_u8) + // CHECK: observe(move + // CHECK: [[b]] = move [[a]] + // CHECK: observe(move + let mut a; + let mut b; + + b = make_fields(0); + + a = make_fields(1); + observe(&raw const a); + b = a; + observe(&raw const b); +} diff --git a/tests/mir-opt/move-elimination/exclusions.constant_index_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.constant_index_projected.MoveElimination.diff new file mode 100644 index 0000000000000..33b96a7981738 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.constant_index_projected.MoveElimination.diff @@ -0,0 +1,41 @@ +- // MIR for `constant_index_projected` before MoveElimination ++ // MIR for `constant_index_projected` after MoveElimination + + fn constant_index_projected(_1: [[u8; 4]; 2]) -> () { + debug array => _1; + let mut _0: (); + let _2: [u8; 4]; + let _3: [u8; 4]; + let mut _4: [u8; 4]; + scope 1 { +- debug first => _2; ++ debug first => _1[0 of 2]; + } + + bb0: { +- StorageLive(_2); +- _2 = copy _1[0 of 2]; +- StorageLive(_3); +- StorageLive(_4); +- _4 = copy _2; +- _3 = std::hint::black_box::<[u8; 4]>(move _4) -> [return: bb1, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ _3 = std::hint::black_box::<[u8; 4]>(move _1[0 of 2]) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_4); +- StorageDead(_3); ++ nop; ++ nop; + _0 = const (); +- StorageDead(_2); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..4cdc1d24ee427 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff @@ -0,0 +1,16 @@ +- // MIR for `index_local_not_projected` before MoveElimination ++ // MIR for `index_local_not_projected` after MoveElimination + + fn index_local_not_projected(_1: [usize; 4]) -> [usize; 1] { + let mut _0: [usize; 1]; + let mut _2: usize; + let mut _3: usize; + + bb0: { + _2 = const 2_usize; + _3 = copy _1[_2]; + _0 = [copy _2]; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff new file mode 100644 index 0000000000000..ddf8630091e4b --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff @@ -0,0 +1,126 @@ +- // MIR for `overlapping_lifetimes` before MoveElimination ++ // MIR for `overlapping_lifetimes` after MoveElimination + + fn overlapping_lifetimes(_1: bool) -> Fields { + debug flag => _1; + let mut _0: Fields; + let _2: Fields; + let _4: (); + let mut _5: *const Fields; + let _6: (); + let mut _7: bool; + let mut _8: Fields; + let mut _9: Fields; + let mut _10: Fields; + let _11: (); + let mut _12: *const Fields; + scope 1 { +- debug src => _2; ++ debug src => _10; + let mut _3: Fields; + scope 2 { +- debug dst => _3; ++ debug dst => _0; + } + } + + bb0: { +- StorageLive(_2); +- _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ nop; ++ _10 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_3); +- StorageLive(_4); +- StorageLive(_5); +- _5 = &raw const _2; ++ nop; ++ nop; ++ nop; ++ _5 = &raw const _10; + _4 = observe(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); +- StorageDead(_4); +- StorageLive(_6); +- StorageLive(_7); +- _7 = copy _1; +- switchInt(move _7) -> [0: bb5, otherwise: bb3]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb5, otherwise: bb3]; + } + + bb3: { +- StorageLive(_8); +- _8 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; ++ nop; ++ _0 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- _3 = move _8; +- StorageDead(_8); +- StorageLive(_9); +- _9 = move _2; +- _3 = move _9; +- StorageDead(_9); ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = move _10; ++ nop; + _6 = const (); + goto -> bb6; + } + + bb5: { +- StorageLive(_10); +- _10 = move _2; +- _3 = move _10; +- StorageDead(_10); ++ nop; ++ nop; ++ _0 = move _10; ++ nop; + _6 = const (); + goto -> bb6; + } + + bb6: { +- StorageDead(_7); +- StorageDead(_6); +- StorageLive(_11); +- StorageLive(_12); +- _12 = &raw const _3; ++ nop; ++ nop; ++ nop; ++ nop; ++ _12 = &raw const _0; + _11 = observe(move _12) -> [return: bb7, unwind unreachable]; + } + + bb7: { +- StorageDead(_12); +- StorageDead(_11); +- _0 = move _3; +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..6c0bb304d3171 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff @@ -0,0 +1,48 @@ +- // MIR for `packed_fields_not_projected` before MoveElimination ++ // MIR for `packed_fields_not_projected` after MoveElimination + + fn packed_fields_not_projected() -> Packed { + let mut _0: Packed; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _3; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _4; + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = copy _1; +- StorageLive(_4); +- _4 = copy _2; ++ nop; ++ _3 = [const 1_u8; 8]; ++ nop; ++ _4 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = Packed { a: move _3, b: move _4 }; +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.rs b/tests/mir-opt/move-elimination/exclusions.rs new file mode 100644 index 0000000000000..613ca8c140520 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rs @@ -0,0 +1,139 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![feature(core_intrinsics, custom_mir, repr_simd)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +pub struct Fields { + data: [u8; 8], + tag: u8, +} + +#[repr(packed)] +struct Packed { + a: [u8; 8], + b: [u8; 8], +} + +#[repr(simd)] +struct U32x4([u32; 4]); + +unsafe extern "C" { + safe fn observe(_: *const Fields); + safe fn make_fields(_: u8) -> Fields; +} + +// EMIT_MIR exclusions.index_local_not_projected.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn index_local_not_projected(a: [usize; 4]) -> [usize; 1] { + // This checks that a local used as an array index is kept as a bare local, + // because it cannot later be rewritten to a projection like `_0[0]`. + // CHECK-LABEL: fn index_local_not_projected( + // CHECK: [[idx:_.*]] = const 2_usize; + // CHECK: {{.*}} = copy _1[{{.*}}[[idx]]{{.*}}]; + // CHECK: _0 = [copy [[idx]]]; + mir! { + let idx: usize; + let b: usize; + + { + idx = 2usize; + b = a[idx]; + RET = [idx]; + Return() + } + } +} + +// EMIT_MIR exclusions.packed_fields_not_projected.MoveElimination.diff +pub fn packed_fields_not_projected() -> Packed { + // This checks that aggregate fields are not remapped into packed struct + // fields, which could create unaligned projected places. + // CHECK-LABEL: fn packed_fields_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: _0 = Packed { a: move [[a]], b: move [[b]] }; + let a = [1; 8]; + let b = [2; 8]; + Packed { a, b } +} + +// EMIT_MIR exclusions.simd_field_not_projected.MoveElimination.diff +pub fn simd_field_not_projected() -> U32x4 { + // This checks that aggregate fields are not remapped into repr(simd) ADTs, + // since optimized MIR must not project into SIMD vectors. + // CHECK-LABEL: fn simd_field_not_projected( + // CHECK: debug lanes => [[lanes:_.*]]; + // CHECK: _0 = U32x4(move [[lanes]]); + let lanes = [1, 2, 3, 4]; + U32x4(lanes) +} + +// EMIT_MIR exclusions.overlapping_lifetimes.MoveElimination.diff +pub fn overlapping_lifetimes(flag: bool) -> Fields { + // This checks the liveness-matrix overlap test for an address-observed + // move-only local: `src` and `dst` only overlap on one branch, but that is + // enough to reject merging them for the whole function. + // CHECK-LABEL: fn overlapping_lifetimes( + // CHECK: debug flag => _1; + // CHECK: debug src => [[src:_[1-9][0-9]*]]; + // CHECK: debug dst => _0; + // CHECK: &raw const [[src]]; + // CHECK: observe + // CHECK: switchInt(move _1) + // CHECK: _0 = make_fields(const 1_u8) + // CHECK: _0 = move [[src]]; + // CHECK: &raw const _0; + // CHECK: observe + let src = make_fields(0); + let mut dst; + observe(&raw const src); + if flag { + dst = make_fields(1); + dst = src; + } else { + dst = src; + } + observe(&raw const dst); + dst +} + +// EMIT_MIR exclusions.rust_call_tuple_not_projected.MoveElimination.diff +pub fn rust_call_tuple_not_projected(f: F) { + // This checks that locals are not remapped into the tuple argument passed + // to a rust-call ABI function. If the tuple itself is never borrowed, alias + // analysis can trivially see that accesses to one argument don't affect the + // others. Merging the arguments into tuple fields from the start can hide + // that independence. + // CHECK-LABEL: fn rust_call_tuple_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: [[tuple:_.*]] = (move [[a]], move [[b]]); + // CHECK: >::call_once(move _1, move [[tuple]]) + let a = [1; 8]; + let b = [2; 8]; + f(a, b); +} + +// Subslice projections cannot appear in debuginfo, so refuse to merge `array` +// and `local`. +// EMIT_MIR exclusions.subslice_not_projected.MoveElimination.diff +pub fn subslice_not_projected(array: [u8; 4]) { + // CHECK-LABEL: fn subslice_not_projected( + // CHECK: debug middle => [[MIDDLE:_[0-9]+]]; + // CHECK: [[MIDDLE]] = copy _1[1..3]; + let [_, middle @ .., _] = array; + std::hint::black_box(middle); +} + +// Forward constant indices are supported in debuginfo and allow remapping. +// EMIT_MIR exclusions.constant_index_projected.MoveElimination.diff +pub fn constant_index_projected(array: [[u8; 4]; 2]) { + // CHECK-LABEL: fn constant_index_projected( + // CHECK: debug first => _1[0 of 2]; + // CHECK: std::hint::black_box::<[u8; 4]>(move _1[0 of 2]) + let [first, ..] = array; + std::hint::black_box(first); +} diff --git a/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..bbde5a905a9e7 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff @@ -0,0 +1,72 @@ +- // MIR for `rust_call_tuple_not_projected` before MoveElimination ++ // MIR for `rust_call_tuple_not_projected` after MoveElimination + + fn rust_call_tuple_not_projected(_1: F) -> () { + debug f => _1; + let mut _0: (); + let _2: [u8; 8]; + let _4: (); + let mut _5: F; + let mut _6: ([u8; 8], [u8; 8]); + let mut _7: [u8; 8]; + let mut _8: [u8; 8]; + scope 1 { +- debug a => _2; ++ debug a => _7; + let _3: [u8; 8]; + scope 2 { +- debug b => _3; ++ debug b => _8; + } + } + + bb0: { +- StorageLive(_2); +- _2 = [const 1_u8; 8]; +- StorageLive(_3); +- _3 = [const 2_u8; 8]; +- StorageLive(_4); +- StorageLive(_5); +- _5 = move _1; +- StorageLive(_6); +- StorageLive(_7); +- _7 = copy _2; +- StorageLive(_8); +- _8 = copy _3; ++ nop; ++ _7 = [const 1_u8; 8]; ++ nop; ++ _8 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + _6 = (move _7, move _8); +- _4 = >::call_once(move _5, move _6) -> [return: bb1, unwind unreachable]; ++ _4 = >::call_once(move _1, move _6) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_8); +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_5); +- StorageDead(_4); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = const (); +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..8d63e07039695 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff @@ -0,0 +1,30 @@ +- // MIR for `simd_field_not_projected` before MoveElimination ++ // MIR for `simd_field_not_projected` after MoveElimination + + fn simd_field_not_projected() -> U32x4 { + let mut _0: U32x4; + let _1: [u32; 4]; + let mut _2: [u32; 4]; + scope 1 { +- debug lanes => _1; ++ debug lanes => _2; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; +- StorageLive(_2); +- _2 = copy _1; ++ nop; ++ _2 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; ++ nop; ++ nop; + _0 = U32x4(move _2); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.subslice_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.subslice_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..85b38460d277a --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.subslice_not_projected.MoveElimination.diff @@ -0,0 +1,40 @@ +- // MIR for `subslice_not_projected` before MoveElimination ++ // MIR for `subslice_not_projected` after MoveElimination + + fn subslice_not_projected(_1: [u8; 4]) -> () { + debug array => _1; + let mut _0: (); + let _2: [u8; 2]; + let _3: [u8; 2]; + let mut _4: [u8; 2]; + scope 1 { +- debug middle => _2; ++ debug middle => _4; + } + + bb0: { +- StorageLive(_2); +- _2 = copy _1[1..3]; +- StorageLive(_3); +- StorageLive(_4); +- _4 = copy _2; ++ nop; ++ _4 = copy _1[1..3]; ++ nop; ++ nop; ++ nop; + _3 = std::hint::black_box::<[u8; 2]>(move _4) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_4); +- StorageDead(_3); ++ nop; ++ nop; + _0 = const (); +- StorageDead(_2); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff b/tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff new file mode 100644 index 0000000000000..7507f7bc81e7e --- /dev/null +++ b/tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff @@ -0,0 +1,27 @@ +- // MIR for `call_destination` before MoveElimination ++ // MIR for `call_destination` after MoveElimination + + fn call_destination() -> () { + let mut _0: (); + let mut _1: u32; + let mut _2: u32; + let mut _3: *const u32; + + bb0: { + _1 = const 0_u32; + _3 = &raw const _1; + _1 = opaque() -> [return: bb1, unwind: bb2]; + } + + bb1: { + _0 = (); + return; + } + + bb2 (cleanup): { + _2 = const 2_u32; + _1 = move _2; + resume; + } + } + diff --git a/tests/mir-opt/move-elimination/unwind.rs b/tests/mir-opt/move-elimination/unwind.rs new file mode 100644 index 0000000000000..0015997f7b543 --- /dev/null +++ b/tests/mir-opt/move-elimination/unwind.rs @@ -0,0 +1,41 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=unwind + +#![crate_type = "lib"] +#![feature(custom_mir, core_intrinsics)] +use std::intrinsics::mir::*; + +#[inline(never)] +fn opaque() -> u32 { + 1 +} + +// EMIT_MIR unwind.call_destination.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn call_destination() { + // CHECK-LABEL: fn call_destination( + // CHECK: [[DEST:_[0-9]+]] = opaque() + // CHECK: [[SRC:_[0-9]+]] = const 2_u32; + // CHECK-NEXT: [[DEST]] = move [[SRC]]; + // Unwinding doesn't deallocate the call destination, it only leaves it + // uninitialized. This checks that dest is not merged with src. + mir! { + let dest: u32; + let src: u32; + let p: *const u32; + { + dest = 0; + p = &raw const dest; + Call(dest = opaque(), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + RET = (); + Return() + } + cleanup (cleanup) = { + src = 2; + dest = Move(src); + UnwindResume() + } + } +} From fd851d560adb2ce7437fb664847a99c88edae50f Mon Sep 17 00:00:00 2001 From: Jules Bertholet Date: Tue, 20 Oct 2026 10:20:16 -0400 Subject: [PATCH 03/10] Make mir-opt blessing work again --- ...ic.runtime-optimized.after.panic-abort.mir | 78 +++++++++++++++++++ ....runtime-optimized.after.panic-unwind.mir} | 0 tests/mir-opt/issues/issue_163598.rs | 1 + 3 files changed, 79 insertions(+) create mode 100644 tests/mir-opt/issues/issue_163598.chain_generic.runtime-optimized.after.panic-abort.mir rename tests/mir-opt/issues/{issue_163598.chain_generic.runtime-optimized.after.mir => issue_163598.chain_generic.runtime-optimized.after.panic-unwind.mir} (100%) diff --git a/tests/mir-opt/issues/issue_163598.chain_generic.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/issues/issue_163598.chain_generic.runtime-optimized.after.panic-abort.mir new file mode 100644 index 0000000000000..28259cd0bc312 --- /dev/null +++ b/tests/mir-opt/issues/issue_163598.chain_generic.runtime-optimized.after.panic-abort.mir @@ -0,0 +1,78 @@ +// MIR for `chain_generic` after runtime-optimized + +fn chain_generic(_1: &(T, U), _2: &(T, U)) -> ControlFlow { + debug l => _1; + debug r => _2; + let mut _0: std::ops::ControlFlow; + scope 1 (inlined core::tuple::::__chaining_le) { + let _3: &T; + let _4: &T; + let mut _5: isize; + let _6: &U; + let _7: &U; + let mut _8: isize; + scope 2 { + } + scope 3 { + } + } + + bb0: { + StorageLive(_5); + StorageLive(_8); + StorageLive(_3); + _3 = &((*_1).0: T); + StorageLive(_4); + _4 = &((*_2).0: T); + _0 = ::__chaining_le(move _3, move _4) -> [return: bb1, unwind unreachable]; + } + + bb1: { + _5 = discriminant(_0); + switchInt(move _5) -> [1: bb2, 0: bb3, otherwise: bb8]; + } + + bb2: { + StorageDead(_4); + StorageDead(_3); + goto -> bb7; + } + + bb3: { + StorageDead(_4); + StorageDead(_3); + StorageLive(_6); + _6 = &((*_1).1: U); + StorageLive(_7); + _7 = &((*_2).1: U); + _0 = ::__chaining_le(move _6, move _7) -> [return: bb4, unwind unreachable]; + } + + bb4: { + _8 = discriminant(_0); + switchInt(move _8) -> [1: bb5, 0: bb6, otherwise: bb8]; + } + + bb5: { + StorageDead(_7); + StorageDead(_6); + goto -> bb7; + } + + bb6: { + StorageDead(_7); + StorageDead(_6); + _0 = const ControlFlow::::Continue(()); + goto -> bb7; + } + + bb7: { + StorageDead(_8); + StorageDead(_5); + return; + } + + bb8: { + unreachable; + } +} diff --git a/tests/mir-opt/issues/issue_163598.chain_generic.runtime-optimized.after.mir b/tests/mir-opt/issues/issue_163598.chain_generic.runtime-optimized.after.panic-unwind.mir similarity index 100% rename from tests/mir-opt/issues/issue_163598.chain_generic.runtime-optimized.after.mir rename to tests/mir-opt/issues/issue_163598.chain_generic.runtime-optimized.after.panic-unwind.mir diff --git a/tests/mir-opt/issues/issue_163598.rs b/tests/mir-opt/issues/issue_163598.rs index 393bce63f4bf6..0da8cb05fba48 100644 --- a/tests/mir-opt/issues/issue_163598.rs +++ b/tests/mir-opt/issues/issue_163598.rs @@ -1,3 +1,4 @@ +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY //@ skip-filecheck //@ compile-flags: -Z mir-opt-level=3 From a1af1ad76ef16906777ab1261f6582bedc8a426f Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 20 Oct 2026 21:04:59 +0200 Subject: [PATCH 04/10] Also reject scalable vectors and DynMetadata projections --- compiler/rustc_middle/src/ty/adt.rs | 13 +++++++++ .../src/move_elimination.rs | 9 +++--- compiler/rustc_mir_transform/src/sroa.rs | 11 +------ compiler/rustc_mir_transform/src/validate.rs | 29 ++++++++----------- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs index 5b57f802c0c4f..d0c4f08ae57fb 100644 --- a/compiler/rustc_middle/src/ty/adt.rs +++ b/compiler/rustc_middle/src/ty/adt.rs @@ -212,6 +212,19 @@ impl<'tcx> AdtDef<'tcx> { self.0.0.repr } + /// Whether MIR may project into the fields of this ADT. + /// + /// MCP#838 bans field projections into SIMD types, including scalable + /// vectors. + /// + /// Codegen requires `DynMetadata` to be manipulated as a whole, with its + /// insides being treated as opaque.. + pub fn can_project_fields(self, tcx: TyCtxt<'tcx>) -> bool { + !self.repr().simd() + && !self.repr().scalable() + && !tcx.is_lang_item(self.did(), LangItem::DynMetadata) + } + pub fn field_representing_type_info( self, tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_mir_transform/src/move_elimination.rs b/compiler/rustc_mir_transform/src/move_elimination.rs index ac6ddc33578e4..e02f1103fbf77 100644 --- a/compiler/rustc_mir_transform/src/move_elimination.rs +++ b/compiler/rustc_mir_transform/src/move_elimination.rs @@ -504,8 +504,7 @@ impl<'tcx> Visitor<'tcx> for PlaceUnification<'_, 'tcx> { } AggregateKind::Adt(adt_did, var_idx, _, _, None) => { let def = self.tcx.adt_def(adt_did); - if def.repr().simd() { - // MCP#838 banned projections into SIMD types. + if !def.can_project_fields(self.tcx) { return; } self.visit_aggregate_assign( @@ -842,12 +841,12 @@ impl<'tcx> MutVisitor<'tcx> for AliasFixup<'_, 'tcx> { location, ) } - // MCP#838 banned projections into SIMD types, so use the - // general aliasing fallback below for those aggregates. + // Types that forbid field projections must use the general + // aliasing fallback below. Rvalue::Aggregate( AggregateKind::Adt(adt_did, var_idx, _, _, None), ref mut operands, - ) if !self.tcx.adt_def(adt_did).repr().simd() => { + ) if self.tcx.adt_def(adt_did).can_project_fields(self.tcx) => { let def = self.tcx.adt_def(adt_did); self.visit_aggregate_assign( *dest, diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index 591c84bb17689..8ee452066a695 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -1,5 +1,4 @@ use rustc_abi::FieldIdx; -use rustc_attr_ir::lang_items::LangItem; use rustc_data_structures::flat_map_in_place::FlatMapInPlace; use rustc_index::IndexVec; use rustc_index::bit_set::{DenseBitSet, GrowableBitSet}; @@ -64,16 +63,8 @@ fn escaping_locals<'tcx>( return true; } if let ty::Adt(def, _args) = ty.kind() - && (def.repr().simd() - || def.repr().scalable() - || tcx.is_lang_item(def.did(), LangItem::DynMetadata)) + && !def.can_project_fields(tcx) { - // Exclude #[repr(simd)] types so that they are not de-optimized into an array - // (MCP#838 banned projections into SIMD types, but if the value is unused - // this pass sees "all the uses are of the fields" and expands it.) - - // codegen wants to see the `DynMetadata`, - // not the inner reference-to-opaque-type. return true; } // Default for non-ADTs diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index c8564b8288cd2..7056b1b1619fb 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -716,24 +716,19 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { // Debug info is allowed to project into pattern types ty::Pat(base, _) => check_equal(self, location, *base), ty::Adt(adt_def, args) => { - // see - if self.tcx.is_lang_item(adt_def.did(), LangItem::DynMetadata) { - self.fail( - location, - format!( - "You can't project to field {f:?} of `DynMetadata` because \ + if !adt_def.can_project_fields(self.tcx) { + let message = + if self.tcx.is_lang_item(adt_def.did(), LangItem::DynMetadata) { + format!( + "You can't project to field {f:?} of `DynMetadata` because \ layout is weird and thinks it doesn't have fields." - ), - ); - } - - if adt_def.repr().simd() || adt_def.repr().scalable() { - self.fail( - location, - format!( - "Projecting into SIMD type {adt_def:?} is banned by MCP#838" - ), - ); + ) + } else { + format!( + "Projecting into SIMD type {adt_def:?} is banned by MCP#838" + ) + }; + self.fail(location, message); } let var = parent_ty.variant_index.unwrap_or(FIRST_VARIANT); From 6d6870614cb11c938e5c5c25d4297637f46edf81 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Wed, 30 Sep 2026 18:41:27 -0500 Subject: [PATCH 05/10] test: Rename `breakpoint.rs` to `intrinsics.rs` We can reuse the test setup for other intrinsics, saving a bit of build time and repeated setup. --- library/core/src/arch.rs | 2 +- tests/assembly-llvm/{breakpoint.rs => intrinsics.rs} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/assembly-llvm/{breakpoint.rs => intrinsics.rs} (100%) diff --git a/library/core/src/arch.rs b/library/core/src/arch.rs index 570427f508554..31df982824ea1 100644 --- a/library/core/src/arch.rs +++ b/library/core/src/arch.rs @@ -78,7 +78,7 @@ pub macro global_asm("assembly template", $(operands,)* $(options($(option),*))? /// The precise instruction is guaranteed only on the following targets: /// - On x86 targets, this produces an `int3` instruction. /// - On aarch64 targets, this produces a `brk #0xf000` instruction. -// When adding more items above, also add cases to the test in `tests/assembly-llvm/breakpoint.rs`. +// When adding more items above, also add cases to the test in `tests/assembly-llvm/intrinsics.rs`. // When stabilizing this, update the comment on `core::intrinsics::breakpoint`. // NOTE: there is an alternative in `process::resumable_breakpoint`. We may or may not // actually want to stabilize both of them. diff --git a/tests/assembly-llvm/breakpoint.rs b/tests/assembly-llvm/intrinsics.rs similarity index 100% rename from tests/assembly-llvm/breakpoint.rs rename to tests/assembly-llvm/intrinsics.rs From f22c20e3d80778746c10b8d54fef03c3dcc78fc6 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Wed, 30 Sep 2026 20:12:02 -0500 Subject: [PATCH 06/10] core: Add examples and assembly tests for `abort_immediate` This isn't the most complicated function, but it currently isn't used anywhere in-tree (typically `intrinsics::abort` is used instead) and there are no examples or tests. Copy the examples from `abort` since this should behave similarly, and also cover it in an assembly test. We don't really need the assembly check for any guarantees but it is handy to reference what the intrinsic actually does, and will let us know if anything changes. I also added tests for `breakpoint` on more platforms since these were easy to do together. --- library/core/src/process.rs | 42 +++++++- tests/assembly-llvm/intrinsics.rs | 163 +++++++++++++++++++++++++++--- 2 files changed, 191 insertions(+), 14 deletions(-) diff --git a/library/core/src/process.rs b/library/core/src/process.rs index e92339f34d68e..3a9ecd77869dc 100644 --- a/library/core/src/process.rs +++ b/library/core/src/process.rs @@ -22,7 +22,7 @@ /// /// `abort_immediate` lowers to a trap instruction on *most* architectures; on /// some architectures it simply lowers to call the unmangled `abort` function. -/// The exact behavior is architecture and system dependent. +/// The exact behavior is architecture- and system-dependent. /// /// On bare-metal (no OS) systems the trap instruction usually causes a /// *hardware* exception to be raised in a *synchronous* fashion; hardware @@ -34,6 +34,46 @@ /// corresponds to `SIGILL` or equivalent, *unless* this signal is handled. /// Other signals such as `SIGABRT`, `SIGTRAP`, `SIGSEGV`, and `SIGBUS` may be /// produced instead, depending on specifics. This is not an exhaustive list. +/// +/// # Examples +/// +/// ```no_run +/// #![feature(abort_immediate)] +/// +/// use core::process; +/// +/// fn main() { +/// println!("aborting"); +/// +/// process::abort_immediate(); +/// +/// // execution never gets here +/// } +/// ``` +/// +/// The `abort_immediate` function terminates the process, so the destructor +/// will not get run on the example below: +/// +/// ```no_run +/// #![feature(abort_immediate)] +/// +/// use core::process; +/// +/// struct HasDrop; +/// +/// impl Drop for HasDrop { +/// fn drop(&mut self) { +/// println!("This will never be printed!"); +/// } +/// } +/// +/// fn main() { +/// let _x = HasDrop; +/// process::abort_immediate(); +/// // the destructor implemented for HasDrop will never get run +/// } +/// ``` +/// #[unstable(feature = "abort_immediate", issue = "154601")] #[rustc_const_unstable(feature = "abort_immediate", issue = "154601")] #[cold] diff --git a/tests/assembly-llvm/intrinsics.rs b/tests/assembly-llvm/intrinsics.rs index 39ee26a4cf5d5..9d89c04f51fb5 100644 --- a/tests/assembly-llvm/intrinsics.rs +++ b/tests/assembly-llvm/intrinsics.rs @@ -1,14 +1,87 @@ -//! Verify that the breakpoint operation emits the instructions we guarantee. +//! Verify that simple intrinsics emit the instructions we expect. //@ add-minicore //@ assembly-output: emit-asm -//@ revisions: AARCH64 I686 X86_64 -//@ [AARCH64] compile-flags: --target aarch64-unknown-linux-gnu +//@ revisions: AARCH64 ARM64EC ARM32 AVR BPF LOONGARCH32 LOONGARCH64 I686 M68K MIPS32 MIPS64 MSP430 PPC32 PPC64 RISCV32 RISCV64 S390X SPARC32 SPARC64 WASM32 WASI32 X86_64 XTENSA +// +//@ [AARCH64] compile-flags: --target=aarch64-unknown-linux-gnu //@ [AARCH64] needs-llvm-components: aarch64 -//@ [I686] compile-flags: --target i686-unknown-linux-gnu +//@ [ARM64EC] compile-flags: --target=arm64ec-pc-windows-msvc +//@ [ARM64EC] filecheck-flags: --check-prefixes AARCH64 +//@ [ARM64EC] needs-llvm-components: aarch64 +// +//@ [BPF] compile-flags: --target=bpfel-unknown-none +//@ [BPF] needs-llvm-components: bpf +// +//@ [AVR] compile-flags: --target=avr-none -Ctarget-cpu=atmega328p +//@ [AVR] needs-llvm-components: avr +// +//@ [ARM32] compile-flags: --target=armv7-unknown-linux-gnueabi +//@ [ARM32] needs-llvm-components: arm +// +//@ [LOONGARCH32] compile-flags: --target=loongarch32-unknown-none +//@ [LOONGARCH32] filecheck-flags: --check-prefixes LOONGARCH +//@ [LOONGARCH32] needs-llvm-components: loongarch +//@ [LOONGARCH64] compile-flags: --target=loongarch64-unknown-linux-gnu +//@ [LOONGARCH64] filecheck-flags: --check-prefixes LOONGARCH +//@ [LOONGARCH64] needs-llvm-components: loongarch +//@ [LOONGARCH32] min-llvm-version: 22 +//@ [LOONGARCH64] min-llvm-version: 22 +// +//@ [M68K] compile-flags: --target=m68k-unknown-linux-gnu +//@ [M68K] needs-llvm-components: m68k +// +//@ [MIPS32] compile-flags: --target=mips-unknown-linux-gnu +//@ [MIPS32] filecheck-flags: --check-prefixes MIPS +//@ [MIPS32] needs-llvm-components: mips +//@ [MIPS64] compile-flags: --target=mips64-unknown-linux-gnuabi64 +//@ [MIPS64] filecheck-flags: --check-prefixes MIPS +//@ [MIPS64] needs-llvm-components: mips +// +//@ [MSP430] compile-flags: --target=msp430-none-elf +//@ [MSP430] needs-llvm-components: msp430 +// +//@ [PPC32] compile-flags: --target=powerpc-unknown-linux-gnu +//@ [PPC32] filecheck-flags: --check-prefixes PPC +//@ [PPC32] needs-llvm-components: powerpc +//@ [PPC64] compile-flags: --target=powerpc64-unknown-linux-gnu +//@ [PPC64] filecheck-flags: --check-prefixes PPC +//@ [PPC64] needs-llvm-components: powerpc +// +//@ [RISCV32] compile-flags: --target=riscv32im-unknown-none-elf +//@ [RISCV32] filecheck-flags: --check-prefixes RISCV +//@ [RISCV32] needs-llvm-components: riscv +//@ [RISCV64] compile-flags: --target=riscv64gc-unknown-linux-gnu +//@ [RISCV64] filecheck-flags: --check-prefixes RISCV +//@ [RISCV64] needs-llvm-components: riscv +// +//@ [S390X] compile-flags: --target=s390x-unknown-linux-gnu +//@ [S390X] needs-llvm-components: systemz +// +//@ [SPARC32] compile-flags: --target=sparc-unknown-linux-gnu +//@ [SPARC32] filecheck-flags: --check-prefixes SPARC +//@ [SPARC32] needs-llvm-components: sparc +//@ [SPARC64] compile-flags: --target=sparc64-unknown-linux-gnu +//@ [SPARC64] filecheck-flags: --check-prefixes SPARC +//@ [SPARC64] needs-llvm-components: sparc +// +//@ [WASM32] compile-flags: --target=wasm32-unknown-unknown +//@ [WASM32] filecheck-flags: --check-prefixes WASM +//@ [WASM32] needs-llvm-components: webassembly +//@ [WASI32] compile-flags: --target=wasm32-wasip3 +//@ [WASI32] filecheck-flags: --check-prefixes WASM +//@ [WASI32] needs-llvm-components: webassembly +// +//@ [I686] compile-flags: --target=i686-unknown-linux-gnu +//@ [I686] filecheck-flags: --check-prefixes X86 //@ [I686] needs-llvm-components: x86 -//@ [X86_64] compile-flags: --target x86_64-unknown-linux-gnu +//@ [X86_64] compile-flags: --target=x86_64-unknown-linux-gnu +//@ [X86_64] filecheck-flags: --check-prefixes X86 //@ [X86_64] needs-llvm-components: x86 +// +//@ [XTENSA] compile-flags: --target=xtensa-esp32-espidf +//@ [XTENSA] needs-llvm-components: xtensa +//@ [XTENSA] min-llvm-version: 23 #![crate_type = "lib"] #![feature(no_core, lang_items, intrinsics, rustc_attrs)] @@ -16,15 +89,79 @@ extern crate minicore; -#[rustc_intrinsic] -#[rustc_nounwind] -fn breakpoint(); +mod intrinsics { + #[rustc_intrinsic] + #[rustc_nounwind] + pub fn abort_immediate() -> !; -// CHECK-LABEL: call_breakpoint + #[rustc_intrinsic] + #[rustc_nounwind] + pub fn breakpoint(); +} + +// Optional `"` since labels are quoted on arm64ec. Forbid calls by default since most platforms +// shouldn't be calling out to libc for this intrinsic. +// +// Note that some platforms (MSP430, AVR) do require an abort builtin, but this is typically a +// loop or hardware reset. +// +// We don't make a user-facing guarantee about what exactly `abort` will do but it also shouldn't +// change. +// +// CHECK-LABEL: do_abort{{"?}}: +// CHECK-NOT: {{ (call|b|bl) }} +// +// AARCH64: brk #0x1 +// AVR: call abort +// BPF: call __bpf_trap +// ARM32: .inst 0xe7ffdefe +// LOONGARCH: ud 0 +// M68K: jsr (abort@PLT,%pc) +// MIPS: break +// MSP430: call #abort +// PPC: trap +// RISCV: unimp +// // S390x lowers to a loop +// S390X: .Ltmp{{.*}}: +// S390X: j .Ltmp{{.*}} +// SPARC: ta 5 +// WASM: unreachable +// X86: ud2 +// XTENSA: ill.n +#[unsafe(no_mangle)] +pub fn do_abort() { + intrinsics::abort_immediate(); +} + +// Optional `"` since labels are quoted on arm64ec. Similar to `abort`, there typically shouldn't +// be any libcalls here. +// +// CHECK-LABEL: do_breakpoint{{"?}}: +// CHECK-NOT: {{ (call|b|bl|j) }} +// +// Documentation guarantees these instructions: +// // AARCH64: brk #0xf000 -// I686: int3 -// X86_64: int3 +// X86: int3 +// +// We do not guarantee exact output on the following platforms but still check them: +// +// ARM32: bkpt #0 +// AVR: call abort +// BPF: call __bpf_trap +// LOONGARCH: break 0 +// M68K: jsr (abort@PLT,%pc) +// MIPS: break +// MSP430: call #abort +// PPC: trap +// RISCV: ebreak +// // S390x lowers to a loop +// S390X: .Ltmp{{.*}}: +// S390X: j .Ltmp{{.*}} +// SPARC: ta 1 +// WASM: unreachable +// XTENSA: break.n #[unsafe(no_mangle)] -pub fn call_breakpoint() { - breakpoint(); +pub fn do_breakpoint() { + intrinsics::breakpoint(); } From c40b599f72b9a79f564587595a6650df16d52acf Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Thu, 1 Oct 2026 16:20:47 -0500 Subject: [PATCH 07/10] core: Add `trap` as a doc alias for `abort_immediate` --- library/core/src/process.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/library/core/src/process.rs b/library/core/src/process.rs index 3a9ecd77869dc..d03e7cba04bef 100644 --- a/library/core/src/process.rs +++ b/library/core/src/process.rs @@ -79,6 +79,7 @@ #[cold] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[doc(alias = "halt")] +#[doc(alias = "trap")] pub const fn abort_immediate() -> ! { crate::intrinsics::abort_immediate() } From 0c95ca9df87759b10be4fb06d644975970e92cf2 Mon Sep 17 00:00:00 2001 From: Josh Triplett Date: Tue, 22 Sep 2026 14:57:48 -0700 Subject: [PATCH 08/10] Add more documentation for `chroot`, including an example and a manpage link --- library/std/src/os/unix/process.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/library/std/src/os/unix/process.rs b/library/std/src/os/unix/process.rs index 9fa731de82691..a1b574f9b6b12 100644 --- a/library/std/src/os/unix/process.rs +++ b/library/std/src/os/unix/process.rs @@ -213,6 +213,33 @@ pub impl(self) trait CommandExt { /// If no directory has been specified with [`process::Command::current_dir`], this will set the /// directory to `/`, to avoid leaving the current directory outside the chroot. (This is an /// intentional difference from the underlying `chroot` system call.) + /// + /// This typically requires privileges, such as root or a specific capability. + /// + /// Note that `chroot` is not guaranteed to be async-signal-safe on all OSes, and functions + /// called between `fork` and `exec` must be async-signal-safe. However, in practice, the + /// ability to `fork` then `chroot` then `exec` is a capability widely provided by Rust targets + /// and relied upon by numerous programs. + /// + /// See the [`chroot`] manual page for more details. + /// + /// [`chroot`]: https://man7.org/linux/man-pages/man2/chroot.2.html + /// + /// # Examples + /// + #[cfg_attr(target_family = "unix", doc = "```no_run")] + #[cfg_attr(not(target_family = "unix"), doc = "```ignore (needs unix)")] + /// #![feature(process_chroot)] + /// use std::process::Command; + /// use std::os::unix::process::CommandExt; + /// + /// Command::new("contained") + /// .chroot("/sandbox") + /// .spawn()? + /// .wait()?; + /// # + /// # Ok::<_, Box>(()) + #[doc = "```"] #[unstable(feature = "process_chroot", issue = "141298")] fn chroot>(&mut self, dir: P) -> &mut process::Command; From 1e12f732d1f30051edac2cddc268667f14200834 Mon Sep 17 00:00:00 2001 From: Josh Triplett Date: Tue, 20 Oct 2026 07:58:54 -0700 Subject: [PATCH 09/10] chroot: Elaborate on how chroot is not sufficient for sandboxing --- library/std/src/os/unix/process.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/library/std/src/os/unix/process.rs b/library/std/src/os/unix/process.rs index a1b574f9b6b12..22ad8974bd425 100644 --- a/library/std/src/os/unix/process.rs +++ b/library/std/src/os/unix/process.rs @@ -221,6 +221,9 @@ pub impl(self) trait CommandExt { /// ability to `fork` then `chroot` then `exec` is a capability widely provided by Rust targets /// and relied upon by numerous programs. /// + /// `chroot` is not, by itself, a complete security or sandboxing measure. It may form some part + /// of a containment mechanism, but fully containing a process requires additional steps. + /// /// See the [`chroot`] manual page for more details. /// /// [`chroot`]: https://man7.org/linux/man-pages/man2/chroot.2.html @@ -233,8 +236,8 @@ pub impl(self) trait CommandExt { /// use std::process::Command; /// use std::os::unix::process::CommandExt; /// - /// Command::new("contained") - /// .chroot("/sandbox") + /// Command::new("binary-within-new-root") + /// .chroot("/new-root") /// .spawn()? /// .wait()?; /// # From 16475dedfa68990e3d16c8ab573865f40f9f2dca Mon Sep 17 00:00:00 2001 From: Jieyou Xu Date: Wed, 21 Oct 2026 15:20:08 +0800 Subject: [PATCH 10/10] Bump `rustls-webpki` to 0.103.16 and `webpki-roots` to 1.0.10 To mitigate the eMudhra Intermediate CA Compromise. --- src/ci/citool/Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ci/citool/Cargo.lock b/src/ci/citool/Cargo.lock index d6ffffb7b9f00..9d891739c1556 100644 --- a/src/ci/citool/Cargo.lock +++ b/src/ci/citool/Cargo.lock @@ -863,9 +863,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.14" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -1176,9 +1176,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ]