Rollup of 4 pull requests

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -109,6 +109,19 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
         cid.promoted.map_or_else(String::new, |p| format!("::{p:?}"))
     );
 
+    // We are about to initialize a frame without any arguments,
+    // and without `#[track_caller]` support.
+    // Make sure that the body expects this.
+    // Note: Promoteds don't require a caller location,
+    // even when inside an `Instance` with `#[track_caller]`
+    assert_eq!(body.arg_count, 0);
+    if cid.promoted.is_none() && cid.instance.def.requires_caller_location(tcx) {
+        // FIXME: Once #131787 is fixed, turn this delayed bug into an assertion.
+        throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
+            tcx.dcx().span_delayed_bug(body.span, "can't evaluate body with `#[track_caller]`")
+        )));
+    }
+
     // This can't use `init_stack_frame` since `body` is not a function,
     // so computing its ABI would fail. It's also not worth it since there are no arguments to pass.
     ecx.push_stack_frame_raw(

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -252,7 +252,7 @@ impl<'tcx> CompileTimeInterpCx<'tcx> {
             }
 
             let msg = Symbol::intern(self.read_str(&msg_place)?);
-            let span = self.find_closest_untracked_caller_location();
+            let span = self.caller_location();
             let (file, line, col) = self.location_triple_for_span(span);
             return Err(ConstEvalErrKind::Panic { msg, file, line, col }).into();
         } else if self.tcx.is_lang_item(def_id, LangItem::PanicFmt) {
@@ -469,7 +469,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
 
     fn panic_nounwind(ecx: &mut InterpCx<'tcx, Self>, msg: &str) -> InterpResult<'tcx> {
         let msg = Symbol::intern(msg);
-        let span = ecx.find_closest_untracked_caller_location();
+        let span = ecx.caller_location();
         let (file, line, col) = ecx.location_triple_for_span(span);
         Err(ConstEvalErrKind::Panic { msg, file, line, col }).into()
     }

compiler/rustc_const_eval/src/interpret/call.rs

@@ -462,6 +462,21 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         let _trace = enter_trace_span!(M, step::init_stack_frame, %instance, tracing_separate_thread = Empty);
         let def_id = instance.def_id();
 
+        match (instance.def.requires_caller_location(*self.tcx), with_caller_location) {
+            (false, true) => {
+                throw_ub_format!(
+                    "calling a non-`#[track_caller]` function from a `#[track_caller]` declaration"
+                )
+            }
+            (true, false) => {
+                throw_ub_format!(
+                    "calling a `#[track_caller]` function from a non-`#[track_caller]` declaration"
+                )
+            }
+            _ => {}
+        }
+        let caller_location = with_caller_location.then(|| self.caller_location());
+
         // The first order of business is to figure out the callee signature.
         // However, that requires the list of variadic arguments.
         // We use the *caller* information to determine where to split the list of arguments,
@@ -542,6 +557,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 .collect::<Vec<_>>()
         );
 
+        // Initialize special caller_location "local", if this function has it.
+        self.frame_mut().track_caller_arg = caller_location;
+
         // Determine whether there is a special VaList argument. This is always the
         // last argument, and since arguments start at index 1 that's `arg_count`.
         let va_list_arg = callee_fn_abi.c_variadic.then(|| mir::Local::from_usize(body.arg_count));
@@ -563,7 +581,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         // The "where they come from" part is easy, we expect the caller to do any special handling
         // that might be required here (e.g. for untupling).
         // If `with_caller_location` is set we pretend there is an extra argument (that
-        // we will not pass; our `caller_location` intrinsic implementation walks the stack instead).
+        // we will not pass normally; our `caller_location` intrinsic implementation handles
+        // that separately).
         assert_eq!(
             args.len() + if with_caller_location { 1 } else { 0 },
             caller_fn_abi.args.len(),
@@ -760,6 +779,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         match instance.def {
             ty::InstanceKind::Intrinsic(def_id) => {
                 assert!(self.tcx.intrinsic(def_id).is_some());
+                assert!(!with_caller_location);
                 // FIXME: Should `InPlace` arguments be reset to uninit?
                 if let Some(fallback) = M::call_intrinsic(
                     self,
@@ -771,11 +791,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 )? {
                     assert!(!self.tcx.intrinsic(fallback.def_id()).unwrap().must_be_overridden);
                     assert_matches!(fallback.def, ty::InstanceKind::Item(_));
+                    // For `#[rustc_intrinsic] fn`, two `Instance`s are involved:
+                    // the original `InstanceKind::Intrinsic` (no caller location arg), and
+                    // the fallback `InstanceKind::Item` (has caller location arg if
+                    // it's `#[track_caller]`).
+                    // We replace the former with the latter.
                     return self.init_fn_call(
                         FnVal::Instance(fallback),
                         (caller_abi, caller_fn_abi),
                         args,
-                        with_caller_location,
+                        fallback.def.requires_caller_location(*self.tcx),
                         destination,
                         target,
                         unwind,

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -1,5 +1,6 @@
 use std::cell::RefCell;
 use std::collections::hash_map::Entry;
+use std::convert::identity;
 
 use either::{Left, Right};
 use rustc_abi::{Align, HasDataLayout, Size, TargetDataLayout};
@@ -18,7 +19,7 @@ use rustc_middle::ty::{
 use rustc_span::{Span, span_bug};
 use rustc_structures::Limit;
 use rustc_target::callconv::FnAbi;
-use tracing::{debug, trace};
+use tracing::trace;
 
 use super::{
     Frame, FrameInfo, GlobalId, InterpErrorKind, InterpResult, MPlaceTy, Machine, MemPlaceMeta,
@@ -376,49 +377,41 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         }
     }
 
-    /// Walks up the callstack from the intrinsic's callsite, searching for the first callsite in a
-    /// frame which is not `#[track_caller]`. This matches the `caller_location` intrinsic,
+    /// Grabs the implicit caller location argument, if there is one,
+    /// or falls back to the "current" span. This matches the `caller_location` intrinsic,
     /// and is primarily intended for the panic machinery.
-    pub(crate) fn find_closest_untracked_caller_location(&self) -> Span {
-        for frame in self.stack().iter().rev() {
-            debug!("find_closest_untracked_caller_location: checking frame {:?}", frame.instance);
-
-            // Assert that the frame we look at is actually executing code currently
-            // (`loc` is `Right` when we are unwinding and the frame does not require cleanup).
-            let loc = frame.loc.left().unwrap();
-
-            // This could be a non-`Call` terminator (such as `Drop`), or not a terminator at all
-            // (such as `box`). Use the normal span by default.
-            let mut source_info = *frame.body.source_info(loc);
-
-            // If this is a `Call` terminator, use the `fn_span` instead.
-            let block = &frame.body.basic_blocks[loc.block];
-            if loc.statement_index == block.statements.len() {
-                debug!(
-                    "find_closest_untracked_caller_location: got terminator {:?} ({:?})",
-                    block.terminator(),
-                    block.terminator().kind,
-                );
-                if let mir::TerminatorKind::Call { fn_span, .. } = block.terminator().kind {
-                    source_info.span = fn_span;
-                }
-            }
-
-            let caller_location = if frame.instance.def.requires_caller_location(*self.tcx) {
-                // We use `Err(())` as indication that we should continue up the call stack since
-                // this is a `#[track_caller]` function.
-                Some(Err(()))
-            } else {
-                None
-            };
-            if let Ok(span) =
-                frame.body.caller_location_span(source_info, caller_location, *self.tcx, Ok)
-            {
-                return span;
-            }
+    pub(crate) fn caller_location(&self) -> Span {
+        let frame = self.frame();
+
+        // Assert that the frame we look at is actually executing code currently
+        // (`loc` is `Right` when we are unwinding and the frame does not require cleanup).
+        let loc = frame.loc.left().unwrap();
+
+        // FIXME: It would be nice to have the following redundant sanity check.
+        // However, `requires_caller_location` would return true on a promoted that's
+        // inside a `#[track_caller]` function, and we currently don't keep track of
+        // enough information to identify such cases happening.
+        // assert_eq!(
+        //     frame.track_caller_arg.is_some(),
+        //     frame.instance.def.requires_caller_location(*self.tcx)
+        // );
+
+        // This could be a non-`Call` terminator (such as `Drop`), or not a terminator at all
+        // (such as `box`). Use the normal span by default.
+        let mut source_info = *frame.body.source_info(loc);
+
+        // If this is a `Call` terminator, use the `fn_span` instead.
+        let block = &frame.body.basic_blocks[loc.block];
+        if loc.statement_index == block.statements.len()
+            && let mir::TerminatorKind::Call { fn_span, .. } = block.terminator().kind
+        {
+            source_info.span = fn_span;
         }
 
-        span_bug!(self.cur_span(), "no non-`#[track_caller]` frame found")
+        // Directly returning `frame.track_caller_arg` would be wrong if we are
+        // in code that got inlined from another function. `caller_location_span`
+        // handles that for us.
+        frame.body.caller_location_span(source_info, frame.track_caller_arg, *self.tcx, identity)
     }
 
     /// Returns the actual dynamic size and alignment of the place at the given type.

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -293,7 +293,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             }
 
             sym::caller_location => {
-                let span = self.find_closest_untracked_caller_location();
+                let span = self.caller_location();
                 let val = self.tcx.span_as_caller_location(span);
                 let val =
                     self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;

compiler/rustc_const_eval/src/interpret/stack.rs

@@ -66,6 +66,8 @@ pub struct Frame<'tcx, Prov: Provenance = CtfeProvenance, Extra = ()> {
     pub(super) body: &'tcx mir::Body<'tcx>,
 
     /// The def_id and args of the current function.
+    /// Note that if this is the body of a promoted, the `instance` points
+    /// to the surrounding body!
     pub(super) instance: ty::Instance<'tcx>,
 
     /// Extra data for the machine.
@@ -96,6 +98,10 @@ pub struct Frame<'tcx, Prov: Provenance = CtfeProvenance, Extra = ()> {
     /// frame is popped.
     pub(super) va_list: Vec<MPlaceTy<'tcx, Prov>>,
 
+    /// The implicit caller location argument that is passed by this function's caller, if any.
+    /// This is `Some` if and only if `instance.def.requires_caller_location` returns `true`.
+    pub(super) track_caller_arg: Option<Span>,
+
     /// The span of the `tracing` crate is stored here.
     /// When the guard is dropped, the span is exited. This gives us
     /// a full stack trace on all tracing statements.
@@ -258,13 +264,14 @@ impl<'tcx, Prov: Provenance> Frame<'tcx, Prov> {
         Frame {
             body: self.body,
             instance: self.instance,
+            extra,
             return_cont: self.return_cont,
             return_place: self.return_place,
             locals: self.locals,
             va_list: self.va_list,
-            loc: self.loc,
-            extra,
+            track_caller_arg: self.track_caller_arg,
             tracing_span: self.tracing_span,
+            loc: self.loc,
         }
     }
 }
@@ -400,6 +407,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             return_place: return_place.clone(),
             locals,
             va_list: vec![],
+            track_caller_arg: None,
             instance,
             tracing_span: SpanGuard::new(),
             extra: (),

compiler/rustc_const_eval/src/interpret/step.rs

@@ -496,15 +496,33 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             }
             ty::FnDef(def_id, args) => {
                 let instance = self.resolve(def_id, args.no_bound_vars().unwrap())?;
-                // Don't compute FnAbi for LLVM intrinsics. Trying to do that would panic.
-                // Rust intrinsics however *do* need a FnAbi as we may invoke the
-                // fallback body like a regular function.
-                let has_fn_abi = !matches!(instance.def, ty::InstanceKind::LlvmIntrinsic(_));
+                let fn_abi =
+                    match instance.def {
+                        // Don't compute FnAbi for LLVM intrinsics. Trying to do that would panic.
+                        ty::InstanceKind::LlvmIntrinsic(_) => None,
+                        // Rust intrinsics don't really have a normal ABI, except for the case
+                        // where we call the fallback body like a regular function.
+                        // So, we compute the ABI based on the fallback body.
+                        // We can't compute the ABI based on the `instance` directly,
+                        // because `#[track_caller]` affects the ABI of the fallback body,
+                        // but not affect the "ABI" of the intrinsic itself.
+                        // Even if there is no fallback body, this computes a valid ABI
+                        // (that we then mostly ignore).
+                        ty::InstanceKind::Intrinsic(def_id) => {
+                            let fallback_instance = ty::Instance {
+                                def: ty::InstanceKind::Item(def_id),
+                                args: instance.args,
+                            };
+                            Some(self.fn_abi_of_instance_no_deduced_attrs(
+                                fallback_instance,
+                                extra_args,
+                            )?)
+                        }
+                        _ => Some(self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)?),
+                    };
                 (
                     FnVal::Instance(instance),
-                    has_fn_abi
-                        .then(|| self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args))
-                        .transpose()?,
+                    fn_abi,
                     instance.def.requires_caller_location(*self.tcx),
                 )
             }

compiler/rustc_const_eval/src/interpret/util.rs

@@ -25,7 +25,7 @@ pub(crate) fn type_implements_dyn_trait<'tcx, M: Machine<'tcx>>(
 
     let ty::Dynamic(preds, _) = trait_ty.kind() else {
         span_bug!(
-            ecx.find_closest_untracked_caller_location(),
+            ecx.cur_span(),
             "Invalid type provided to type_implements_predicates. U must be dyn Trait, got {trait_ty}."
         );
     };

compiler/rustc_type_ir/src/binder.rs

@@ -136,6 +136,7 @@ impl<I: Interner, T> Binder<I, T> {
         self.as_ref().map_bound(f)
     }
 
+    #[inline]
     pub fn map_bound<F, U: TypeVisitable<I>>(self, f: F) -> Binder<I, U>
     where
         F: FnOnce(T) -> U,
@@ -152,6 +153,7 @@ impl<I: Interner, T> Binder<I, T> {
     /// There are some uses of `map_bound` which are incredibly hot and trivially
     /// correct. Only use this if this significantly improves the performance of
     /// builds with debug assertions.
+    #[inline]
     pub fn map_bound_no_validate_bound_vars<F, U: TypeVisitable<I>>(self, f: F) -> Binder<I, U>
     where
         F: FnOnce(T) -> U,