diff --git a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs index 1ba48e6829070..010d398427cea 100644 --- a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs +++ b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs @@ -3,6 +3,7 @@ use rustc_abi::{BackendRepr, CanonAbi, ExternAbi, RegKind, X86Call}; use rustc_hir::{CRATE_HIR_ID, HirId}; use rustc_middle::mir::{self, Location, traversal}; +use rustc_middle::span_bug; use rustc_middle::ty::layout::{FnAbiRequest, codegen_handle_fn_abi_err}; use rustc_middle::ty::{self, Instance, InstanceKind, Ty, TyCtxt}; use rustc_span::def_id::DefId; @@ -212,12 +213,44 @@ fn check_call_site_abi<'tcx>( loc: impl Fn() -> (Span, HirId) + Copy, ) { let extern_abi = callee.fn_sig(tcx).abi(); - if extern_abi.is_rustic_abi() || extern_abi == ExternAbi::LlvmIntrinsic { + if extern_abi.is_rustic_abi() { // We directly handle the soundness of Rust ABIs -- so let's skip the majority of // call sites to avoid a perf regression. + return; + } + if extern_abi == ExternAbi::LlvmIntrinsic { // We disable all checks for the llvm-intrinsic ABI to allow linking to arbitrary // LLVM intrinsics - return; + let typing_env = ty::TypingEnv::fully_monomorphized(); + match *callee.kind() { + ty::FnDef(def_id, args) => { + let instance = ty::Instance::expect_resolve( + tcx, + typing_env, + def_id, + args.no_bound_vars().unwrap(), + DUMMY_SP, + ); + let InstanceKind::LlvmIntrinsic(..) = instance.def else { + span_bug!( + loc().0, + "`extern \"llvm-intrinsic\"` may only be used for LLVM intrinsics" + ); + }; + let name = tcx + .codegen_fn_attrs(def_id) + .symbol_name + .expect("llvm-intrinsic must be foreign function"); + assert!( + name.as_str().starts_with("llvm."), + "llvm-intrinsic name must start with llvm." + ); + return; + } + _ => { + span_bug!(loc().0, "`extern \"llvm-intrinsic\"` may only be used for direct calls") + } + }; } let typing_env = ty::TypingEnv::fully_monomorphized(); let callee_abi = match *callee.kind() { @@ -250,8 +283,15 @@ fn check_call_site_abi<'tcx>( DUMMY_SP, ); if let InstanceKind::LlvmIntrinsic(..) = instance.def { - // LLVM intrinsics don't have an ABI, so there is nothing to check. - return; + let name = tcx + .codegen_fn_attrs(def_id) + .symbol_name + .expect("llvm-intrinsic must be foreign function"); + // llvm.wasm.throw is the only unwinding LLVM intrinsic we have. + // FIXME maybe replace with inline asm? + if name.as_str() != "llvm.wasm.throw" { + span_bug!(loc().0, "llvm.* function must use `extern \"llvm-intrinsic\"`"); + } } match tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty()))) { Ok(callee_abi) => callee_abi, diff --git a/tests/codegen-llvm/call-llvm-intrinsics.rs b/tests/codegen-llvm/call-llvm-intrinsics.rs index dc7e0249cb6ac..951ad64688396 100644 --- a/tests/codegen-llvm/call-llvm-intrinsics.rs +++ b/tests/codegen-llvm/call-llvm-intrinsics.rs @@ -14,7 +14,7 @@ impl Drop for A { } } -extern "C" { +extern "llvm-intrinsic" { #[link_name = "llvm.sqrt.f32"] fn sqrt(x: f32) -> f32; } diff --git a/tests/codegen-llvm/loongarch-abi/call-llvm-intrinsics.rs b/tests/codegen-llvm/loongarch-abi/call-llvm-intrinsics.rs index 36eb2dde7afbc..bb652459a4cf4 100644 --- a/tests/codegen-llvm/loongarch-abi/call-llvm-intrinsics.rs +++ b/tests/codegen-llvm/loongarch-abi/call-llvm-intrinsics.rs @@ -13,7 +13,7 @@ impl Drop for A { } } -extern "C" { +extern "llvm-intrinsic" { #[link_name = "llvm.sqrt.f32"] fn sqrt(x: f32) -> f32; } diff --git a/tests/codegen-llvm/riscv-abi/call-llvm-intrinsics.rs b/tests/codegen-llvm/riscv-abi/call-llvm-intrinsics.rs index c384eeae39fb0..0576cd386ba15 100644 --- a/tests/codegen-llvm/riscv-abi/call-llvm-intrinsics.rs +++ b/tests/codegen-llvm/riscv-abi/call-llvm-intrinsics.rs @@ -15,7 +15,7 @@ extern crate minicore; use minicore::*; -extern "C" { +extern "llvm-intrinsic" { #[link_name = "llvm.sqrt.f32"] fn sqrt(x: f32) -> f32; } diff --git a/tests/run-make/simd-ffi/simd.rs b/tests/run-make/simd-ffi/simd.rs index 3f12dabdb65ea..54419b0c0dbda 100644 --- a/tests/run-make/simd-ffi/simd.rs +++ b/tests/run-make/simd-ffi/simd.rs @@ -10,7 +10,7 @@ #[repr(simd)] pub struct f32x4([f32; 4]); -extern "C" { +extern "llvm-intrinsic" { #[link_name = "llvm.sqrt.v4f32"] fn vsqrt(x: f32x4) -> f32x4; } @@ -23,7 +23,7 @@ pub fn foo(x: f32x4) -> f32x4 { #[repr(simd)] pub struct i32x4([i32; 4]); -extern "C" { +extern "llvm-intrinsic" { // _mm_sll_epi32 #[cfg(all(any(target_arch = "x86", target_arch = "x86-64"), target_feature = "sse2"))] #[link_name = "llvm.x86.sse2.psll.d"] diff --git a/tests/run-make/wasm-exceptions-nostd/src/panicking.rs b/tests/run-make/wasm-exceptions-nostd/src/panicking.rs index 414c9f6a1650b..ccc569039a084 100644 --- a/tests/run-make/wasm-exceptions-nostd/src/panicking.rs +++ b/tests/run-make/wasm-exceptions-nostd/src/panicking.rs @@ -2,7 +2,7 @@ fn eh_personality() {} mod internal { - extern "C" { + extern "C-unwind" { #[link_name = "llvm.wasm.throw"] pub fn wasm_throw(tag: i32, ptr: *mut u8) -> !; } diff --git a/tests/ui/feature-gates/link-llvm-intrinsics-enabled.rs b/tests/ui/feature-gates/link-llvm-intrinsics-enabled.rs index 9118950d568fd..1a228a7f603c7 100644 --- a/tests/ui/feature-gates/link-llvm-intrinsics-enabled.rs +++ b/tests/ui/feature-gates/link-llvm-intrinsics-enabled.rs @@ -4,7 +4,7 @@ #![allow(dead_code)] #![feature(link_llvm_intrinsics)] -extern "C" { +extern "llvm-intrinsic" { #[link_name = "llvm.sqrt.f32"] fn sqrt(x: f32) -> f32; } diff --git a/tests/ui/scalable-vectors/async.rs b/tests/ui/scalable-vectors/async.rs index 916c64ee74358..d5c18a7d22b0f 100644 --- a/tests/ui/scalable-vectors/async.rs +++ b/tests/ui/scalable-vectors/async.rs @@ -17,7 +17,7 @@ pub struct svint32_t(i32); #[target_feature(enable = "sve")] pub unsafe fn svdup_n_s32(op: i32) -> svint32_t { - extern "C" { + extern "llvm-intrinsic" { #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.dup.x.nxv4i32")] fn _svdup_n_s32(op: i32) -> svint32_t; } diff --git a/tests/ui/scalable-vectors/closure-capture.rs b/tests/ui/scalable-vectors/closure-capture.rs index d6a45f76e214d..f06c6a1218ea1 100644 --- a/tests/ui/scalable-vectors/closure-capture.rs +++ b/tests/ui/scalable-vectors/closure-capture.rs @@ -16,7 +16,7 @@ pub struct svint32_t(i32); #[inline(never)] #[target_feature(enable = "sve")] pub unsafe fn svdup_n_s32(op: i32) -> svint32_t { - extern "C" { + extern "llvm-intrinsic" { #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.dup.x.nxv4i32")] fn _svdup_n_s32(op: i32) -> svint32_t; } @@ -26,7 +26,7 @@ pub unsafe fn svdup_n_s32(op: i32) -> svint32_t { #[inline] #[target_feature(enable = "sve,sve2")] pub unsafe fn svxar_n_s32(op1: svint32_t, op2: svint32_t) -> svint32_t { - extern "C" { + extern "llvm-intrinsic" { #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.xar.nxv4i32")] fn _svxar_n_s32(op1: svint32_t, op2: svint32_t, imm3: i32) -> svint32_t; } diff --git a/tests/ui/scalable-vectors/copy-clone.rs b/tests/ui/scalable-vectors/copy-clone.rs index 7576b6ec18dcb..d0623e34f755c 100644 --- a/tests/ui/scalable-vectors/copy-clone.rs +++ b/tests/ui/scalable-vectors/copy-clone.rs @@ -9,7 +9,7 @@ pub struct svint32_t(i32); #[target_feature(enable = "sve")] pub unsafe fn svdup_n_s32(op: i32) -> svint32_t { - extern "C" { + extern "llvm-intrinsic" { #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.dup.x.nxv4i32")] fn _svdup_n_s32(op: i32) -> svint32_t; //~^ WARN: `extern` block uses type `svint32_t`, which is not FFI-safe diff --git a/tests/ui/scalable-vectors/debuginfo-does-not-spill.rs b/tests/ui/scalable-vectors/debuginfo-does-not-spill.rs index f7cc455dc8efd..c7df5c24f284b 100644 --- a/tests/ui/scalable-vectors/debuginfo-does-not-spill.rs +++ b/tests/ui/scalable-vectors/debuginfo-does-not-spill.rs @@ -22,7 +22,7 @@ pub struct svbool4_t(bool); impl std::convert::Into for svbool4_t { #[inline(always)] fn into(self) -> svbool_t { - unsafe extern "C" { + unsafe extern "llvm-intrinsic" { #[link_name = "llvm.aarch64.sve.convert.to.svbool.nxv4i1"] fn convert_to_svbool(b: svbool4_t) -> svbool_t; } @@ -31,7 +31,7 @@ impl std::convert::Into for svbool4_t { } pub fn svwhilelt_b32_u64(op1: u64, op2: u64) -> svbool_t { - unsafe extern "C" { + unsafe extern "llvm-intrinsic" { #[link_name = "llvm.aarch64.sve.whilelo.nxv4i1.u64"] fn _svwhilelt_b32_u64(op1: u64, op2: u64) -> svbool4_t; } diff --git a/tests/ui/scalable-vectors/require-target-feature.rs b/tests/ui/scalable-vectors/require-target-feature.rs index 42194d2351a90..248a1db1c43eb 100644 --- a/tests/ui/scalable-vectors/require-target-feature.rs +++ b/tests/ui/scalable-vectors/require-target-feature.rs @@ -25,7 +25,7 @@ impl Copy for svint32_t {} #[inline(never)] #[target_feature(enable = "sve")] pub unsafe fn svdup_n_s32(op: i32) -> svint32_t { - extern "C" { + extern "llvm-intrinsic" { #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.dup.x.nxv4i32")] fn _svdup_n_s32(op: i32) -> svint32_t; //~^ WARN: `extern` block uses type `svint32_t`, which is not FFI-safe diff --git a/tests/ui/scalable-vectors/value-type.rs b/tests/ui/scalable-vectors/value-type.rs index 31a9ee8344eb5..639e1a781b17d 100644 --- a/tests/ui/scalable-vectors/value-type.rs +++ b/tests/ui/scalable-vectors/value-type.rs @@ -15,7 +15,7 @@ pub struct svint32_t(i32); #[target_feature(enable = "sve")] pub unsafe fn svdup_n_s32(op: i32) -> svint32_t { - extern "C" { + extern "llvm-intrinsic" { #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.dup.x.nxv4i32")] fn _svdup_n_s32(op: i32) -> svint32_t; //~^ WARN: `extern` block uses type `svint32_t`, which is not FFI-safe