feat(rust) add native link option

.github/workflows/main.yml

@@ -178,6 +178,23 @@ jobs:
     - run: rustup component add rust-src
     - run: cargo miri test -p wit-bindgen --all-features
 
+  native_e2e:
+    name: Native E2E
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Install Rust
+      run: rustup update stable --no-self-update && rustup default stable
+    # Builds the plugin in `crates/rust/tests/native-e2e` as a native cdylib,
+    # loads it, and drives it through the import resolver and core ABI.
+    - run: cargo test -p wit-bindgen-rust --test native_e2e
+
   check:
     name: Check
     runs-on: ubuntu-latest
@@ -269,6 +286,7 @@ jobs:
     needs:
       - test
       - test_unit
+      - native_e2e
       - rustfmt
       - build
       - verify-publish

.gitignore

@@ -11,3 +11,7 @@ crates/guest-rust/src/cabi_realloc.o
 wit_component
 
 /wit-bindgen.sln
+
+# Built by crates/rust/tests/native_e2e.rs
+crates/rust/tests/native-e2e/Cargo.lock
+crates/rust/tests/native-e2e/target

crates/core/src/lib.rs

@@ -16,6 +16,7 @@ mod async_;
 pub use async_::AsyncFilterSet;
 mod chainable_method;
 pub use chainable_method::{ChainableMethodFilterSet, ChainingMode};
+pub mod symbol_name;
 
 #[derive(Default, Copy, Clone, PartialEq, Eq, Debug)]
 pub enum Direction {

crates/cpp/src/symbol_name.rs → crates/core/src/symbol_name.rs

@@ -1,4 +1,4 @@
-use wit_bindgen_core::abi;
+use crate::abi;
 
 fn hexdigit(v: u32) -> char {
     if v < 10 {

crates/cpp/src/lib.rs

@@ -9,12 +9,13 @@ use std::{
     process::{Command, Stdio},
     str::FromStr,
 };
-use symbol_name::{make_external_component, make_external_symbol};
 use wit_bindgen_c::to_c_ident;
 use wit_bindgen_core::{
     Files, InterfaceGenerator, Source, Types, WorldGenerator,
     abi::{self, AbiVariant, Bindgen, Bitcast, LiftLower, WasmSignature, WasmType},
-    name_package_module, uwrite, uwriteln,
+    name_package_module,
+    symbol_name::{make_external_component, make_external_symbol},
+    uwrite, uwriteln,
     wit_parser::{
         Alignment, ArchitectureSize, Docs, Function, FunctionKind, Handle, Int, InterfaceId, Param,
         Resolve, SizeAlign, Stability, Type, TypeDef, TypeDefKind, TypeId, TypeOwner, WorldId,
@@ -24,7 +25,6 @@ use wit_bindgen_core::{
 use wit_parser::TypeIdVisitor;
 
 // mod wamr;
-mod symbol_name;
 
 pub const RESOURCE_IMPORT_BASE_CLASS_NAME: &str = "ResourceImportBase";
 pub const RESOURCE_EXPORT_BASE_CLASS_NAME: &str = "ResourceExportBase";

crates/guest-rust/src/lib.rs

@@ -894,6 +894,49 @@ extern crate std;
 /// });
 /// ```
 ///
+/// ## Native (non-WebAssembly) targets
+///
+/// Generated bindings also compile for native targets, which is useful for
+/// testing component code without a wasm runtime or for building it as a
+/// `cdylib` plugin. Native linkers don't accept the `:`, `/`, `#`, `[` and
+/// `]` characters that canonical ABI symbol names use, so on native targets
+/// symbols are hex-encoded with the scheme in
+/// `wit_bindgen_core::symbol_name` (the same one the C++ generator uses).
+///
+/// Imports are not resolved by the native linker. Each import instead calls
+/// through a function pointer looked up on first use from a host-installed
+/// resolver. After loading the library a host calls the exported
+///
+/// ```c
+/// void __wit_bindgen_set_import_resolver(
+///     void *(*resolver)(void *ctx, const char *module, const char *name),
+///     void *ctx);
+/// ```
+///
+/// with a callback mapping an import's core module and function name (e.g.
+/// `my:pkg/iface@1.0.0` and `[method]res.frob`) to a function pointer with
+/// the import's core signature, or null if the host doesn't implement it
+/// (see `wit_bindgen::rt::ImportResolver`). Everything links whether or not
+/// a host is present, and calling an import with no implementation aborts
+/// with a message naming it. Install the resolver once, before calling any
+/// export: each import caches the pointer it was given.
+///
+/// Exports, including post-return functions, async callbacks, and resource
+/// destructors, are exported under their hex-encoded core export names. The
+/// runtime crate also exports `__wit_bindgen_cabi_realloc` so hosts can
+/// allocate guest-owned memory when lowering arguments.
+///
+/// Each world additionally exports a marker
+/// `const char *__wit_bindgen_world_<world>(void)`, where `<world>` is the
+/// hex-encoded `<package>/<world><type_section_suffix>` (e.g.
+/// `my:pkg@1.0.0/my-world`), which returns that name. `dlsym` can't otherwise
+/// tell a host whether a library implements the world it expects, and calling
+/// into the wrong world corrupts memory rather than failing, so hosts should
+/// look this symbol up before anything else. The marker is keyed like the
+/// `component-type` custom section on wasm: binding the same world twice in
+/// one binary needs a `type_section_suffix` (and `export_prefix` for the
+/// export names).
+///
 /// [WIT package]: https://component-model.bytecodealliance.org/design/packages.html
 #[cfg(feature = "macros")]
 pub use wit_bindgen_rust_macro::generate;

crates/guest-rust/src/rt/async_support.rs

@@ -27,33 +27,49 @@ macro_rules! rtdebug {
 
 /// Helper macro to deduplicate foreign definitions of wasm functions.
 ///
-/// This automatically imports when on wasm targets and then defines a dummy
-/// panicking shim for native targets to support native compilation but fail at
-/// runtime.
+/// On wasm targets this declares the canonical ABI built-ins as ordinary
+/// linker-resolved imports. On native targets each one instead becomes a shim
+/// that asks the host's import resolver for its implementation on first call
+/// (see `rt::native_imports`), identified by the same module and name, so the
+/// generated code is the same on both targets and only who satisfies the
+/// import differs.
 macro_rules! extern_wasm {
     (
-        $(#[$extern_attr:meta])*
+        #[link(wasm_import_module = $module:literal)]
         unsafe extern "C" {
             $(
-                $(#[$func_attr:meta])*
-                $vis:vis fn $func_name:ident ( $($args:tt)* ) $(-> $ret:ty)?;
+                #[link_name = $name:literal]
+                $vis:vis fn $func_name:ident ( $($arg:ident : $ty:ty),* $(,)? ) $(-> $ret:ty)?;
             )*
         }
     ) => {
         $(
             #[cfg(not(target_family = "wasm"))]
-            #[allow(unused, reason = "dummy shim for non-wasm compilation, never invoked")]
-            $vis unsafe fn $func_name($($args)*) $(-> $ret)? {
-                unreachable!();
+            #[allow(dead_code, reason = "mirrors the wasm import set even if unused natively")]
+            $vis unsafe fn $func_name($($arg: $ty),*) $(-> $ret)? {
+                static CACHE: ::core::sync::atomic::AtomicPtr<()> =
+                    ::core::sync::atomic::AtomicPtr::new(::core::ptr::null_mut());
+                // Named so as not to shadow any parameter.
+                let mut __impl = CACHE.load(::core::sync::atomic::Ordering::Acquire);
+                if __impl.is_null() {
+                    __impl = crate::rt::resolve_import(
+                        crate::rt::native_imports::cstr(concat!($module, "\0")),
+                        crate::rt::native_imports::cstr(concat!($name, "\0")),
+                    );
+                    CACHE.store(__impl, ::core::sync::atomic::Ordering::Release);
+                }
+                let __func: unsafe extern "C" fn($($ty),*) $(-> $ret)? =
+                    unsafe { ::core::mem::transmute(__impl) };
+                unsafe { __func($($arg),*) }
             }
         )*
 
         #[cfg(target_family = "wasm")]
-        $(#[$extern_attr])*
+        #[link(wasm_import_module = $module)]
         unsafe extern "C" {
             $(
-                $(#[$func_attr])*
-                $vis fn $func_name($($args)*) $(-> $ret)?;
+                #[link_name = $name]
+                $vis fn $func_name($($arg: $ty),*) $(-> $ret)?;
             )*
         }
     };

crates/guest-rust/src/rt/async_support/cabi.rs

@@ -76,21 +76,42 @@
 use alloc::boxed::Box;
 use core::ffi::c_void;
 
-extern_wasm! {
-    unsafe extern "C" {
-        /// Sets the global task pointer to `ptr` provided. Returns the previous
-        /// value.
-        ///
-        /// This function acts as a dual getter and a setter. To get the
-        /// current task pointer a dummy `ptr` can be provided (e.g. NULL) and then
-        /// it's passed back when you're done working with it. When setting the
-        /// current task pointer it's recommended to call this and then call it
-        /// again with the previous value when the tasks's work is done.
-        ///
-        /// For executors they need to ensure that the `ptr` passed in lives for
-        /// the entire lifetime of the component model task.
-        pub fn wasip3_task_set(ptr: *mut wasip3_task) -> *mut wasip3_task;
-    }
+#[cfg(target_family = "wasm")]
+unsafe extern "C" {
+    /// Sets the global task pointer to `ptr` provided. Returns the previous
+    /// value.
+    ///
+    /// This function acts as a dual getter and a setter. To get the
+    /// current task pointer a dummy `ptr` can be provided (e.g. NULL) and then
+    /// it's passed back when you're done working with it. When setting the
+    /// current task pointer it's recommended to call this and then call it
+    /// again with the previous value when the tasks's work is done.
+    ///
+    /// For executors they need to ensure that the `ptr` passed in lives for
+    /// the entire lifetime of the component model task.
+    pub fn wasip3_task_set(ptr: *mut wasip3_task) -> *mut wasip3_task;
+}
+
+/// Native counterpart of the C-defined `wasip3_task_set` above. Uses
+/// thread local so there is no possible panic on multi thread when polling
+/// two async exports at the same time.
+#[cfg(all(not(target_family = "wasm"), feature = "std"))]
+pub unsafe fn wasip3_task_set(ptr: *mut wasip3_task) -> *mut wasip3_task {
+    use core::cell::Cell;
+    std::thread_local!(
+        static CURRENT: Cell<*mut wasip3_task> = const { Cell::new(core::ptr::null_mut()) }
+    );
+    CURRENT.with(|current| current.replace(ptr))
+}
+
+/// Without `std` there are no thread-locals on stable Rust, so this falls
+/// back to one global slot, which requires the host to not poll two async
+/// exports at the same time.
+#[cfg(all(not(target_family = "wasm"), not(feature = "std")))]
+pub unsafe fn wasip3_task_set(ptr: *mut wasip3_task) -> *mut wasip3_task {
+    use core::sync::atomic::{AtomicPtr, Ordering};
+    static CURRENT: AtomicPtr<wasip3_task> = AtomicPtr::new(core::ptr::null_mut());
+    CURRENT.swap(ptr, Ordering::AcqRel)
 }
 
 /// The first version of `wasip3_task` which implies the existence of the

crates/guest-rust/src/rt/async_support/error_context.rs

@@ -74,10 +74,10 @@ extern_wasm! {
     #[link(wasm_import_module = "$root")]
     unsafe extern "C" {
         #[link_name = "[error-context-new-utf8]"]
-        fn new(_: *const u8, _: usize) -> u32;
+        fn new(ptr: *const u8, len: usize) -> u32;
         #[link_name = "[error-context-drop]"]
-        fn drop(_: u32);
+        fn drop(handle: u32);
         #[link_name = "[error-context-debug-message-utf8]"]
-        fn debug_message(_: u32, _: &mut RetPtr);
+        fn debug_message(handle: u32, ret: &mut RetPtr);
     }
 }

crates/guest-rust/src/rt/async_support/waitable_set.rs

@@ -75,8 +75,8 @@ extern_wasm! {
         #[link_name = "[waitable-join]"]
         fn join(waitable: u32, set: u32);
         #[link_name = "[waitable-set-wait]"]
-        fn wait(_: u32, _: *mut [u32; 2]) -> u32;
+        fn wait(set: u32, event: *mut [u32; 2]) -> u32;
         #[link_name = "[waitable-set-poll]"]
-        fn poll(_: u32, _: *mut [u32; 2]) -> u32;
+        fn poll(set: u32, event: *mut [u32; 2]) -> u32;
     }
 }

crates/guest-rust/src/rt/mod.rs

@@ -153,7 +153,7 @@ pub fn maybe_link_cabi_realloc() {
 /// `cabi_realloc` module above. It's otherwise never explicitly called.
 ///
 /// For more information about this see `./ci/rebuild-libwit-bindgen-cabi.sh`.
-#[cfg(any(target_env = "p1", target_env = ""))]
+#[cfg(any(target_env = "p1", target_env = "", not(target_arch = "wasm32")))]
 pub unsafe fn cabi_realloc(
     old_ptr: *mut u8,
     old_len: usize,
@@ -192,6 +192,12 @@ pub unsafe fn cabi_realloc(
     return ptr;
 }
 
+#[cfg(not(target_arch = "wasm32"))]
+mod native_imports;
+
+#[cfg(not(target_arch = "wasm32"))]
+pub use native_imports::{ImportResolver, resolve_import};
+
 /// Provide a hook for generated export functions to run static constructors at
 /// most once.
 ///