diff --git a/Makefile b/Makefile index 99d83ac..76b70ea 100644 --- a/Makefile +++ b/Makefile @@ -52,10 +52,12 @@ cross: python3 tools/check-entry.py # kore-host drives real image files under POSIX record locks, which Miri does -# not model; `make test` exercises its procedure. +# not model; `make test` exercises its procedure. kore-rt's `bridge` suite runs +# the worker/host state machine on real threads, so Miri checks every hand-over +# of the cell the Lua VM and a C program's host share with thread 0. miri: cargo miri test --locked --workspace --all-features --exclude kore-wasi --exclude kore-host - cargo miri test --locked --manifest-path native/Cargo.toml -p kore-rt erased_poll_and_drop + cargo miri test --locked --manifest-path native/Cargo.toml -p kore-rt -- erased_poll_and_drop bridge:: native-check: python3 tools/check-native.py diff --git a/crates/c/src/program.rs b/crates/c/src/program.rs index cc57c56..d0bc2cb 100644 --- a/crates/c/src/program.rs +++ b/crates/c/src/program.rs @@ -12,10 +12,10 @@ //! founding's, sized for a runtime, and thread 0 is the process owner: the //! one thread that may stop the others. //! - **Thread 0 is the host.** It keeps the task's executor and its service -//! faces, and serves what the program's threads ask of it through one -//! request cell and one word, the shape `kore-lua`'s VM bridge and `tcc` -//! share: the program's filesystem session ([`kore_fs_run`](crate::api::kore_fs_run)) -//! and its end. +//! faces, and serves the program's filesystem session +//! ([`kore_fs_run`](crate::api::kore_fs_run)) across `kore-rt`'s bridge — +//! the exchange `kore-lua`'s VM and `tcc` speak too: the main thread is the +//! bridge's worker, and its one work item is the program. //! - **Any thread can end the program.** `exit` and `_Exit` reach //! [`__kore_program_exit`] once the C library has run its `atexit` //! handlers; the first request wins. The host stops every other thread, @@ -26,46 +26,45 @@ //! thread 0 (the host) main thread (C) //! ─────────────────── ─────────────── //! found the main thread ────────────────────▶ program(): main(), then exit -//! park on the wake channel … kore_fs_run(step, user) -//! ◀──────── SESSION_POSTED, kick_parent ───── futex::wait on the word +//! serve: park on the wake channel … kore_fs_run(step, user) +//! ◀──────────────────────────── the call ──── parked on its doorbell //! run the session (its C callbacks run here) -//! SESSION_SERVED, kick ─────────────────────▶ returns the session's status +//! the answer ───────────────────────────────▶ returns the session's status //! … exit(status): atexit handlers -//! ◀──────── the exit word, kick_parent ────── the thread ends +//! ◀─────────── the exit word; the bridge's ── the thread ends //! join it, raise the barrier, finalize //! end(status): the C library's exit //! ``` //! -//! Every park is a doorbell park; nothing here spins. The ordering is -//! `kore-lua`'s: a poster stores, then kicks; the host stands its post, -//! re-reads both words, then parks, so a kick that found no post standing -//! announced a store the re-read sees. Every thread of the program can kick -//! the host: the main thread holds the wake channel's send, and each thread it -//! starts is founded holding one too (`thread_controller`). +//! The bridge owns the word, the park-and-kick ordering, the session cell's +//! hand-over, and the main thread's end: a main thread that panics or loses +//! its park is reported gone, never waited for. What stays here is the +//! program's lifecycle, which no other engine shares: the **exit word**, which +//! any thread of the program may post — every thread it starts is founded +//! holding a send into the host's wake channel (`thread_controller`), so its +//! kick reaches the host's park — and the decision, once the program ends, +//! whether the main thread is ending too (then the host joins it, so the +//! barrier is clean and the finalizers run) or still running (then the +//! barrier stops it). //! //! A launch that cannot carry the host (no channel space or heap, a stack //! window that cannot place the main thread) runs the program on thread 0, as //! the C library alone would: allocation, threads, time and random work, a //! filesystem session is refused, and only thread 0's exit ends the program. -use core::cell::{Cell, UnsafeCell}; +use core::cell::Cell; use core::ffi::{c_int, c_void}; -use core::future::poll_fn; -use core::pin::pin; -use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use core::task::Poll; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use kore_ffi::{KoreFsError, KoreFsStep, KoreStatus, Session}; use crate::exit::{Exit, ExitWord}; use kore_std::ExitCode; -use kore_std::abi::session::Status; use kore_std::fs::Fs; -use kore_std::runtime::chan::{NoticeRx, RecvError}; +use kore_std::runtime::bridge::{Bridge, Engine, Gone, Settled, Side, Work, WorkerHost}; use kore_std::runtime::entry::{self, Rt}; use kore_std::runtime::executor::{self, Executor}; -use kore_std::runtime::futex::{self, WaitOutcome}; -use kore_std::runtime::thread::{self, DoorbellWorker, WorkerStack}; +use kore_std::runtime::thread::{self, WorkerStack}; use kore_std::runtime::transport::TrapTransport; use kore_std::runtime::{crash, startup}; @@ -81,9 +80,6 @@ static PROGRAM: AtomicUsize = AtomicUsize::new(0); static END: AtomicUsize = AtomicUsize::new(0); /// Set by the first [`__kore_program_start`]; a second never founds a host. static STARTED: AtomicBool = AtomicBool::new(false); -/// The host serves: the main thread exists and thread 0 parks for it. Set -/// before the main thread is founded, since its first call may need it. -static HOSTED: AtomicBool = AtomicBool::new(false); /// This thread's part in the program. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -105,9 +101,11 @@ pub(crate) fn on_host() -> bool { ROLE.get() == Role::Host } -/// Whether a host serves: thread 0 answers what other threads post. +/// Whether a host serves: thread 0 answers what other threads post. The +/// bridge says so from the founding's claim until the host joins the main +/// thread at the end. pub(crate) fn hosted() -> bool { - HOSTED.load(Ordering::Acquire) + SESSION.is_hosted() } /// Whether the calling thread is the main thread, the one the host can kick. @@ -120,19 +118,17 @@ pub(crate) fn on_main() -> bool { /// The program's end, once asked for (see [`ExitWord`]). static EXIT: ExitWord = ExitWord::new(); -// ── The session cell ──────────────────────────────────────────────────── - -// The word's states. The main thread owns the cell at IDLE and SERVED, the -// host at POSTED; every hand-over is a Release store matched by an Acquire. -const IDLE: u32 = 0; -const POSTED: u32 = 1; -const SERVED: u32 = 2; +/// The end the host reaches when it cannot read one the program asked for. +const INTERNAL: Exit = Exit { + status: ExitCode::INTERNAL.get(), + finalize: false, + from_main: false, +}; -struct Request { - state: AtomicU32, - payload: UnsafeCell, -} +// ── The session ───────────────────────────────────────────────────────── +/// What crosses the bridge: the session the main thread asks for, and how it +/// ended. struct Payload { // main thread → host: the session's callback program. step: Option, @@ -142,34 +138,37 @@ struct Payload { error: KoreFsError, } -// SAFETY: `payload` is touched only by the side the word hands it to (above). -unsafe impl Sync for Request {} - -static SESSION: Request = Request { - state: AtomicU32::new(IDLE), - payload: UnsafeCell::new(Payload { - step: None, - user: core::ptr::null_mut(), - status: KoreStatus::Ok, - error: KoreFsError::None, - }), -}; +// SAFETY: `user` crosses under the bridge's hand-over: `kore_fs_run`'s caller +// keeps whatever it names alive for the session, and the main thread is +// parked inside the call until the host answers it. +unsafe impl Send for Payload {} + +static SESSION: Bridge = Bridge::new(Payload { + step: None, + user: core::ptr::null_mut(), + status: KoreStatus::Ok, + error: KoreFsError::None, +}); -impl Request { - /// # Safety - /// The caller observed (Acquire) a state that hands it the cell, and holds - /// no other reference across the store that hands it back. - #[allow(clippy::mut_from_ref)] - unsafe fn payload(&self) -> &mut Payload { - unsafe { &mut *self.payload.get() } +/// The main thread as the bridge's worker: its one work item is the program, +/// which ends the process rather than finishing. +struct Main; + +impl Engine for Main { + type Payload = Payload; + + fn bridge() -> &'static Bridge { + &SESSION } -} -/// Anything the host must answer: an end, a session, or a mapping. -fn pending() -> bool { - EXIT.posted().is_some() - || SESSION.state.load(Ordering::Acquire) == POSTED - || crate::mmap::posted() + fn run(_program: Work) { + ROLE.set(Role::Main); + // SAFETY: `__kore_program_start` stored a `Program` before founding + // this thread, and the founding's session ordered that store before + // this load. + let program: Program = unsafe { core::mem::transmute(PROGRAM.load(Ordering::Acquire)) }; + unsafe { program() } + } } // ── Entry points the C library calls ──────────────────────────────────── @@ -196,8 +195,10 @@ pub unsafe extern "C" fn __kore_program_start(program: Option, end: Opt ROLE.set(Role::Host); PROGRAM.store(program as usize, Ordering::Release); END.store(end as usize, Ordering::Release); - match Founded::found() { - Some(host) => host.serve(end), + // Lives as long as this frame, which never returns. + let transport = TrapTransport::new(); + match Founded::found(&transport) { + Some(host) => host.serve(&transport, end), None => unsafe { program() }, } } @@ -222,15 +223,19 @@ pub extern "C" fn __kore_program_exit(status: c_int, finalize: c_int) { crate::stdio::flush_now(); startup::barrier(finalize) } - role if HOSTED.load(Ordering::Acquire) => { + role if SESSION.is_hosted() => { let _ = EXIT.post(Exit { status, finalize, from_main: role == Role::Main, }); - // Refused only when no post stood, and then the host re-reads the - // word before it parks: the store above reaches it either way. - let _ = thread::kick_parent(); + // The main thread ends as the bridge's worker, which kicks the + // host and lets it join this thread; any other thread kicks. A + // refused kick found no post standing, and the host re-reads both + // words before it parks: the stores above reach it either way. + if !SESSION.leave() { + let _ = thread::kick_parent(); + } thread::exit(ExitCode::OK) } // No host: only thread 0 owns the process. Anywhere else the barrier @@ -267,53 +272,32 @@ pub extern "C" fn abort() -> ! { } /// The main thread's half of [`kore_fs_run`](crate::api::kore_fs_run): post -/// the session to the host and wait for its answer. +/// the session to the host and wait for its answer. Refused anywhere but the +/// main thread of a hosted program. /// /// # Safety /// `kore_fs_run`'s contract for `step` and `user`. pub(crate) unsafe fn session(step: KoreFsStep, user: *mut c_void) -> (KoreStatus, KoreFsError) { - if ROLE.get() != Role::Main || !HOSTED.load(Ordering::Acquire) { - return (KoreStatus::Refused, KoreFsError::None); - } - // The main thread is the only poster, and it waits out each request: the - // cell is idle whenever control is here. - { - // SAFETY: IDLE hands the cell to the main thread. - let payload = unsafe { SESSION.payload() }; - payload.step = Some(step); - payload.user = user; - } - SESSION.state.store(POSTED, Ordering::Release); - // Refused only when no post stood, and then the host re-reads the word - // before it parks. - let _ = thread::kick_parent(); - loop { - match SESSION.state.load(Ordering::Acquire) { - SERVED => break, - state => { - if futex::wait(&SESSION.state, state, None) == WaitOutcome::Unsupported { - // The main thread is founded with its own doorbell, so it - // can always park; if it cannot, spinning is the one - // thing left, and this host will not. End the program. - fatal() - } - } - } - } - let answer = { - // SAFETY: SERVED hands the cell back to the main thread. - let payload = unsafe { SESSION.payload() }; - payload.step = None; - payload.user = core::ptr::null_mut(); - (payload.status, payload.error) - }; - SESSION.state.store(IDLE, Ordering::Release); - answer + SESSION + .call( + |cell| { + cell.step = Some(step); + cell.user = user; + }, + |cell| { + cell.step = None; + cell.user = core::ptr::null_mut(); + (cell.status, cell.error) + }, + ) + .unwrap_or((KoreStatus::Refused, KoreFsError::None)) } /// This library's panic: the record first, then the end. A panic on thread 0 /// stops the program there; anywhere else it asks the host to, so the process /// ends with the panic rather than running on without the panicking thread. +/// The main thread's panic also reaches the host as the bridge's word, which +/// `crash::report` publishes once the record is out. pub(crate) fn panicked(info: &core::panic::PanicInfo<'_>) -> ! { crash::report(info); match ROLE.get() { @@ -321,7 +305,7 @@ pub(crate) fn panicked(info: &core::panic::PanicInfo<'_>) -> ! { crate::stdio::flush_now(); startup::barrier(false) } - role if HOSTED.load(Ordering::Acquire) => { + role if SESSION.is_hosted() => { let _ = EXIT.post(Exit { status: ExitCode::PANIC.get(), finalize: false, @@ -334,37 +318,21 @@ pub(crate) fn panicked(info: &core::panic::PanicInfo<'_>) -> ! { thread::exit(ExitCode::PANIC) } -/// A host invariant broke on a program thread: end the program with -/// `INTERNAL`, never by spinning. -fn fatal() -> ! { - let _ = EXIT.post(Exit { - status: ExitCode::INTERNAL.get(), - finalize: false, - from_main: ROLE.get() == Role::Main, - }); - let _ = thread::kick_parent(); - thread::exit(ExitCode::INTERNAL) -} - // ── The host ──────────────────────────────────────────────────────────── -/// The host's wake channel broke or a park was refused: it cannot hear the -/// program, so it ends it. -struct Fatal; - /// Thread 0 with its program founded: the runtime context, the executor's /// doorbell, the launcher's stream when the launch carried one, and the main /// thread. -struct Founded { +struct Founded<'t> { rt: Rt, doorbell: executor::Doorbell, output: Option<&'static crate::stdio::Endpoint>, - worker: DoorbellWorker, + main: WorkerHost<'t, Main>, } -impl Founded { +impl<'t> Founded<'t> { /// Found the main thread. `None` when the launch cannot carry the host. - fn found() -> Option { + fn found(transport: &'t TrapTransport) -> Option { let rt = entry::thread_context()?; let doorbell = rt.doorbell()?; // The C streams reach the launcher through this thread: map its @@ -373,242 +341,164 @@ impl Founded { // Where `mmap` places: this task's free link span, published before // any thread that could map exists. crate::mmap::arm(&rt); - HOSTED.store(true, Ordering::Release); // A mint grant of its own: the first `pthread_create` on the main // thread starts kore-rt's thread controller there, which mints the // doorbells its threads park on (`thread_controller`). Without one the // controller still runs, taking the deadline park. let grant = thread::channel_space_endowment(); - match thread::spawn_with_doorbell_on( - main_thread, - grant.as_slice(), + let main = WorkerHost::
::spawn( + transport, WorkerStack::Bytes(crate::api::KORE_MAIN_STACK_BYTES), - ) { - Some(worker) => Some(Self { - rt, - doorbell, - output, - worker, - }), - None => { - HOSTED.store(false, Ordering::Release); - // No host will drain the stream: writes report `EOF`, as - // they do for a launch that carried none. - if output.is_some() { - crate::stdio::abandon(); - } - None + grant.as_slice(), + |_| {}, + ); + let Some(main) = main else { + // No host will drain the stream: writes report `EOF`, as they do + // for a launch that carried none. + if output.is_some() { + crate::stdio::abandon(); } - } + return None; + }; + Some(Self { + rt, + doorbell, + output, + main, + }) } /// Serve the program until it ends, then end the process. - fn serve(self, end: End) -> ! { + fn serve(self, transport: &TrapTransport, end: End) -> ! { let Self { rt, doorbell, output, - worker, + main, } = self; - let transport = TrapTransport::new(); let exit = { let mut executor = Executor::new(entry::transport(), doorbell); - executor.run(async { - let exit = host(&rt, &transport, &worker, output).await; - // Whatever ended the program, what it wrote reaches the - // launcher, and then the stream's close, before the process - // ends: the launcher shows all of it before the status. - if let Some(output) = output { - crate::stdio::finish(output, &|| { - let _ = worker.kick(); - }) - .await; - } - exit - }) + executor.run(host(&rt, transport, main, output)) }; - let exit = exit.unwrap_or(Exit { - status: ExitCode::INTERNAL.get(), - finalize: false, - from_main: false, - }); - if exit.from_main { - // The main thread asked and is ending itself: wait for it, so a - // program that ran on it alone is stopped cleanly. - let _ = worker.join(); - } else { - // Still running: the barrier stops it, and its channels end with - // the process. Its join ticket must not try to reclaim it first. - core::mem::forget(worker); - } startup::barrier(exit.finalize); // SAFETY: the C library's exit, as `__kore_program_start` received it. unsafe { end(exit.status) } } } -/// The main thread's entry: the program, as the C library handed it over. -fn main_thread() { - ROLE.set(Role::Main); - // SAFETY: `__kore_program_start` stored a `Program` before founding this - // thread, and the founding's session ordered that store before this load. - let program: Program = unsafe { core::mem::transmute(PROGRAM.load(Ordering::Acquire)) }; - unsafe { program() } -} +/// What the program's other threads hand thread 0 beside the main thread's +/// sessions: the bridge's side source. +/// - The output ring (`stdio`), which every thread writes. It is drained a +/// pass at a time. A launch that carried no stream has none. +/// - A thread's `mmap` or `munmap` (`mmap`). Thread 0 carries it out, since +/// it holds the supply's root. Its caller always kicks, so only the ring +/// is armed. +struct Beside(Option<&'static crate::stdio::Endpoint>); + +impl Side for Beside { + fn pending(&self) -> bool { + crate::mmap::posted() || (self.0.is_some() && crate::stdio::pending()) + } -/// The host's loop: answer each session the main thread posts, until any -/// thread asks for the end. -async fn host( - rt: &Rt, - transport: &TrapTransport, - worker: &DoorbellWorker, - output: Option<&'static crate::stdio::Endpoint>, -) -> Result { - let mut wake = NoticeRx::new(transport, worker.wake_recv()); - let mut filesystem = Filesystem::new(rt, transport); - // The one writer the host can kick, woken when its room is made. - let wake_main = || { - let _ = worker.kick(); - }; - loop { - if let Some(exit) = EXIT.posted() { - return Ok(exit); + fn arm(&self) { + if self.0.is_some() { + crate::stdio::arm(); } - if let Some(output) = output { - // One bounded pass: the end and the next session are served - // between passes, however fast the program writes. - crate::stdio::drain(output, &wake_main).await; - if let Some(exit) = EXIT.posted() { - return Ok(exit); - } - } - // A thread's `mmap` or `munmap`: thread 0 holds the supply's root. - if crate::mmap::serve() { - wake_main(); - continue; - } - if SESSION.state.load(Ordering::Acquire) == POSTED { - // SAFETY: POSTED hands the cell to the host until it stores SERVED. - let (step, user) = { - let payload = unsafe { SESSION.payload() }; - (payload.step, payload.user) - }; - let Some(step) = step else { - return Err(Fatal); - }; - // An end requested mid-session ends the program without waiting - // for the session: the barrier stops the waiting main thread, and - // the abandoned session's owner is quarantined, as cancellation - // leaves it. - let session = pin!(filesystem.run(step, user)); - let ended = pin!(until_exit(&mut wake, output, &wake_main)); - let outcome = race(session, ended).await; - let Some((status, error)) = outcome else { - continue; - }; - { - // SAFETY: still POSTED; the host owns the cell. - let payload = unsafe { SESSION.payload() }; - payload.status = status; - payload.error = error; - } - SESSION.state.store(SERVED, Ordering::Release); - let _ = worker.kick(); - continue; - } - // Armed before the recheck `park` makes, so a writer that published - // meanwhile either kicks or is seen. - crate::stdio::arm(); - park(&mut wake, || pending() || crate::stdio::pending()).await?; } -} -/// Poll `first` and then `second` until one completes: `Some` with the first's -/// output, or `None` when the second finished first. -async fn race( - mut first: core::pin::Pin<&mut A>, - mut second: core::pin::Pin<&mut B>, -) -> Option { - poll_fn(|cx| { - if let Poll::Ready(output) = first.as_mut().poll(cx) { - return Poll::Ready(Some(output)); + /// A mapping first: its caller waits on the cell, and serving it runs to + /// its answer before the pass's first await, so a pass dropped beside a + /// session cannot cut it short. Then one pass of the ring. `kick_main` reaches the one waiter the host can kick: the + /// main thread, waiting on the mapping cell or for room in the ring. + /// `stdio::drain` is cancellation-safe at every wait, as a pass beside a + /// session must be. + async fn pass(&mut self, kick_main: &dyn Fn()) { + if crate::mmap::serve() { + kick_main(); } - if second.as_mut().poll(cx).is_ready() { - return Poll::Ready(None); - } - Poll::Pending - }) - .await -} - -/// How soon a receive refused for want of a standing post looks again: the -/// tree's figure for a rotating notice receiver (the WASI host's handoff). -const WOULD_BLOCK_RETRY_NS: u64 = 100_000; - -/// Park until a kick, a spurious wake, or `ready` holds, having stood the -/// wake post before re-reading what `ready` reads. -async fn park( - wake: &mut NoticeRx<'_, TrapTransport>, - ready: impl Fn() -> bool, -) -> Result<(), Fatal> { - loop { - match wake.try_recv() { - Ok(Some(_)) => return Ok(()), - Ok(None) => { - if ready() { - return Ok(()); - } - let Some(post) = wake.standing_post() else { - return Err(Fatal); - }; - let _ = executor::next_wake_on(post).await; - } - Err(RecvError::Refused(Status::WouldBlock)) => { - // No post stands to wake on: look again shortly, as the - // tree's other receivers do, rather than wait for a wake - // nothing armed. - let _ = executor::next_wake_for(WOULD_BLOCK_RETRY_NS).await; - } - Err(_) => return Err(Fatal), + if let Some(endpoint) = self.0 { + crate::stdio::drain(endpoint, kick_main).await; } } } -/// Resolve once an end is posted, parking on the wake channel meanwhile and -/// draining what the program writes: the session being served is not a -/// reason to wake, only an end or output is. On a broken channel it never -/// resolves, since the host can no longer hear an end; the session it races -/// then runs to its own completion. Dropped when the session ends first, -/// which the drain survives (`stdio::drain`). Each pass yields, so the -/// session it races is polled however fast the program writes. -async fn until_exit( - wake: &mut NoticeRx<'_, TrapTransport>, +/// The host's loop: answer each session the main thread asks for, and relay +/// what every thread writes, until any thread asks for the end. Then relay +/// the rest and close the stream, and join the main thread if it is ending +/// too, so the barrier is clean, or leave it to the barrier if it is not. +async fn host( + rt: &Rt, + transport: &TrapTransport, + mut main: WorkerHost<'_, Main>, output: Option<&'static crate::stdio::Endpoint>, - wake_main: &dyn Fn(), -) { - let ended = || EXIT.posted().is_some(); - while !ended() { - if let Some(output) = output { - crate::stdio::drain(output, wake_main).await; - executor::yield_now().await; - } - // Another thread's mapping does not wait for the session to end, - // and the session is polled between two of them. - if crate::mmap::serve() { - wake_main(); - executor::yield_now().await; - continue; +) -> Exit { + let mut filesystem = Filesystem::new(rt, transport); + let mut side = Beside(output); + // An end requested mid-session ends the program without waiting for the + // session: the barrier stops the waiting main thread, and the abandoned + // session's owner is quarantined, as cancellation leaves it. What the + // program writes is relayed meanwhile, beside the session, so a program + // that never stops printing still ends when it asks to. + let served = main + .serve_beside( + async |cell: &mut Payload| { + let (status, error) = match cell.step { + Some(step) => filesystem.run(step, cell.user).await, + // A session posted without its program: nothing to run. + None => (KoreStatus::Refused, KoreFsError::None), + }; + cell.status = status; + cell.error = error; + }, + || EXIT.posted().is_some(), + &mut side, + ) + .await; + let (exit, joined) = match served { + // A thread asked for the end. If it was the main thread, that thread + // is ending: the bridge reads it gone once it has. + Ok(None) => match EXIT.posted() { + Some(exit) => (exit, exit.from_main), + None => (INTERNAL, false), + }, + // The main thread is gone: it asked for the end first, or it panicked + // (its record is published), or it lost its park. + Ok(Some(Settled::Gone(gone))) => { + let status = match gone { + Gone::Panicked => ExitCode::PANIC.get(), + _ => ExitCode::INTERNAL.get(), + }; + let exit = EXIT.posted().unwrap_or(Exit { + status, + finalize: false, + from_main: true, + }); + (exit, true) } - crate::stdio::arm(); - if park(wake, || { - ended() || crate::stdio::pending() || crate::mmap::posted() + // The program never finishes its item; and a host that cannot hear + // the main thread cannot tell whether it is ending. + Ok(Some(Settled::Idle)) | Err(_) => (INTERNAL, false), + }; + // Whatever ended the program, what it wrote reaches the launcher, and + // then the stream's close, before the process ends: the launcher shows + // all of it before the status. The main thread may still be waiting for + // room, so it is finished before the main thread is retired. + if let Some(output) = output { + crate::stdio::finish(output, &|| { + let _ = main.kick(); }) - .await - .is_err() - { - core::future::pending::<()>().await; - } + .await; + } + if joined { + // The main thread asked and is ending itself: wait for it, so a + // program that ran on it alone is stopped cleanly. + let _ = main.join().await; + } else { + // Still running: the barrier stops it, and its channels end with the + // process. Its join ticket must not try to reclaim it first. + let _ = main.abandon(); } + exit } /// The program's filesystem: attached on first use, one native owner for the diff --git a/crates/lua/src/bridge.rs b/crates/lua/src/bridge.rs index cd071eb..f6ec119 100644 --- a/crates/lua/src/bridge.rs +++ b/crates/lua/src/bridge.rs @@ -4,50 +4,44 @@ //! C frame that called it, and that frame may sit inside a coroutine, a //! metamethod, or a `pcall`. The store, however, is reached only through //! the task's async filesystem client on its executor. So the VM runs on a -//! kore-rt **worker thread** and the executor stays on thread 0; the two -//! share one request cell and one word: +//! kore-rt **worker thread** and the executor stays on thread 0, across +//! `kore::runtime::bridge`: the `Interpreter` engine, whose work items are +//! evaluations and whose calls are every verb that needs a service. //! //! ```text //! thread 0 (executor) worker (VM) //! ─────────────────── ─────────── -//! write eval → EVAL_POSTED, kick ──────────▶ wait(EVAL_DONE) returns -//! park on the wake channel runs the chunk … -//! … C `require` → write op -//! ◀──────── CALL_POSTED, kick_parent ─────── wait(CALL_POSTED) +//! found the worker: libraries, policy ─────▶ build the VM, finish: ready +//! post the eval ───────────────────────────▶ runs the chunk … +//! … C `require` → call +//! ◀──────────────────────────── the call ─── parked on its doorbell //! serve the op (async) -//! write reply → CALL_SERVED, kick ─────────▶ returns; EVAL_POSTED -//! park on the wake channel +//! the answer ──────────────────────────────▶ returns; the chunk resumes //! … chunk ends -//! ◀──────── EVAL_DONE, kick_parent ──────── wait(EVAL_DONE) +//! ◀───────────────────── finish(outcome) ─── waits for the next item //! take stdout/stderr //! ``` //! +//! The bridge owns the word, the park-and-kick ordering, the cell's +//! hand-over and the worker's end: a VM thread that panics, cannot park, or +//! cannot be built is reported gone, never waited for. This module keeps +//! what crosses it — an evaluation and its output, a call and its answer — +//! and the dispatch that answers a call ([`serve_call`]). +//! //! **One transport, many ops.** `require`, `kore.time.sleep`, and every //! later verb that needs a service — this crate's own or an embedder's, //! through [`Extension`] — are the same round trip with a different -//! payload, so they share one word-state pair -//! (`CALL_POSTED` / `CALL_SERVED`), one worker-side helper ([`host_call`]), -//! and one thread-0 dispatcher ([`serve_call`]). Adding a verb is a new op -//! constant and a new arm — never a new state, and never another copy of -//! the ordering discipline below. Verbs that need *no* service do not come -//! here at all: `kore.time.now` reads the clock page and `kore.random` -//! draws from the CPU, both directly on the VM thread. -//! -//! Every park is a doorbell park; nothing spins. The worker parks with -//! `futex::wait` on the word (its own doorbell); thread 0 parks on the -//! **wake channel** the worker holds a send to (`DoorbellWorker::wake_recv`, -//! `executor::next_wake_on`) — a launched task has no self-kick for a -//! worker to inherit, so `futex::wake` could not reach it. Lost wakeups -//! are excluded by discipline, not luck: the worker stores the word, then -//! kicks; thread 0 stands its post, then re-loads the word, then parks. A -//! kick that found no standing post was refused, and that ordering means -//! the store it announced is visible to the re-load. The -//! word's holder owns the cell: the worker while it reads `EVAL_POSTED` / -//! `CALL_SERVED` / `QUIT`, thread 0 while it reads `EVAL_DONE` / -//! `CALL_POSTED` / `WORKER_GONE`. Buffers cross the boundary as owned -//! `Vec`s — the heap frees a block from any thread onto its owning shard's -//! remote list — or as pointers into a frame the owner keeps alive across -//! the whole phase (the eval's script). +//! payload, so they share one call shape ([`kore_lua_host_call`]) and one +//! thread-0 dispatcher ([`serve_call`]). Adding a verb is a new op +//! constant and a new arm — never a new state, and never a copy of the +//! bridge's ordering. Verbs that need *no* service do not come here at +//! all: `kore.time.now` reads the clock page and `kore.random` draws from +//! the CPU, both directly on the VM thread. Buffers cross the boundary as +//! owned `Vec`s — the heap frees a block from any thread onto its owning +//! shard's remote list — or as a borrow the worker keeps alive across the +//! whole call (a write's bytes). The eval's script is the item's own copy, +//! never a borrow of thread 0's frame: an `eval` future may be dropped +//! mid-evaluation, and the VM thread would go on reading it. //! //! Resolution: a source module `.lua` from `SOURCE_SEARCH` — the //! task's own tree, then `libs/` — wins over a native `.so`, which @@ -61,25 +55,21 @@ //! executable's exported C API. use alloc::vec::Vec; -use core::cell::UnsafeCell; use core::ffi::{CStr, c_char, c_int}; -use core::sync::atomic::{AtomicU32, Ordering}; use kore::abi::clock::TimerError; -use kore::abi::session::Status; use kore::abi::svc::FS_NAME_BYTES; use kore::fs::Fs; use kore::runtime::{ - chan::{NoticeRx, RecvError}, + bridge::{Bridge, Engine, Gone, Next, Refused, Settled, Work, WorkerHost}, entry::Rt, - futex::{self, WaitOutcome}, - thread::{self, DoorbellWorker}, + thread::WorkerStack, transport::TrapTransport, }; use kore_late::{LoadError, Loader, Search}; use crate::vm::{EvalResult, ReplResult, Vm}; -use crate::{Answer, Extension, OP_EMBEDDER_BASE, Policy}; +use crate::{Answer, Extension, Library, OP_EMBEDDER_BASE, Policy}; /// The bound on a source module (`.lua`). Sized like a generous /// script, not like a library: the whole buffer is staged in the task's @@ -95,19 +85,6 @@ const SOURCE_SEARCH: &[&[u8]] = &[b"", kore_late::LIBS_VIEW.as_bytes()]; /// The prefix a native module's opener carries: `luaopen_`. const OPENER_PREFIX: &[u8] = b"luaopen_"; -// The word's states. See the module doc for who owns the cell in each. -const EVAL_DONE: u32 = 0; -const EVAL_POSTED: u32 = 1; -/// The worker posted a host call — a `require`, a sleep, an embedder's -/// library call, any operation that needs thread 0's executor. Thread 0 -/// owns the cell until it stores [`CALL_SERVED`]. One pair serves every -/// op: the VM is a synchronous C engine and every service face lives on -/// thread 0, so the transport is identical and only the payload differs. -const CALL_POSTED: u32 = 2; -const CALL_SERVED: u32 = 3; -const QUIT: u32 = 4; -const WORKER_GONE: u32 = 5; - /// The host-call op vocabulary, as the C side names them (`kore_lua.c`). /// One flat namespace over one transport; a new standard-library verb is a /// new constant and a new `serve_call` arm, never a new word state. An @@ -199,7 +176,8 @@ pub enum Outcome { Exited, /// REPL only: the chunk needs a continuation line. Incomplete, - /// REPL only: the expression buffer could not be allocated. + /// An evaluation's buffer could not be allocated: the REPL's expression + /// buffer, or the copy of the script the evaluation owns. ResourceRefused, /// The worker could not construct a VM. Startup, @@ -210,11 +188,17 @@ pub enum Outcome { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Fatal; -struct Payload { - // thread 0 → worker: the evaluation. +/// What crosses the bridge. +pub struct Payload { + // thread 0 → worker, with the founding: what the VM is built with — + // the embedder's monomorphized libraries and policy. + libraries: &'static [Library], + policy: Policy, + // thread 0 → worker: the evaluation. `script` is the item's own copy, + // in a buffer reused from one evaluation to the next; the worker takes + // it for the evaluation and hands it back with the results. kind: u32, - script: *const u8, - script_len: usize, + script: Vec, chunk_name: *const u8, chunk_name_len: usize, // worker → thread 0: its result. @@ -251,104 +235,93 @@ struct Payload { text: Vec, } -struct Cell { - state: AtomicU32, - payload: UnsafeCell, -} +// SAFETY: the pointers cross under the bridge's hand-over and stay alive +// while the other side reads them: the chunk name is a `'static` literal, a +// call's input lives in the Lua frame of the parked worker until the call is +// answered, and `error` names a `'static` string. +unsafe impl Send for Payload {} -// SAFETY: `payload` is touched only by the thread the word's current state -// hands the cell to (module doc); every hand-over is a Release store on -// `state` matched by the other side's Acquire load. -unsafe impl Sync for Cell {} +static BRIDGE: Bridge = Bridge::new(Payload { + libraries: &[], + policy: Policy::INTERACTIVE, + kind: KIND_CHUNK, + script: Vec::new(), + chunk_name: core::ptr::null(), + chunk_name_len: 0, + outcome: Outcome::Ok, + stdout: Vec::new(), + stderr: Vec::new(), + op: 0, + args: [0; 3], + name: [0; FS_NAME_BYTES], + name_len: 0, + input: core::ptr::null(), + input_len: 0, + reply: REPLY_ERROR, + source: Vec::new(), + opener: None, + error: core::ptr::null(), + status: STATUS_UNAVAILABLE, + results: [0; 2], + text: Vec::new(), +}); -static CELL: Cell = Cell { - state: AtomicU32::new(EVAL_DONE), - payload: UnsafeCell::new(Payload { - kind: KIND_CHUNK, - script: core::ptr::null(), - script_len: 0, - chunk_name: core::ptr::null(), - chunk_name_len: 0, - outcome: Outcome::Ok, - stdout: Vec::new(), - stderr: Vec::new(), - op: 0, - args: [0; 3], - name: [0; FS_NAME_BYTES], - name_len: 0, - input: core::ptr::null(), - input_len: 0, - reply: REPLY_ERROR, - source: Vec::new(), - opener: None, - error: core::ptr::null(), - status: STATUS_UNAVAILABLE, - results: [0; 2], - text: Vec::new(), - }), -}; +// ---------------------------------------------------------------- worker -/// The cell, for the side the word currently hands it to. +/// The VM as a bridge engine: build it with the embedder's libraries and +/// policy, then evaluate each item until thread 0 asks it to quit. /// -/// # Safety -/// The caller has observed (Acquire) a state that makes it the owner, and -/// holds no other reference across the store that hands the cell back. -unsafe fn payload<'a>() -> &'a mut Payload { - unsafe { &mut *CELL.payload.get() } -} +/// Not generic over the [`Extension`]: the extension may borrow what its +/// `main` owns, and an engine outlives no such borrow. What the worker needs +/// of it is `'static` data, which arrives in the first work item. +struct Interpreter; -// ---------------------------------------------------------------- worker +impl Engine for Interpreter { + type Payload = Payload; -/// The worker's entry: build the VM, then serve evaluations until `QUIT`. -fn vm_thread() { - let Some(mut vm) = Vm::new_with_policy(E::LIBRARIES, E::POLICY) else { - // SAFETY: the worker owns the cell while the word reads EVAL_DONE - // and thread 0 has not yet posted (it posts only after spawn). - unsafe { payload().outcome = Outcome::Startup }; - CELL.state.store(WORKER_GONE, Ordering::Release); - let _ = thread::kick_parent(); - return; - }; - loop { - match CELL.state.load(Ordering::Acquire) { - EVAL_POSTED => { - run_eval(&mut vm); - CELL.state.store(EVAL_DONE, Ordering::Release); - let _ = thread::kick_parent(); - } - QUIT => { - drop(vm); - CELL.state.store(WORKER_GONE, Ordering::Release); - let _ = thread::kick_parent(); - return; - } - WORKER_GONE => return, - other => { - if futex::wait(&CELL.state, other, None) == WaitOutcome::Unsupported { - // No doorbell to park on: a spin would be the only - // alternative, and that is the one thing this bridge - // refuses to be. Leave; thread 0 reads WORKER_GONE. - drop(vm); - CELL.state.store(WORKER_GONE, Ordering::Release); - let _ = thread::kick_parent(); - return; - } - } + fn bridge() -> &'static Bridge { + &BRIDGE + } + + fn run(mut first: Work) { + let (libraries, policy) = first.with(|cell| (cell.libraries, cell.policy)); + let Some(mut vm) = Vm::new_with_policy(libraries, policy) else { + first.fail(|cell| cell.outcome = Outcome::Startup); + return; + }; + // The VM is built: the first item, done. + first.finish(|_| ()); + while let Ok(Next::Work(mut work)) = BRIDGE.next() { + let (outcome, script, stdout, stderr) = run_eval(&mut vm, &mut work); + work.finish(|cell| { + // The buffer goes back for the next evaluation to reuse. + cell.script = script; + cell.outcome = outcome; + cell.stdout = stdout; + cell.stderr = stderr; + }); } + // Asked to quit (or unable to wait any longer): the VM is dropped + // before the bridge reads the worker gone, on the body's return. + drop(vm); } } -fn run_eval(vm: &mut Vm) { - // SAFETY: EVAL_POSTED — the worker owns the cell; thread 0 keeps the - // script and chunk name alive until it reads EVAL_DONE. - let cell = unsafe { payload() }; - let script = if cell.script.is_null() { - &[][..] - } else { - unsafe { core::slice::from_raw_parts(cell.script, cell.script_len) } - }; - cell.outcome = match cell.kind { - KIND_REPL => match vm.eval_repl(script) { +/// Evaluate the item: the outcome, the script's buffer (to hand back), and +/// the output. +fn run_eval(vm: &mut Vm, work: &mut Work) -> (Outcome, Vec, Vec, Vec) { + // The script is taken, not borrowed: the worker owns it for the whole + // evaluation, so nothing thread 0 does with the cell can reach it. + let (kind, script, chunk_name, chunk_name_len) = work.with(|cell| { + ( + cell.kind, + core::mem::take(&mut cell.script), + cell.chunk_name, + cell.chunk_name_len, + ) + }); + let outcome = match kind { + KIND_REPL => match vm.eval_repl(&script) { ReplResult::Complete(true) => Outcome::Ok, ReplResult::Complete(false) => Outcome::Failed, ReplResult::CompileFailed => Outcome::CompileFailed, @@ -361,8 +334,8 @@ fn run_eval(vm: &mut Vm) { // name with every KIND_CHUNK evaluation, so this is never the // dangling-null case `from_raw_parts` forbids. let name: &'static [u8] = - unsafe { core::slice::from_raw_parts(cell.chunk_name, cell.chunk_name_len) }; - match vm.eval(script, name) { + unsafe { core::slice::from_raw_parts(chunk_name, chunk_name_len) }; + match vm.eval(&script, name) { EvalResult::Ok => Outcome::Ok, EvalResult::CompileFailed => Outcome::CompileFailed, EvalResult::Failed => Outcome::Failed, @@ -371,57 +344,20 @@ fn run_eval(vm: &mut Vm) { } } }; - cell.stdout = vm.take_stdout(); - cell.stderr = vm.take_stderr(); + (outcome, script, vm.take_stdout(), vm.take_stderr()) } -/// The transport every host call shares: write the request, hand the cell -/// to thread 0, park on the doorbell, read the response back, and resume -/// the evaluation. One function owns the whole ordering so no caller can -/// get it wrong, and a new standard-library verb is a `write`/`read` pair -/// rather than another copy of this discipline. -/// -/// `Err` means the bridge is gone and nothing was served; the cell is not -/// handed back and the evaluation is not resumed. -/// -/// # Safety -/// Worker thread only, inside an evaluation (the word reads `EVAL_POSTED`). -unsafe fn host_call( - write: impl FnOnce(&mut Payload), - read: impl FnOnce(&Payload) -> T, -) -> Result { - if CELL.state.load(Ordering::Acquire) != EVAL_POSTED { - return Err(c"host call outside an evaluation"); - } - // SAFETY: EVAL_POSTED — the worker owns the cell. - write(unsafe { payload() }); - CELL.state.store(CALL_POSTED, Ordering::Release); - let _ = thread::kick_parent(); - loop { - match CELL.state.load(Ordering::Acquire) { - CALL_SERVED => break, - WORKER_GONE | QUIT => return Err(c"host is shutting down"), - other => { - if futex::wait(&CELL.state, other, None) == WaitOutcome::Unsupported { - return Err(c"worker cannot park"); - } - } - } +/// What a refused host call tells the script. +fn refused(refusal: Refused) -> &'static CStr { + match refusal { + Refused::NotWorker | Refused::NoWork => c"host call outside an evaluation", + Refused::Nested => c"host call made inside another", } - // SAFETY: CALL_SERVED — the worker owns the cell again. - let value = read(unsafe { payload() }); - // Back inside the evaluation. No kick: thread 0 does not wait on this - // word — it parks on its wake doorbell (`park_until_kicked`) and reads - // `CALL_SERVED` and `EVAL_POSTED` alike as "the worker is busy". It - // has nothing to do until the worker's next post, and both of those - // (`CALL_POSTED`, `EVAL_DONE`) kick. - CELL.state.store(EVAL_POSTED, Ordering::Release); - Ok(value) } /// C `require`'s half (`kore_lua.c`): post the name, park until served, /// hand back what thread 0 stored. Called on the worker, inside an -/// evaluation, so the word reads `EVAL_POSTED` on entry. +/// evaluation. /// /// # Safety /// `name` points at `len` bytes; the out-pointers are valid. @@ -445,31 +381,29 @@ pub unsafe extern "C" fn kore_lua_host_require( return fail(c"module name is not a store name"); } let requested = unsafe { core::slice::from_raw_parts(name.cast::(), len) }; - let served = unsafe { - host_call( - |cell| { - cell.op = OP_REQUIRE; - cell.name[..len].copy_from_slice(requested); - cell.name_len = len; - cell.reply = REPLY_ERROR; - cell.opener = None; - cell.error = core::ptr::null(); - }, - |cell| match cell.reply { - REPLY_SOURCE => ( - REPLY_SOURCE, - cell.source.as_ptr().cast(), - cell.source.len(), - None, - ), - REPLY_NATIVE => (REPLY_NATIVE, core::ptr::null(), 0, cell.opener), - other => (other, cell.error, 0, None), - }, - ) - }; + let served = BRIDGE.call( + |cell| { + cell.op = OP_REQUIRE; + cell.name[..len].copy_from_slice(requested); + cell.name_len = len; + cell.reply = REPLY_ERROR; + cell.opener = None; + cell.error = core::ptr::null(); + }, + |cell| match cell.reply { + REPLY_SOURCE => ( + REPLY_SOURCE, + cell.source.as_ptr().cast(), + cell.source.len(), + None, + ), + REPLY_NATIVE => (REPLY_NATIVE, core::ptr::null(), 0, cell.opener), + other => (other, cell.error, 0, None), + }, + ); let (reply, reply_bytes, reply_len, reply_opener) = match served { Ok(served) => served, - Err(message) => return fail(message), + Err(refusal) => return fail(refused(refusal)), }; unsafe { *bytes = reply_bytes; @@ -637,11 +571,7 @@ pub fn seed() -> u32 { /// is compiled. Worker-side, inside the evaluation. #[unsafe(no_mangle)] pub extern "C" fn kore_lua_host_release() { - if CELL.state.load(Ordering::Acquire) == EVAL_POSTED { - // SAFETY: EVAL_POSTED — the worker owns the cell. - let cell = unsafe { payload() }; - cell.source = Vec::new(); - } + let _ = BRIDGE.with_work(|cell| cell.source = Vec::new()); } /// Every value-returning host call's half (`kore_lua.c`): post the op and @@ -653,7 +583,8 @@ pub extern "C" fn kore_lua_host_release() { /// copies it into a Lua string before returning). /// /// # Safety -/// The out-pointers are valid for writes. +/// The out-pointers are valid for writes; `input` points at `input_len` +/// bytes (or is null), alive until this returns. #[unsafe(no_mangle)] pub unsafe extern "C" fn kore_lua_host_call( op: u32, @@ -676,30 +607,30 @@ pub unsafe extern "C" fn kore_lua_host_call( } STATUS_UNAVAILABLE }; - let served = unsafe { - host_call( - |cell| { - cell.op = op; - cell.args = [arg0, arg1, arg2]; - cell.input = input; - cell.input_len = if input.is_null() { 0 } else { input_len }; - cell.status = STATUS_UNAVAILABLE; - cell.results = [0; 2]; - cell.text = Vec::new(); - }, - |cell| { - ( - cell.status, - cell.results, - cell.text.as_ptr().cast::(), - cell.text.len(), - ) - }, - ) - }; + let served = BRIDGE.call( + |cell| { + cell.op = op; + cell.args = [arg0, arg1, arg2]; + cell.input = input; + cell.input_len = if input.is_null() { 0 } else { input_len }; + cell.status = STATUS_UNAVAILABLE; + cell.results = [0; 2]; + cell.text = Vec::new(); + }, + |cell| { + cell.input = core::ptr::null(); + cell.input_len = 0; + ( + cell.status, + cell.results, + cell.text.as_ptr().cast::(), + cell.text.len(), + ) + }, + ); let (status, results, reply_text, reply_len) = match served { Ok(served) => served, - Err(message) => return fail(message), + Err(refusal) => return fail(refused(refusal)), }; unsafe { *result = results[0]; @@ -783,51 +714,35 @@ impl<'r, 't> Services<'r, 't> { } } -/// The running VM: the worker, its doorbell, and the wake channel thread 0 -/// parks on. +/// The running VM: its worker, as thread 0 hosts it. pub struct Host<'t> { - worker: DoorbellWorker, - wake: NoticeRx<'t, TrapTransport>, + host: WorkerHost<'t, Interpreter>, } impl<'t> Host<'t> { - /// Spawn the VM's worker. `None` when the task holds no channel space - /// (its profile lacks `channels`) or the thread cannot be founded. + /// Spawn the VM's worker, built with `E`'s libraries and policy. `None` + /// when the task holds no channel space (its profile lacks `channels`) + /// or the thread cannot be founded. pub fn start(transport: &'t TrapTransport) -> Option { - let worker = thread::spawn_with_doorbell(vm_thread::, &[])?; - let wake = NoticeRx::new(transport, worker.wake_recv()); - Some(Self { worker, wake }) - } - - /// Park until the worker kicks (or a spurious executor wake), having - /// first stood the wake post and re-read the word — the module doc's - /// discipline. `seen` is the state thread 0 last observed. - async fn park_until_kicked(&mut self, seen: u32) -> Result<(), Fatal> { - loop { - match self.wake.try_recv() { - // A kick landed; the caller re-reads the word. - Ok(Some(_)) => return Ok(()), - Ok(None) => { - if CELL.state.load(Ordering::Acquire) != seen { - return Ok(()); - } - let Some(post) = self.wake.standing_post() else { - return Err(Fatal); - }; - let _ = kore::runtime::executor::next_wake_on(post).await; - } - Err(RecvError::Refused(Status::WouldBlock)) => { - let _ = kore::runtime::executor::next_wake().await; - } - Err(_) => return Err(Fatal), - } - } + let host = WorkerHost::::spawn(transport, WorkerStack::Band, &[], |cell| { + cell.libraries = E::LIBRARIES; + cell.policy = E::POLICY; + cell.outcome = Outcome::Ok; + })?; + Some(Self { host }) } /// Run `script` on the VM and collect its outcome and output, serving - /// every `require` it makes on the way. `chunk_name` is Lua's chunk + /// every call it makes on the way. `chunk_name` is Lua's chunk /// name (NUL-terminated, static); `repl` selects the REPL's /// expression-then-statement grammar. + /// + /// The evaluation runs on a copy of `script`, so this future may be + /// dropped at any await: the VM thread is not interrupted, and it reads + /// nothing of the caller's. The next `eval` then first finishes the + /// abandoned evaluation, serving its remaining calls and discarding its + /// output, before it posts its own. [`Outcome::ResourceRefused`] when the + /// heap cannot hold the copy; nothing is posted then. pub async fn eval( &mut self, services: &mut Services<'_, '_>, @@ -836,108 +751,94 @@ impl<'t> Host<'t> { chunk_name: &'static [u8], repl: bool, ) -> Result<(Outcome, Vec, Vec), Fatal> { - match CELL.state.load(Ordering::Acquire) { - EVAL_DONE => {} - WORKER_GONE => { - // SAFETY: WORKER_GONE — thread 0 owns the cell. - let outcome = unsafe { payload().outcome }; - return Ok((outcome, Vec::new(), Vec::new())); - } - _ => return Err(Fatal), + // The previous item has settled first — the first time, the VM's + // construction, which makes no calls. + let settled = self + .host + .serve(async |cell: &mut Payload| serve_call(services, extension, cell).await) + .await + .map_err(|_| Fatal)?; + if let Settled::Gone(gone) = settled { + return self.gone(gone); } + // The evaluation owns its script: it is copied into the cell, never + // lent from this frame, because this future may be dropped + // mid-evaluation (a `select`, a deadline) while the VM thread goes on + // reading it. One bounded copy into the buffer the previous + // evaluation handed back; a script the heap cannot hold a copy of is + // refused before anything is posted. { - // SAFETY: EVAL_DONE — thread 0 owns the cell. - let cell = unsafe { payload() }; - cell.kind = if repl { KIND_REPL } else { KIND_CHUNK }; - cell.script = script.as_ptr(); - cell.script_len = script.len(); - cell.chunk_name = chunk_name.as_ptr(); - cell.chunk_name_len = chunk_name.len(); + let cell = self.host.payload().ok_or(Fatal)?; + cell.script.clear(); + if cell.script.try_reserve(script.len()).is_err() { + return Ok((Outcome::ResourceRefused, Vec::new(), Vec::new())); + } + cell.script.extend_from_slice(script); } - CELL.state.store(EVAL_POSTED, Ordering::Release); - // A refused kick is not a failure: it means no post stood, and the - // worker re-tests the word after it arms one (`futex::wait`'s - // contract), so the store above reaches it regardless. - let _ = self.worker.kick(); - loop { - match CELL.state.load(Ordering::Acquire) { - state @ (EVAL_POSTED | CALL_SERVED) => { - self.park_until_kicked(state).await?; - } - CALL_POSTED => { - serve_call(services, extension).await; - CELL.state.store(CALL_SERVED, Ordering::Release); - let _ = self.worker.kick(); - } - EVAL_DONE => { - // SAFETY: EVAL_DONE — thread 0 owns the cell; the - // script pointer is dead from here (its frame is ours). - let cell = unsafe { payload() }; - cell.script = core::ptr::null(); - cell.script_len = 0; - return Ok(( - cell.outcome, - core::mem::take(&mut cell.stdout), - core::mem::take(&mut cell.stderr), - )); - } - _ => return Err(Fatal), + self.host + .post(|cell| { + cell.kind = if repl { KIND_REPL } else { KIND_CHUNK }; + cell.chunk_name = chunk_name.as_ptr(); + cell.chunk_name_len = chunk_name.len(); + }) + .map_err(|_| Fatal)?; + let settled = self + .host + .serve(async |cell: &mut Payload| serve_call(services, extension, cell).await) + .await + .map_err(|_| Fatal)?; + match settled { + Settled::Idle => { + let cell = self.host.payload().ok_or(Fatal)?; + Ok(( + cell.outcome, + core::mem::take(&mut cell.stdout), + core::mem::take(&mut cell.stderr), + )) } + Settled::Gone(gone) => self.gone(gone), + } + } + + /// What an evaluation answers once the worker is gone: a VM that could + /// not be built says so in its outcome ([`Outcome::Startup`]); any other + /// loss is the bridge's. + fn gone(&mut self, gone: Gone) -> Result<(Outcome, Vec, Vec), Fatal> { + match (gone, self.host.payload()) { + (Gone::Failed, Some(cell)) => Ok((cell.outcome, Vec::new(), Vec::new())), + _ => Err(Fatal), } } /// Stop the worker and join it, so the task's last thread is thread 0. - /// Joins only a worker that acknowledged `QUIT` (or is already gone): + /// Joins only a worker that acknowledged the quit (or is already gone): /// a worker mid-evaluation is parked on the bridge and would never /// return, so it is left to task teardown rather than waited on. - pub async fn quit(mut self) { - match CELL.state.load(Ordering::Acquire) { - EVAL_DONE => { - CELL.state.store(QUIT, Ordering::Release); - let _ = self.worker.kick(); - loop { - match CELL.state.load(Ordering::Acquire) { - WORKER_GONE => break, - state => { - if self.park_until_kicked(state).await.is_err() { - return; - } - } - } - } - } - WORKER_GONE => {} - _ => return, - } - let _ = self.worker.join(); + pub async fn quit(self) { + let _ = self.host.quit().await; } } -/// Answer the host call the cell holds. CALL_POSTED — thread 0 owns the -/// cell for the whole call. One dispatcher: a new standard-library verb -/// is an arm here and nothing about the transport changes. -async fn serve_call(services: &mut Services<'_, '_>, extension: &mut E) { - // SAFETY: CALL_POSTED. - let op = unsafe { payload() }.op; - if op == OP_REQUIRE { - serve_require(services, E::POLICY).await; +/// Answer the host call the cell holds; thread 0 holds the cell for the +/// whole call. One dispatcher: a new standard-library verb is an arm here +/// and nothing about the transport changes. +async fn serve_call( + services: &mut Services<'_, '_>, + extension: &mut E, + cell: &mut Payload, +) { + if cell.op == OP_REQUIRE { + serve_require(services, cell, E::POLICY).await; } else { - serve_value(services, extension, op, E::POLICY).await; + serve_value(services, extension, cell, E::POLICY).await; } } -/// Answer the `require` the cell holds: read the source, or link the -/// native module and look its opener up. -async fn serve_require(services: &mut Services<'_, '_>, policy: Policy) { - let (name, len) = { - // SAFETY: CALL_POSTED. - let cell = unsafe { payload() }; - (cell.name, cell.name_len) - }; - let reply = resolve(services, &name[..len.min(FS_NAME_BYTES)], policy).await; - // SAFETY: CALL_POSTED. - let cell = unsafe { payload() }; - match reply { +/// Answer a `require`: read the source, or link the native module and +/// look its opener up. +async fn serve_require(services: &mut Services<'_, '_>, cell: &mut Payload, policy: Policy) { + let len = cell.name_len.min(FS_NAME_BYTES); + match resolve(services, &cell.name[..len], policy).await { Reply::Source(source) => { cell.reply = REPLY_SOURCE; cell.source = source; @@ -964,37 +865,35 @@ async fn serve_require(services: &mut Services<'_, '_>, policy: Policy) { async fn serve_value( services: &mut Services<'_, '_>, extension: &mut E, - op: u32, + cell: &mut Payload, policy: Policy, ) { + let op = cell.op; if policy.is_restricted() && !matches!( op, OP_FS_READ_FILE | OP_FS_STAT | OP_FS_LIST | OP_FS_ENTRIES | OP_FS_RIGHTS ) { - return finish_answer(Answer::Unavailable(alloc::string::String::from( - "host operation unavailable to read-only worker", - ))); + return finish_answer( + cell, + Answer::Unavailable(alloc::string::String::from( + "host operation unavailable to read-only worker", + )), + ); } - let args = { - // SAFETY: CALL_POSTED. - unsafe { payload() }.args - }; + let args = cell.args; // The call's byte argument, borrowed from the Lua frame the parked // worker is holding open. Read here, once, so no arm below repeats // the reasoning about why this pointer is live. // - // SAFETY: CALL_POSTED — thread 0 owns the cell, and the worker set - // this window from a Lua value on its own stack before parking. It - // stays parked, and so the value stays rooted, until CALL_SERVED. - let input: &[u8] = unsafe { - let cell = payload(); - if cell.input.is_null() || cell.input_len == 0 { - &[] - } else { - core::slice::from_raw_parts(cell.input, cell.input_len) - } + // SAFETY: the worker set this window from a Lua value on its own stack + // before it called, and it stays parked inside the call, so the value + // stays rooted, until the call is answered. + let input: &[u8] = if cell.input.is_null() || cell.input_len == 0 { + &[] + } else { + unsafe { core::slice::from_raw_parts(cell.input, cell.input_len) } }; // The contiguous block, plus the one verb numbered past `os.exit`. if (OP_FS_OPEN..=OP_FS_LIST).contains(&op) @@ -1004,7 +903,7 @@ async fn serve_value( ) { let answer = serve_fs(services, op, args, input, policy).await; - return finish_answer(answer); + return finish_answer(cell, answer); } let answer = match op { // The executor's own timer — the one verb `kore.time` cannot serve @@ -1017,7 +916,7 @@ async fn serve_value( // with none, so an empty answer is the honest one, not a refusal. OP_PROC_ARGS => { let launch = services.args.clone(); - return finish(STATUS_OK, [0; 2], launch); + return finish(cell, STATUS_OK, [0; 2], launch); } // Recorded, not acted on: the raise that follows this call is what // unwinds the evaluation, running every `` on the way out. @@ -1030,17 +929,17 @@ async fn serve_value( op if op >= OP_EMBEDDER_BASE => extension.serve(op, args).await, _ => Answer::Unavailable(alloc::string::String::from("unknown host call")), }; - finish_answer(answer) + finish_answer(cell, answer) } /// Write one [`Answer`] back into the cell. Every op ends here, so the /// three shapes a script sees are decided in exactly one place. -fn finish_answer(answer: Answer) { +fn finish_answer(cell: &mut Payload, answer: Answer) { match answer { - Answer::Value(result, result2) => finish(STATUS_OK, [result, result2], Vec::new()), - Answer::Bytes(count, bytes) => finish(STATUS_OK, [count, bytes.len() as u64], bytes), - Answer::Refused(text) => finish(STATUS_REFUSED, [0; 2], text.into_bytes()), - Answer::Unavailable(text) => finish(STATUS_UNAVAILABLE, [0; 2], text.into_bytes()), + Answer::Value(result, result2) => finish(cell, STATUS_OK, [result, result2], Vec::new()), + Answer::Bytes(count, bytes) => finish(cell, STATUS_OK, [count, bytes.len() as u64], bytes), + Answer::Refused(text) => finish(cell, STATUS_REFUSED, [0; 2], text.into_bytes()), + Answer::Unavailable(text) => finish(cell, STATUS_UNAVAILABLE, [0; 2], text.into_bytes()), } } @@ -1104,9 +1003,7 @@ async fn serve_fs( /// Write one value-op outcome back into the cell. Every op answers the /// same three things, so every arm ends here. -fn finish(status: c_int, results: [u64; 2], text: Vec) { - // SAFETY: CALL_POSTED. - let cell = unsafe { payload() }; +fn finish(cell: &mut Payload, status: c_int, results: [u64; 2], text: Vec) { cell.status = status; cell.results = results; cell.text = text; diff --git a/crates/lua/src/lib.rs b/crates/lua/src/lib.rs index 54387ba..d84a303 100644 --- a/crates/lua/src/lib.rs +++ b/crates/lua/src/lib.rs @@ -280,8 +280,9 @@ impl Policy { /// interfaces. /// /// Selection is by type instantiation, not runtime dispatch: `Host::start` -/// is generic over `E`, so `LIBRARIES` reaches the worker thread as -/// monomorphized static data with no global to publish it. +/// is generic over `E`, so `LIBRARIES` and `POLICY` are monomorphized static +/// data the host hands the worker thread in its first work item, with no +/// global to publish them. /// /// There is deliberately **no `'static` bound**. The worker thread reads /// only [`Extension::LIBRARIES`], which is `&'static` in its own right; diff --git a/docs/languages.md b/docs/languages.md index 42d764b..601c567 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -91,11 +91,14 @@ that joins `libkore.so` to the program's own `libc.a` — and gets: - `kore_fs_run(step, user, &error)` (`kore/program.h`), which runs a `KoreFsStep` program as the program's filesystem session. -The host is the shape `kore-lua`'s VM bridge has. `main` runs on a main -thread, a `DoorbellWorker` with a 1 MiB guarded stack -(`KORE_MAIN_STACK_BYTES`). Thread 0 is the process owner, and keeps the -executor and the launch's service faces. The two share one request cell and -one word. Every park is a doorbell park; nothing spins. +The host is the shape `kore-lua`'s VM and Kore's `tcc` have, and speaks +the same seam, kore-rt's `bridge`. `main` runs on a main thread, the +bridge's worker, with a 1 MiB guarded stack (`KORE_MAIN_STACK_BYTES`). +Thread 0 is the process owner, and keeps the executor and the launch's +service faces. The two share one request cell and one word, whose states, +ordering and hand-over are the bridge's. Every park is a doorbell park; +nothing spins, and a main thread that panics or cannot park is reported to +the host rather than waited for. - **Filesystem.** `kore_fs_run` from the main thread posts the session and waits. The host attaches the launch's filesystem face on first use, rooted @@ -120,10 +123,11 @@ one word. Every park is a doorbell park; nothing spins. stream's capacity when it has none — a slow terminal holds the program back rather than losing its output — and wakes the writers waiting for space, a pass at a time, serving the program's end and its sessions - between passes. The host's own writes (a filesystem step's callbacks) are - kept in order beside the ring, before any call another thread has in - progress. A launch without the stream has no sink, and a - write to either stream returns `EOF`. + between passes and draining beside a session in flight: the ring is the + bridge's side source (`WorkerHost::serve_beside`). The host's own writes + (a filesystem step's callbacks) are kept in order beside the ring, before + any call another thread has in progress. A launch without the stream has + no sink, and a write to either stream returns `EOF`. - **The end.** The C library's `exit` and `_Exit` call `__kore_program_exit(status, finalize)` after their `atexit` handlers, on whatever thread called them. The first request wins and reaches the host, @@ -203,7 +207,9 @@ mapped there. The rules follow from that: holder may revoke, so the caller posts its request to thread 0, which holds the supply's root, and waits as a filesystem session does. A mapping is therefore the process's: any thread may `munmap` it, whole, by its base - and its length. + and its length. Thread 0 answers a request between the main thread's + sessions and beside one in flight, as it drains the ring: both are the + bridge's side source. - **`munmap` parks the region.** The supply's cursors and the window's only advance, so carved bytes and placed addresses never return to them. The mapping is removed and a stale access faults, but the program keeps the diff --git a/docs/provenance.md b/docs/provenance.md index 737162c..fe98e14 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -351,3 +351,57 @@ C program (`docs/languages.md`). It is the first libkore package a C program on Kore links without a Rust host. Its shared object is a distribution's one-line `cdylib` over it. `tools/check-program.py` links that object for the three Kore ISAs and checks its shape and its exact exported ABI. + +## One bridge for a synchronous engine on a worker thread + +The export records Kore `6a8c754bc959e9489a6fb18be642fcc881a65c48`, a commit +on Kore's `ft/festive-euler-sghq2d`, which carries the bridge and merges +MicroPerceptron/kore#1494's branch at `f2644e5b` (`eca08c04`). Since the last +export recorded here (`0018956a`), that branch's exports moved through four +Kore commits, and this export carries all four: +- `2a0730dd`, where threads that found take a founder's arena; +- `6c57f344`, where a founding's staged space waits for the reap; +- `2d4998ba`, which brings Kore's C library, `kore-c-shim`; +- `10c7b400`, where `kore-rt` names a task's link window and its free span + once (`738c3591`), and `kore-c-shim` gains `ENXIO`, `EFAULT` and `ENOTSUP`. + +Against the export at `10c7b400`, the closure keeps its 27 packages and gains +one file (402): + +- `kore-rt`'s `bridge`: one request cell and one word between a synchronous + engine on a worker thread and its async host. The word's states, the + park-and-kick ordering, the cell's hand-over and the worker's end live in one + place (`Bridge`, `Engine`, `Work`, `WorkerHost`), with a host suite that + drives the state machine on real threads. The host can also serve a side + source beside the worker (`Side`, `serve_beside`): work other threads hand it + through the same wake channel, armed and re-read with the word. +- `ThreadBlock`'s `hosted_bridge` tail slot, which names the bridge a worker + serves. With it come a bridge-worker notice in `entry::panic_refuse` and + `crash::report`, once the panic record is published, and the worker's exit + hook: a worker that ends its thread through `thread::exit` without saying + its last word has it said for it. +- `futex`'s sweep test judges only the rows it claims (`6a8c754b`). The sweep + reads every row, and the suite's other tests hold rows of their own + meanwhile, so it failed whenever one overlapped it: 8 runs in 300 of + `futex::tests`. +- The services' `boot.toml` digests are the base's seal at `f2644e5b`. Kore's + half reseals over its pin of this branch, so that seal follows this export + and cannot be in it. + +`kore-lua`'s VM bridge and `kore-c`'s program host now speak the seam, as Kore's +`tcc` and its Lua REPL do upstream. Each keeps only its payload and its +dispatch. `kore-c` founds its main thread through `WorkerHost::spawn`, holding +the channel-space grant `b9eb3c2` gave it. What the program's other threads +hand thread 0 is the bridge's side source: the output ring `0460f5c` brought, +and the `mmap` cell `c876e43` brought. No exported manifest changed, so +`native/Cargo.lock` did not. `make miri` also runs the `bridge` suite, so Miri +checks every hand-over of the cell. + +Locally, with the pinned nightly, `make fmt test check docs owners cross miri +bindings-check` passed on this export. That covers: +- the native `kore-rt` suite (477, the `bridge` suite's 13 among them) and the + `kore-dagfs` and `kore-vol` suites; +- the three Kore user-target checks, which take in `kore-lua`'s bridge; +- `check-program`, which links `libkore.so` for all three ISAs with exactly its + declared 129 exports, the five `c876e43` brought among them; +- Miri over the `bridge` suite (14 tests with `erased_poll_and_drop`). diff --git a/native/crates/rt/src/bridge.rs b/native/crates/rt/src/bridge.rs new file mode 100644 index 0000000..927f452 --- /dev/null +++ b/native/crates/rt/src/bridge.rs @@ -0,0 +1,1935 @@ +//! The **bridge** — a synchronous engine on a worker thread, served by an +//! async host. +//! +//! Some engines cannot await. Lua's `require` must hand a module back to the +//! C frame that called it, from inside a coroutine, a metamethod or a +//! `pcall`; TinyCC opens a file it discovered mid-parse; an independent C +//! program's `main` calls the filesystem whenever it likes. What those calls +//! need is reached only through the task's async clients, on its executor. So +//! the engine runs on a kore-rt **worker thread**, the executor stays on the +//! thread that founded it (the **host**: thread 0 in every engine today), and +//! the two share one [`Bridge`] — one request cell and one word: +//! +//! ```text +//! host (executor) worker (engine) +//! ─────────────── ─────────────── +//! spawn: first item, WORK; found ────────────▶ Engine::run(work) … +//! park on the wake channel … call(): request, CALL +//! ◀───────────────── CALL, kick_parent ──────── futex::wait on the word +//! serve the call (async) +//! the answer, WORK; kick ─────────────────────▶ read the answer; resume +//! park on the wake channel … finish(): results, IDLE +//! ◀───────────────── IDLE, kick_parent ──────── next(): futex::wait +//! post(): next item, WORK; kick ──────────────▶ … +//! quit(): QUIT; kick ─────────────────────────▶ run returns +//! ◀───────────── GONE·Quit, kick_parent ─────── the thread ends; join +//! ``` +//! +//! This module owns everything about that exchange except what crosses it. +//! An engine supplies its payload — the work, its results, and each call's +//! request and answer, as one flat structure — and the dispatch that answers a +//! call. The states, the ordering, the hand-over of the cell, and what happens +//! when the worker is gone are here, once. Lua's REPL and tool worker, TinyCC +//! and `libkore.so`'s C program host are the same exchange with different +//! payloads; before this module each of them carried its own copy of the word +//! and its discipline, and the copies had drifted apart in exactly the places +//! the discipline is subtle. +//! +//! # The word +//! +//! | Word | Holds the cell | Entered by | +//! |-------------|----------------|------------------------------------------------| +//! | `VACANT` | nobody | [`Bridge::new`]; a `WorkerHost::join` | +//! | `IDLE` | host | the host's claim; the worker's [`Work::finish`] | +//! | `WORK` | worker | founding, `WorkerHost::post`, an answer | +//! | `CALL` | host | the worker's [`call`](Bridge::call) | +//! | `QUIT` | worker | the host's `WorkerHost::quit` | +//! | `GONE`·why | host, for good | the worker, as its last act ([`Gone`]) | +//! +//! **The word names the cell's holder**, and only the holder touches the +//! payload. Every hand-over is a `Release` on the word after the holder's last +//! write, matched by the other side's `Acquire` before its first read, so +//! whatever the holder wrote is visible to the next one. The payload is reached +//! only through closures this module runs after it has read a state that hands +//! the cell to the caller, so no borrow outlives the state that justified it. +//! +//! **Founding is the hand-off.** `WorkerHost::spawn` writes the first work +//! item while it holds the claimed bridge, stores `WORK`, and only then founds +//! the worker: the worker is born holding its work, with nothing to wait for. +//! +//! **Two transitions are made by the side that does not hold the cell**, and +//! both touch only the word: a worker that is gone moves a host-held `IDLE` or +//! `CALL` to `GONE`. Each races one transition of the host's — a `post` out of +//! `IDLE`, an answer out of `CALL` — and both sides compare-and-swap, so +//! exactly one wins and neither overwrites the other. Every other transition +//! is made by the holder alone and could be a plain store; the races are the +//! whole reason the host's own transitions are CASes. +//! +//! # Parks, kicks, and why no wakeup is lost +//! +//! Every park is a doorbell park; nothing here spins beyond `futex::wait`'s +//! own bounded monitor. +//! +//! - **The worker** parks with [`futex::wait`] on the word, on its own +//! doorbell. The host answers by storing the word and then kicking that +//! doorbell (`DoorbellWorker::kick`). `futex::wait` stands its post +//! *before* it re-reads the word, so a kick that found no post standing was +//! refused before the worker looked — and the store it announced, sequenced +//! before the kick, is what the worker then reads. +//! - **The host** parks on the **wake channel** the worker holds a send into +//! (`DoorbellWorker::wake_recv`): a launched task has no self-kick for a +//! worker to inherit, so [`futex::wake`] could not reach thread 0. The +//! worker stores the word and then kicks the channel +//! (`thread::kick_parent`); the host stands its post, re-reads the word, +//! and only then parks with +//! [`next_wake_on`](crate::executor::next_wake_on). A kick that found no +//! post was refused before the post stood, so the re-read sees its store; a +//! kick into a standing post resolves the park at once. +//! +//! - **A side source** (`Side`, metal only) is work other +//! threads hand the host through the same wake channel: a C program's +//! output, which any of its threads may write. Its producers kick only +//! while the host is armed, and the host arms after its post stands and +//! then re-reads the source beside the word, so the same argument holds: +//! a kick that found no post was refused before the arm, and the re-read +//! sees what the producer published first. The host drains the source a +//! bounded pass at a time, between the worker's calls and beside one in +//! flight, so neither waits for the other. +//! +//! A refused kick is therefore never an error on either side, and a wake is +//! never a promise: both sides re-read the word after every return. +//! +//! # What a call may borrow +//! +//! While the word reads `CALL`, the worker is parked inside +//! [`Bridge::call`], and `call` never returns while it does: an answer moves +//! the word to `WORK` first, and a worker that gives up a call **exits its +//! thread instead of returning**. An exited thread's stack stays mapped until +//! it is joined, and the host joins only once the word reads `GONE` — after it +//! has finished with the call, since the give-up is what it then reads. So a +//! request may borrow from the frame that made it (a path, the bytes of a +//! write) and the host may read through those borrows for the whole call. +//! +//! # A worker that is gone +//! +//! `GONE` is the worker's last word on the bridge, and it always says it, +//! whichever way it leaves ([`Gone`]): its body returns; it answers a quit; it +//! fails a work item; its probe at entry finds it cannot park, or a park is +//! refused mid-call or between items; it ends its thread itself; or it panics. +//! The last two need no help from the engine. The worker's runtime block names +//! its bridge from its first instruction, and the worker's entry registers the +//! thread's exit hook (`thread::on_exit`), which `thread::exit` runs before +//! it commits: a worker that ends its thread without having said its last +//! word has it said for it. Every kore-rt panic path — `entry::panic_refuse`, +//! `crash::panic`, `crash::report` — publishes `GONE`·[`Panicked`](Gone::Panicked) +//! and kicks the host *after* the panic record is published, so a host that +//! ends the process on it cannot cut the report short; the exit hook names a +//! panic too, for an exit taken while one is in flight. That slot is also how a +//! worker-side call proves its caller is the worker: holdings are per thread, +//! and so is the right to touch a worker's cell. +//! +//! # What an engine keeps +//! +//! Its payload — `Send`, because it crosses threads; an engine whose payload +//! carries raw pointers says why they may cross in its own `unsafe impl Send` — +//! one `static` [`Bridge`] over it, an [`Engine`] naming the two and the +//! worker's body, and the async dispatch it hands `WorkerHost::serve`. A new +//! verb is a field and an arm in the engine; never a new state, and never +//! another copy of the ordering. +//! +//! The host suite drives this state machine with real threads (and under +//! Miri, which checks every hand-over for a data race) through a park that +//! only ever returns spuriously — what `futex::wait` may always do. It proves +//! the states, the races and the hand-over; it does not, and cannot, prove the +//! kernel's half of the park discipline above, which the boot gates and the +//! engines' keytests exercise where it runs. + +use core::cell::UnsafeCell; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +use crate::barrier::Line; +use crate::futex::{self, WaitOutcome}; + +// ─── The word ────────────────────────────────────────────────────────── + +// The phase is the word's low byte; `GONE` carries its reason above it. +const VACANT: u32 = 0; +const IDLE: u32 = 1; +const WORK: u32 = 2; +const CALL: u32 = 3; +const QUIT: u32 = 4; +const GONE: u32 = 5; +const PHASE: u32 = 0xff; +const REASON_SHIFT: u32 = 8; + +/// A value the word never holds — the park probe's expectation, so the probe +/// can never actually park (`futex::wait` answers `Retry` on a thread that can +/// park, `Unsupported` on one that cannot). +#[cfg(any(test, target_os = "none"))] +const NEVER: u32 = u32::MAX; + +/// Why a worker is gone: the reason its last word on the bridge carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Gone { + /// The worker's body returned, or the worker ended itself: through + /// [`Bridge::leave`], or by ending its thread (`thread::exit`, whose exit + /// hook says it). Results it finished before are in the payload. + Ended, + /// The worker answered the host's `WorkerHost::quit`. + Quit, + /// The worker could not do its work item and takes no other + /// ([`Work::fail`]); the payload says why. + Failed, + /// The worker cannot park on its doorbell — found by the probe at entry, + /// or by a park the kernel refused later — so it could wait for an answer + /// only by spinning, which this bridge refuses to do. + Unparked, + /// The worker panicked; its panic record was published first. + Panicked, +} + +impl Gone { + const fn word(self) -> u32 { + let reason = match self { + Self::Ended => 1, + Self::Quit => 2, + Self::Failed => 3, + Self::Unparked => 4, + Self::Panicked => 5, + }; + GONE | reason << REASON_SHIFT + } + + const fn from_reason(reason: u32) -> Self { + match reason { + 1 => Self::Ended, + 2 => Self::Quit, + 3 => Self::Failed, + 4 => Self::Unparked, + // 5, and any reason this module never writes: a word that cannot + // be read is a worker that cannot be trusted, which the host must + // treat as gone — failing closed, never as running. + _ => Self::Panicked, + } + } +} + +/// The word, read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum State { + Vacant, + Idle, + Work, + Call, + Quit, + Gone(Gone), +} + +const fn decode(word: u32) -> State { + match word & PHASE { + VACANT => State::Vacant, + IDLE => State::Idle, + WORK => State::Work, + CALL => State::Call, + QUIT => State::Quit, + // `GONE`, and any phase this module never writes: gone, as above. + _ => State::Gone(Gone::from_reason(word >> REASON_SHIFT)), + } +} + +/// Publish `reason` as a worker's last word, from wherever the word stands, +/// without touching the payload. `true` when this call published it; `false` +/// when the worker was already gone (or no host holds the bridge). +/// +/// From `WORK` and `QUIT` the worker holds the cell and could store; from a +/// host-held `IDLE` or `CALL` it must not, because the host's `post` and +/// answer move the word too. One CAS loop serves all four, so exactly one of a +/// racing pair wins and the loser sees the winner's word. A worker that ends +/// itself (`Ended`) while asked to quit has answered the quit. +fn settle(word: &AtomicU32, reason: Gone) -> bool { + let mut seen = word.load(Ordering::Acquire); + loop { + let gone = match decode(seen) { + State::Quit if reason == Gone::Ended => Gone::Quit.word(), + State::Idle | State::Work | State::Call | State::Quit => reason.word(), + State::Gone(_) | State::Vacant => return false, + }; + match word.compare_exchange_weak(seen, gone, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return true, + Err(now) => seen = now, + } + } +} + +// ─── The worker's ends ───────────────────────────────────────────────── + +/// What the worker side needs from the machine: who it is, how it parks, how +/// it tells its host, and how it ends without returning. +/// +/// [`Native`] is the only implementation outside this module's tests; the +/// split exists so the state machine above it is ordinary code the host suite +/// can drive, the shape `futex` and `barrier` take with their own metal halves. +trait Ends { + /// Name this thread the worker over `word`, before anything else runs on + /// it. `false` when the thread cannot hold the name (no runtime block). + #[cfg(any(test, target_os = "none"))] + fn adopt(&self, word: &AtomicU32) -> bool; + /// This thread is the worker over `word`. + fn is_worker(&self, word: &AtomicU32) -> bool; + /// This thread is no longer a worker (its last word is published). + fn release(&self, word: &AtomicU32); + /// Park while `word` holds `seen`; `futex::wait`'s contract, spurious + /// returns included. + fn park(&self, word: &AtomicU32, seen: u32) -> WaitOutcome; + /// Tell the host the word moved. A refused kick is harmless (module doc). + fn kick_host(&self); + /// End this thread without returning; its stack stays mapped until it is + /// joined. + fn exit(&self) -> !; +} + +/// The worker's real ends: its runtime block, its doorbell, its wake send. +struct Native; + +impl Ends for Native { + #[cfg(any(test, target_os = "none"))] + fn adopt(&self, word: &AtomicU32) -> bool { + imp::adopt(word_address(word)) + } + + fn is_worker(&self, word: &AtomicU32) -> bool { + imp::hosted() == word_address(word) + } + + fn release(&self, word: &AtomicU32) { + imp::release(word_address(word)); + } + + fn park(&self, word: &AtomicU32, seen: u32) -> WaitOutcome { + futex::wait(word, seen, None) + } + + fn kick_host(&self) { + let _ = imp::kick_host(); + } + + fn exit(&self) -> ! { + crate::thread::exit(crate::exit::ExitCode::INTERNAL) + } +} + +fn word_address(word: &AtomicU32) -> usize { + word as *const AtomicU32 as usize +} + +// ─── The bridge ──────────────────────────────────────────────────────── + +/// The request cell a worker and its host share: one word and one payload. +/// +/// A `static`, named by its [`Engine`] — the worker's calls come from deep +/// inside an engine (a C callback, a stdio sink) that has no context to pass +/// down, and the cell must outlive a worker a host abandons. Its methods are +/// the **worker's** side; `WorkerHost` is the host's. +pub struct Bridge

{ + /// The word, alone on its destructive-interference span: the worker + /// monitors this line in `futex::wait`'s brief spin, and a store to the + /// payload beside it would end that monitor for nothing. + word: Line, + /// A worker-side payload borrow is live — one of this bridge's closures is + /// running on the worker. Only the worker touches it. + borrowed: AtomicBool, + payload: UnsafeCell

, +} + +// SAFETY: the payload is touched only by the side the word hands it to, and +// every hand-over is a `Release` on the word matched by the other side's +// `Acquire` (module doc). It crosses threads, hence `P: Send`. +unsafe impl Sync for Bridge

{} + +/// Why a worker-side operation was refused. Nothing was posted or changed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refused { + /// The caller is not this bridge's worker thread. + NotWorker, + /// The worker holds no work item: it is between items, asked to quit, or + /// gone. + NoWork, + /// The payload is already borrowed on this thread: the call came from + /// inside another of this bridge's closures, which would alias it. + Nested, +} + +impl

Bridge

{ + /// A vacant bridge over `payload`, ready to be a `static`. + pub const fn new(payload: P) -> Self { + Self { + word: Line::new(VACANT), + borrowed: AtomicBool::new(false), + payload: UnsafeCell::new(payload), + } + } + + /// A host holds this bridge: from `WorkerHost::spawn`'s claim until its + /// `WorkerHost::join`, including while the worker is gone. + pub fn is_hosted(&self) -> bool { + decode(self.word.0.load(Ordering::Acquire)) != State::Vacant + } + + fn word(&self) -> &AtomicU32 { + &self.word.0 + } + + fn state(&self) -> State { + decode(self.word().load(Ordering::Acquire)) + } + + /// Run `f` over the payload for the worker, refusing a nested borrow. + /// + /// Caller: the worker, having read (`Acquire`) a state that hands it the + /// cell, with no store since that hands it back. + fn worker_borrow(&self, f: impl FnOnce(&mut P) -> R) -> Result { + if self.borrowed.swap(true, Ordering::Relaxed) { + return Err(Refused::Nested); + } + // SAFETY: the caller holds the cell (contract above), and `borrowed` + // proves no other borrow of it is live on this, the only thread that + // may hold it now. + let value = f(unsafe { &mut *self.payload.get() }); + self.borrowed.store(false, Ordering::Relaxed); + Ok(value) + } + + /// The payload for the host. + /// + /// # Safety + /// The host has read (`Acquire`) a state that hands it the cell — `IDLE`, + /// `CALL` or `GONE`, or the claim it just made — and holds no other + /// reference into the payload. + #[cfg(any(test, target_os = "none"))] + #[allow(clippy::mut_from_ref)] + unsafe fn host_payload(&self) -> &mut P { + // SAFETY: caller contract. The worker's only moves out of a host-held + // state touch the word alone (`settle`), never the payload. + unsafe { &mut *self.payload.get() } + } + + // ── The worker's side ───────────────────────────────────────────── + + /// **Post a call and park until the host answers it.** `request` writes + /// the call into the payload; `answer` reads the host's answer back out. + /// + /// The worker must be inside a work item. The request may borrow from the + /// caller's frame: this returns only after the host has answered, and a + /// worker that can no longer park for the answer gives the call up and + /// ends its thread rather than return into a frame the host may still be + /// reading (module doc) — the host then reads `GONE`·[`Unparked`](Gone::Unparked). + pub fn call( + &self, + request: impl FnOnce(&mut P), + answer: impl FnOnce(&mut P) -> R, + ) -> Result { + self.call_with(&Native, request, answer) + } + + fn call_with( + &self, + ends: &impl Ends, + request: impl FnOnce(&mut P), + answer: impl FnOnce(&mut P) -> R, + ) -> Result { + if !ends.is_worker(self.word()) { + return Err(Refused::NotWorker); + } + if self.state() != State::Work { + return Err(Refused::NoWork); + } + self.worker_borrow(request)?; + // The hand-over, then the kick that brings the host to read it. + self.word().store(CALL, Ordering::Release); + ends.kick_host(); + loop { + match self.state() { + State::Work => break, + State::Call => { + if ends.park(self.word(), CALL) == WaitOutcome::Unsupported { + // The kernel refused a park it accepted at the probe: + // not transient, and the answer could now be waited + // for only by spinning. Give the call up — out of + // `CALL` alone, never out of an answer the host + // already made: that CAS fails, and the loop reads + // the answer's `WORK`. + if self + .word() + .compare_exchange( + CALL, + Gone::Unparked.word(), + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + ends.release(self.word()); + ends.kick_host(); + ends.exit(); + } + } + } + // Nothing else moves the word while this worker is parked in + // its call. If something did, the cell's holder is unknown: + // never return into the request's frame with it unknown. + _ => { + ends.release(self.word()); + ends.kick_host(); + ends.exit(); + } + } + } + self.worker_borrow(answer) + } + + /// The payload, for the worker inside a work item — a stdio sink, a + /// release the engine makes between calls. + pub fn with_work(&self, f: impl FnOnce(&mut P) -> R) -> Result { + self.with_work_as(&Native, f) + } + + fn with_work_as(&self, ends: &impl Ends, f: impl FnOnce(&mut P) -> R) -> Result { + if !ends.is_worker(self.word()) { + return Err(Refused::NotWorker); + } + if self.state() != State::Work { + return Err(Refused::NoWork); + } + self.worker_borrow(f) + } + + /// **Wait for the host's next word**: a work item, a quit, or — when this + /// worker can no longer park — the end. + /// + /// The worker must have finished its last item ([`Work::finish`]). An item + /// still unfinished is still the current one, and is handed back as + /// [`Next::Work`] again. + pub fn next(&'static self) -> Result, Refused> { + self.next_with(&Native) + } + + fn next_with(&'static self, ends: &impl Ends) -> Result, Refused> { + if !ends.is_worker(self.word()) { + return Err(Refused::NotWorker); + } + loop { + match self.state() { + State::Work => return Ok(Next::Work(Work::new(self))), + State::Quit => return Ok(Next::Quit), + State::Gone(gone) => return Ok(Next::Gone(gone)), + State::Idle => { + if ends.park(self.word(), IDLE) == WaitOutcome::Unsupported + && self.leave_with(ends, Gone::Unparked) + { + return Ok(Next::Gone(Gone::Unparked)); + } + } + // No call is outstanding between items, and a vacant bridge + // has no host to wait for. + State::Call | State::Vacant => return Err(Refused::NoWork), + } + } + } + + /// **End this worker**: publish `GONE`·[`Ended`](Gone::Ended) — or + /// [`Quit`](Gone::Quit), when the host had asked — and kick the host. + /// For a worker that ends its thread itself instead of returning from + /// [`Engine::run`], at a point of its choosing: a C program's main thread + /// leaves once its exit status is posted, so the host reads both. A + /// `thread::exit` that comes first says the same through the exit hook, + /// and the trampoline says it for a body that returns. `false` when the + /// caller is not the worker or the worker is already gone. + pub fn leave(&self) -> bool { + self.leave_with(&Native, Gone::Ended) + } + + fn leave_with(&self, ends: &impl Ends, reason: Gone) -> bool { + if !ends.is_worker(self.word()) { + return false; + } + let published = settle(self.word(), reason); + if published { + ends.release(self.word()); + ends.kick_host(); + } + published + } +} + +/// The host's side: the decisions `WorkerHost` acts on (and the tests drive). +#[cfg(any(test, target_os = "none"))] +impl

Bridge

{ + /// Claim a vacant bridge for a new worker: `VACANT` → `IDLE`. One host per + /// bridge at a time. + fn claim(&self) -> bool { + self.word() + .compare_exchange(VACANT, IDLE, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + + /// The first item is written: hand it over with the founding. + fn hand_over_first(&self) { + self.word().store(WORK, Ordering::Release); + } + + /// Back to vacant: no worker reads the cell (it was never founded, or it + /// is joined). + fn vacate(&self) { + self.word().store(VACANT, Ordering::Release); + } + + /// What the host must do next. + fn host_step(&self) -> HostStep { + let word = self.word().load(Ordering::Acquire); + match decode(word) { + State::Idle => HostStep::Settled(Settled::Idle), + State::Gone(gone) => HostStep::Settled(Settled::Gone(gone)), + State::Call => HostStep::Serve, + // The worker holds the cell; park until the word moves from here. + State::Work | State::Quit => HostStep::Wait(word), + // Unreachable while a host exists; never a state to wait in. + State::Vacant => HostStep::Settled(Settled::Gone(Gone::Ended)), + } + } + + /// The call is answered: `CALL` → `WORK`. `false` when the worker gave the + /// call up first; the answer then has no reader and nothing is kicked. + fn answer(&self) -> bool { + self.word() + .compare_exchange(CALL, WORK, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + + /// Post the next item: `work` writes it while the host holds the idle + /// cell, then `IDLE` → `WORK`. + fn post_with(&self, work: impl FnOnce(&mut P)) -> Result<(), NotIdle> { + match self.state() { + State::Idle => {} + State::Gone(gone) => return Err(NotIdle::Gone(gone)), + _ => return Err(NotIdle::Busy), + } + // SAFETY: `IDLE` hands the host the cell; a worker leaving it moves + // the word alone. + work(unsafe { self.host_payload() }); + match self + .word() + .compare_exchange(IDLE, WORK, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => Ok(()), + // The worker left while the item was written; it never saw it. + Err(now) => Err(match decode(now) { + State::Gone(gone) => NotIdle::Gone(gone), + _ => NotIdle::Busy, + }), + } + } + + /// Ask an idle worker to quit: `IDLE` → `QUIT`. + fn ask_quit(&self) -> QuitAsk { + match self + .word() + .compare_exchange(IDLE, QUIT, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => QuitAsk::Asked, + Err(now) => match decode(now) { + // Leaving already, or asked before. + State::Gone(_) | State::Quit => QuitAsk::Leaving, + // Mid-item: a worker inside an engine cannot be stopped from + // here, and would never answer. + _ => QuitAsk::Busy, + }, + } + } +} + +/// The host's next move, as the word dictates it. +#[cfg(any(test, target_os = "none"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum HostStep { + /// A call waits: serve it. + Serve, + /// The worker holds the cell; park until the word leaves this value. + Wait(u32), + /// The worker has yielded the cell for now (or for good). + Settled(Settled), +} + +#[cfg(any(test, target_os = "none"))] +enum QuitAsk { + Asked, + Leaving, + Busy, +} + +/// What the worker handed back to the host, once a `WorkerHost::serve` +/// has nothing more to answer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Settled { + /// The worker finished its item and waits for the next; its results are + /// in the payload (`WorkerHost::payload`). + Idle, + /// The worker is gone, and says why. Results it finished first are in the + /// payload. + Gone(Gone), +} + +/// Why a `WorkerHost::post` was refused; the item was not handed over. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NotIdle { + /// The worker holds the cell: an item, a call or a quit is under way. + Busy, + /// The worker is gone. + Gone(Gone), +} + +/// The host's wake channel is broken: it can no longer hear the worker, so it +/// cannot serve it. Nothing here is retried; the worker is left to the task's +/// teardown. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Fatal; + +/// How a `WorkerHost` let its worker go. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Retired { + /// The worker is gone and its thread joined; the bridge is vacant again. + Joined, + /// The worker was left as it was — mid-item, or out of the host's hearing + /// — for the task's teardown to reclaim. The bridge stays held. + Abandoned, +} + +// ─── The engine ──────────────────────────────────────────────────────── + +/// A synchronous engine hosted on a worker thread: its payload, its bridge, +/// and the worker's body. +/// +/// A unit type per engine (generic, when the engine is: Lua's extension). The +/// worker's entry is monomorphized over it, so the bridge and the body are +/// bound at compile time and no entry pointer is stored anywhere. +pub trait Engine: 'static { + /// What crosses the bridge: the work, its results, and each call's + /// request and answer. + type Payload: Send + 'static; + + /// The one bridge this engine's worker and host share. + fn bridge() -> &'static Bridge; + + /// The worker's body, entered on the worker thread holding the first work + /// item the host prepared at `WorkerHost::spawn`. Return to end + /// the worker: the bridge then reads `GONE`·[`Ended`](Gone::Ended), or + /// [`Quit`](Gone::Quit) after [`Next::Quit`]. + fn run(work: Work); +} + +/// **A work item, held by the worker** — the proof that the cell is the +/// worker's until it [`finish`](Self::finish)es or [`fail`](Self::fail)s. +/// +/// Stays on the worker thread: holdings are per thread, and so is the cell. +#[must_use = "a work item ends with finish or fail; a body that returns holding one ends the worker"] +pub struct Work { + bridge: &'static Bridge

, + _thread: PhantomData<*const ()>, +} + +impl

Work

{ + fn new(bridge: &'static Bridge

) -> Self { + Self { + bridge, + _thread: PhantomData, + } + } + + /// The payload: the item the host wrote, and room for its results. + /// + /// Copy out what the engine needs and return: a closure that runs the + /// engine would make the engine's own [`Bridge::call`]s nested borrows, + /// which they refuse. + /// + /// # Panics + /// When called from inside one of the bridge's own closures — a nested + /// borrow of the payload, refused like `RefCell`'s — or after the worker + /// ended itself ([`Bridge::leave`]) still holding this item: the cell is + /// the host's then, and so is [`finish`](Self::finish)'s word. + pub fn with(&mut self, f: impl FnOnce(&mut P) -> R) -> R { + assert!( + self.bridge.state() == State::Work, + "bridge: a work item used after its worker left" + ); + match self.bridge.worker_borrow(f) { + Ok(value) => value, + Err(_) => { + panic!("bridge: a work item's payload borrowed inside another of its borrows") + } + } + } + + /// Write the item's results and hand the cell back: `WORK` → `IDLE`, and + /// kick the host. The worker then waits for its next word with + /// [`Bridge::next`], or returns from its body. + pub fn finish(mut self, results: impl FnOnce(&mut P)) { + self.with(results); + self.bridge.word().store(IDLE, Ordering::Release); + Native.kick_host(); + } + + /// The worker cannot do this item and will take no other: `why` writes + /// what the host should know, then `WORK` → `GONE`·[`Failed`](Gone::Failed). + /// The body should then return. + pub fn fail(mut self, why: impl FnOnce(&mut P)) { + self.with(why); + self.bridge + .word() + .store(Gone::Failed.word(), Ordering::Release); + Native.release(self.bridge.word()); + Native.kick_host(); + } +} + +/// The host's next word to a worker, as [`Bridge::next`] reads it. +pub enum Next { + /// A work item. + Work(Work

), + /// The host asks the worker to end: return from [`Engine::run`] (after + /// releasing whatever the engine holds). + Quit, + /// The worker is gone — it could no longer park — and should return. + Gone(Gone), +} + +/// The worker's entry, over `E`: adopt the bridge, probe the park, run the +/// body, settle the word. +#[cfg(any(test, target_os = "none"))] +fn enter(ends: &impl Ends) { + let bridge = E::bridge(); + let word = bridge.word(); + // First, so a panic from here on reaches the host. + if !ends.adopt(word) { + // A thread with no runtime block cannot park either. + if settle(word, Gone::Unparked) { + ends.kick_host(); + } + return; + } + // Founding hands the worker `WORK`; anything else is not this worker's. + if bridge.state() != State::Work { + ends.release(word); + return; + } + // Can this thread park? `futex::wait` decides before it compares, so a + // wait for a value the word never holds answers `Retry` on a thread that + // can and `Unsupported` on one that cannot — and never parks. + if ends.park(word, NEVER) == WaitOutcome::Unsupported { + let _ = bridge.leave_with(ends, Gone::Unparked); + return; + } + E::run(Work::new(bridge)); + let _ = bridge.leave_with(ends, Gone::Ended); +} + +/// A panicking thread's last word on its bridge, if it is a worker — called by +/// kore-rt's panic paths once the panic record is published (module doc). +pub(crate) fn panicked() { + last_word(Gone::Panicked); +} + +/// Say `reason` as this thread's last word on its bridge, if it is a worker +/// still on one: for the panic paths and the exit hook, which end a worker +/// that never said its own. A worker that did is released, and this is +/// nothing. +fn last_word(reason: Gone) { + let address = imp::hosted(); + if address == 0 { + return; + } + // SAFETY: a worker's runtime block names its bridge's word from the + // worker's first instruction until its last word, and an `Engine`'s + // bridge is `'static`. + let word = unsafe { &*(address as *const AtomicU32) }; + if settle(word, reason) { + imp::release(address); + let _ = imp::kick_host(); + } +} + +// ─── The host ────────────────────────────────────────────────────────── + +#[cfg(target_os = "none")] +pub use host::{Side, WorkerHost}; + +#[cfg(target_os = "none")] +mod host { + use core::future::poll_fn; + use core::marker::PhantomData; + use core::pin::{Pin, pin}; + use core::task::Poll; + + use kore_abi::session::Status; + + use super::{ + Engine, Fatal, HostStep, NotIdle, QuitAsk, Retired, Settled, State, decode, enter, + }; + use crate::chan::{NoticeRx, RecvError}; + use crate::executor; + use crate::thread::{self, DoorbellWorker, WorkerCap, WorkerStack}; + use crate::transport::TrapTransport; + + /// The worker's entry, as the founding calls it. + fn trampoline() { + enter::(&super::Native); + } + + /// How soon a park whose receive was refused for want of a standing post + /// looks again. A rotating receiver's handoff is in flight, and nothing + /// is armed to wake the host when it lands. This is the figure the + /// tree's other notice receivers retry a handoff at (`chan`, `process`). + const HANDOFF_RETRY_NS: u64 = 100_000; + + /// A **side source** the host serves beside the worker's calls, kicked + /// through the same wake channel: work other threads hand thread 0, such + /// as a C program's output, which any of its threads may write. + /// + /// The host runs the source's passes between the worker's calls and + /// beside one in flight, so neither waits for the other. A pass never + /// ends a serve: only the serve's stop does. Producers kick the host only + /// while it is armed, so a run of work costs no trap per item. + pub trait Side { + /// Whether the source has work for the host now. + fn pending(&self) -> bool; + /// Have the source's producers kick the wake channel from now on. + /// The host arms after its wake post stands and before it re-reads + /// [`pending`](Self::pending). Work published after the arm kicks a + /// standing post, and work published before it is seen (module doc). + fn arm(&self); + /// One bounded pass over the source's work. `kick_worker` wakes the + /// worker, for work it waits on beside the bridge (room it needs). + /// A pass beside a call is dropped mid-way when the call completes + /// first, so it must be cancellation-safe. + fn pass(&mut self, kick_worker: &dyn Fn()) -> impl Future; + } + + /// No side source: [`WorkerHost::serve`] and + /// [`serve_until`](WorkerHost::serve_until). + impl Side for () { + fn pending(&self) -> bool { + false + } + + fn arm(&self) {} + + async fn pass(&mut self, _: &dyn Fn()) {} + } + + /// **The host's half of a bridge**: the worker it founded over + /// `E`'s bridge, and the wake channel it parks on. + /// + /// Stays on the thread that spawned it: the worker's doorbell send and + /// the wake channel's recv are holdings of that thread. + #[must_use = "a hosted worker is quit or joined, or deliberately abandoned"] + pub struct WorkerHost<'t, E: Engine> { + worker: DoorbellWorker, + wake: NoticeRx<'t, TrapTransport>, + _engine: PhantomData<(fn() -> E, *const ())>, + } + + impl<'t, E: Engine> WorkerHost<'t, E> { + /// **Found a worker over `E`'s bridge**, holding its first work item. + /// + /// `first` writes that item while the host holds the claimed bridge; + /// then the word reads `WORK` and the worker is founded on `stack` — + /// founding is the hand-off, so the item is the worker's from its + /// first instruction. `extra` endows it as + /// [`spawn_with_doorbell_on`](thread::spawn_with_doorbell_on) does. + /// + /// `None` when another host holds the bridge, or the founding refuses + /// (no `CHANNEL_SPACE`, no heap, a stack the window cannot place); the + /// bridge is vacant again then, since nothing reads it. + pub fn spawn( + transport: &'t TrapTransport, + stack: WorkerStack, + extra: &[WorkerCap], + first: impl FnOnce(&mut E::Payload), + ) -> Option { + let bridge = E::bridge(); + if !bridge.claim() { + return None; + } + // SAFETY: the claim made the word `IDLE`, and no worker exists to + // move it. + first(unsafe { bridge.host_payload() }); + bridge.hand_over_first(); + let Some(worker) = thread::spawn_with_doorbell_on(trampoline::, extra, stack) else { + bridge.vacate(); + return None; + }; + let wake = NoticeRx::new(transport, worker.wake_recv()); + Some(Self { + worker, + wake, + _engine: PhantomData, + }) + } + + /// The payload, while the host holds the cell between items or after + /// the worker is gone — where a settled item's results are. + pub fn payload(&mut self) -> Option<&mut E::Payload> { + let bridge = E::bridge(); + match bridge.state() { + // SAFETY: `IDLE` and `GONE` hand the host the cell, and the + // `&mut self` borrow keeps every other host operation out + // until this one ends. + State::Idle | State::Gone(_) => Some(unsafe { bridge.host_payload() }), + _ => None, + } + } + + /// **Hand the worker its next item**: `work` writes it into the idle + /// cell, the word moves to `WORK`, and the worker is kicked. + pub fn post(&mut self, work: impl FnOnce(&mut E::Payload)) -> Result<(), NotIdle> { + E::bridge().post_with(work)?; + let _ = self.worker.kick(); + Ok(()) + } + + /// **Kick the worker's doorbell**, for an engine whose worker also + /// waits on words of its own beside the bridge (a C program's main + /// thread, waiting for room for its output). A kick is a hint: the + /// worker re-reads whatever it waits on, so a spurious one costs a + /// re-read. `false` when the kernel refused it. + pub fn kick(&self) -> bool { + self.worker.kick() + } + + /// **Serve the worker's calls until it yields the cell**: `answer` + /// dispatches each call, holding the payload for the whole call. + /// Returns at once if the worker is already idle or gone. + pub async fn serve( + &mut self, + answer: impl AsyncFnMut(&mut E::Payload), + ) -> Result { + match self.drive(answer, None:: bool>, &mut ()).await? { + Some(settled) => Ok(settled), + // `drive` stops only for a stop condition, and there is none. + None => Err(Fatal), + } + } + + /// [`serve`](Self::serve) that also stops — `Ok(None)` — once `stop` + /// holds: before a park, after every wake, and while a call is being + /// served. A call cut short is left unanswered (its future dropped, as + /// any cancellation leaves it) and its worker parked; the host may + /// then retire the worker without waiting for it. + /// + /// For a host whose other threads can end it — a C program's + /// `exit` from any thread, which posts its own word and kicks the + /// same wake channel. + pub async fn serve_until( + &mut self, + answer: impl AsyncFnMut(&mut E::Payload), + stop: impl Fn() -> bool, + ) -> Result, Fatal> { + self.drive(answer, Some(stop), &mut ()).await + } + + /// [`serve_until`](Self::serve_until) with a [`Side`] source served + /// beside the worker: a pass whenever the source has work, between + /// the worker's calls and beside a call in flight. The stop and the + /// worker's next call are read between passes, so a source whose + /// producers never pause still lets the serve stop. + /// + /// For a C program's host, whose every thread writes the output the + /// host relays. + pub async fn serve_beside( + &mut self, + answer: impl AsyncFnMut(&mut E::Payload), + stop: impl Fn() -> bool, + side: &mut impl Side, + ) -> Result, Fatal> { + self.drive(answer, Some(stop), side).await + } + + async fn drive bool>( + &mut self, + mut answer: impl AsyncFnMut(&mut E::Payload), + stop: Option, + side: &mut impl Side, + ) -> Result, Fatal> { + let bridge = E::bridge(); + let Self { worker, wake, .. } = self; + let worker = &*worker; + let stopped = || stop.as_ref().is_some_and(|stop| stop()); + let kick_worker = || { + let _ = worker.kick(); + }; + loop { + if stopped() { + return Ok(None); + } + // One pass, then the worker: the stop and its next call are + // read between passes, however fast the side's producers are. + if side.pending() { + side.pass(&kick_worker).await; + if stopped() { + return Ok(None); + } + } + match bridge.host_step() { + HostStep::Settled(settled) => return Ok(Some(settled)), + HostStep::Wait(seen) => { + let moved = + || bridge.word().load(core::sync::atomic::Ordering::Acquire) != seen; + park(wake, || moved() || stopped() || armed(side)).await?; + } + HostStep::Serve => { + // SAFETY: `CALL` hands the host the cell until it + // answers, and the worker is parked inside `call`, + // which does not return while the word reads `CALL` + // (module doc): whatever the request borrows from its + // frame is alive for the whole call. + let payload = unsafe { bridge.host_payload() }; + if stop.is_none() { + answer(payload).await; + } else { + let call = pin!(answer(payload)); + let ended = pin!(until(wake, &stopped, side, &kick_worker)); + if race(call, ended).await.is_none() { + return Ok(None); + } + } + // A worker that gave the call up already said so. + if bridge.answer() { + let _ = worker.kick(); + } + } + } + } + } + + /// **Ask the worker to end, and join it.** An idle worker is asked + /// (`QUIT`) and the host parks until it is gone; a worker already + /// leaving is waited for. A worker mid-item would never answer, so it + /// is [`abandon`](Self::abandon)ed instead. + pub async fn quit(self) -> Retired { + match E::bridge().ask_quit() { + QuitAsk::Asked => { + let _ = self.worker.kick(); + } + QuitAsk::Leaving => {} + QuitAsk::Busy => return self.abandon(), + } + self.join().await + } + + /// **Wait until the worker is gone, then join its thread** and vacate + /// the bridge. For a worker the host knows is ending — it asked, or the + /// worker said so in the engine's own words; a worker that is not + /// ending is waited for as long as it runs. + pub async fn join(mut self) -> Retired { + let bridge = E::bridge(); + loop { + if let State::Gone(_) = bridge.state() { + break; + } + let seen = bridge.word().load(core::sync::atomic::Ordering::Acquire); + if decode(seen) == State::Call { + // A call no one will serve: the worker cannot reach `GONE`. + return self.abandon(); + } + let moved = || bridge.word().load(core::sync::atomic::Ordering::Acquire) != seen; + if park(&mut self.wake, moved).await.is_err() { + return self.abandon(); + } + } + // `GONE` is the worker's last act on the bridge; joining waits for + // the thread itself to end. + let Self { worker, wake, .. } = self; + drop(wake); + match worker.join() { + Ok(_) => { + bridge.vacate(); + Retired::Joined + } + // The ticket is spent with the error: its band and stack stay + // the task's until teardown, and so does the bridge. + Err(_) => Retired::Abandoned, + } + } + + /// **Leave the worker as it is** for the task's teardown: no join, no + /// retire, no syscall. The bridge stays held, since the worker may + /// still run. + pub fn abandon(self) -> Retired { + core::mem::forget(self); + Retired::Abandoned + } + } + + /// Park on the wake channel until a kick lands or `ready` holds — standing + /// the post first and re-reading after (module doc). + async fn park( + wake: &mut NoticeRx<'_, TrapTransport>, + ready: impl Fn() -> bool, + ) -> Result<(), Fatal> { + loop { + match wake.try_recv() { + // A kick landed; the caller re-reads the word. + Ok(Some(_)) => return Ok(()), + Ok(None) => { + if ready() { + return Ok(()); + } + let Some(post) = wake.standing_post() else { + return Err(Fatal); + }; + let _ = executor::next_wake_on(post).await; + } + Err(RecvError::Refused(Status::WouldBlock)) => { + // No post stands to wake on while a rotating receiver's + // handoff is in flight, and nothing is armed to say when + // it lands: look again shortly rather than wait for a wake + // nothing will make. + let _ = executor::next_wake_for(HANDOFF_RETRY_NS).await; + } + Err(_) => return Err(Fatal), + } + } + } + + /// The side source's part of a park's re-read: arm it, then read it. + fn armed(side: &impl Side) -> bool { + side.arm(); + side.pending() + } + + /// Resolve once `stopped` holds, parking on the wake channel meanwhile + /// and running the side's passes: the call being served is not a reason + /// to wake, only a stop or side work is. On a broken channel it never + /// resolves, since the host can no longer hear a stop, and the call it + /// races runs to its own end. Each pass yields, so the call is polled + /// however fast the side's producers are; a pass in flight is dropped + /// when the call completes first ([`Side::pass`]). + async fn until( + wake: &mut NoticeRx<'_, TrapTransport>, + stopped: &impl Fn() -> bool, + side: &mut impl Side, + kick_worker: &dyn Fn(), + ) { + while !stopped() { + if side.pending() { + side.pass(kick_worker).await; + executor::yield_now().await; + continue; + } + if park(wake, || stopped() || armed(side)).await.is_err() { + core::future::pending::<()>().await; + } + } + } + + /// Poll `first`, then `second`, until one completes: `Some` with the + /// first's output, `None` when the second finished first. + async fn race( + mut first: Pin<&mut A>, + mut second: Pin<&mut B>, + ) -> Option { + poll_fn(|cx| { + if let Poll::Ready(output) = first.as_mut().poll(cx) { + return Poll::Ready(Some(output)); + } + if second.as_mut().poll(cx).is_ready() { + return Poll::Ready(None); + } + Poll::Pending + }) + .await + } +} + +// ─── The metal half ──────────────────────────────────────────────────── + +/// The worker's runtime block names its bridge (by its word's address), its +/// exit hook says its last word, and its wake send reaches the host. +#[cfg(target_os = "none")] +mod imp { + use super::Gone; + + pub(super) fn adopt(word: usize) -> bool { + crate::entry::geometry::set_hosted_bridge(word); + // However the thread ends from here, `thread::exit` runs this first. + // The hook is the entry's to set, as the thread controller sets its + // children's: this entry is the worker's, so no one else sets it. + crate::thread::on_exit(exiting, 0); + true + } + + /// The worker's exit hook: a worker still on its bridge as its thread + /// ends has not said its last word, so its end says it. It runs after + /// the thread's own code, and neither blocks nor ends the thread. + fn exiting(_: usize) { + let reason = if crate::entry::geometry::panicking() { + Gone::Panicked + } else { + Gone::Ended + }; + super::last_word(reason); + } + + pub(super) fn hosted() -> usize { + crate::entry::geometry::hosted_bridge() + } + + pub(super) fn release(word: usize) { + if hosted() == word { + crate::entry::geometry::set_hosted_bridge(0); + } + } + + pub(super) fn kick_host() -> bool { + crate::thread::kick_parent() + } +} + +/// Off metal no thread is a worker: no runtime block names a bridge, and there +/// is no wake send to kick — the honest refusal, never a simulated host. +#[cfg(not(target_os = "none"))] +mod imp { + #[cfg(test)] + pub(super) fn adopt(_word: usize) -> bool { + false + } + + pub(super) fn hosted() -> usize { + 0 + } + + pub(super) fn release(_word: usize) {} + + pub(super) fn kick_host() -> bool { + false + } +} + +#[cfg(test)] +mod tests { + //! The state machine on real threads. The worker's park only ever + //! returns spuriously (it yields), which `futex::wait` may always do; the + //! host polls instead of parking. What this proves is the states, the + //! races and the hand-over — every payload access below is checked for a + //! data race under Miri — not the kernel's park discipline (module doc). + + use super::*; + use core::sync::atomic::AtomicUsize; + use std::boxed::Box; + use std::thread; + use std::vec::Vec; + + /// A worker's ends on the host. `parks` counts the parks this worker may + /// take before the kernel "refuses" one (`usize::MAX`: never). + struct TestEnds { + worker: bool, + parks: AtomicUsize, + } + + impl TestEnds { + const fn worker() -> Self { + Self { + worker: true, + parks: AtomicUsize::new(usize::MAX), + } + } + + const fn outsider() -> Self { + Self { + worker: false, + parks: AtomicUsize::new(usize::MAX), + } + } + + const fn parking_only(parks: usize) -> Self { + Self { + worker: true, + parks: AtomicUsize::new(parks), + } + } + } + + /// The panic a test worker's `exit` raises, so a test can tell an exit + /// from a return. + struct Exited; + + impl Ends for TestEnds { + fn adopt(&self, _: &AtomicU32) -> bool { + self.worker + } + + fn is_worker(&self, _: &AtomicU32) -> bool { + self.worker + } + + fn release(&self, _: &AtomicU32) {} + + fn park(&self, word: &AtomicU32, seen: u32) -> WaitOutcome { + let left = self.parks.load(Ordering::Relaxed); + if left == 0 { + return WaitOutcome::Unsupported; + } + if left != usize::MAX { + self.parks.store(left - 1, Ordering::Relaxed); + } + if word.load(Ordering::SeqCst) != seen { + return WaitOutcome::Retry; + } + thread::yield_now(); + WaitOutcome::Woken + } + + fn kick_host(&self) {} + + fn exit(&self) -> ! { + std::panic::resume_unwind(Box::new(Exited)) + } + } + + /// The host's side, polled: serve calls with `answer` until the worker + /// yields the cell. + fn serve

(bridge: &Bridge

, mut answer: impl FnMut(&mut P)) -> Settled { + loop { + match bridge.host_step() { + HostStep::Settled(settled) => return settled, + HostStep::Serve => { + // SAFETY: `CALL` hands the host the cell. + answer(unsafe { bridge.host_payload() }); + let _ = bridge.answer(); + } + HostStep::Wait(_) => thread::yield_now(), + } + } + } + + /// Found a "worker" over `E` on a std thread, the way `spawn` does. + fn spawn( + ends: &'static TestEnds, + first: impl FnOnce(&mut E::Payload), + ) -> thread::JoinHandle<()> { + let bridge = E::bridge(); + assert!(bridge.claim(), "a vacant bridge is claimed once"); + first(unsafe { bridge.host_payload() }); + bridge.hand_over_first(); + thread::spawn(move || enter::(ends)) + } + + #[derive(Default)] + struct Sums { + input: u64, + reply: u64, + total: u64, + calls: u32, + } + + #[test] + fn every_state_round_trips_through_the_word() { + for gone in [ + Gone::Ended, + Gone::Quit, + Gone::Failed, + Gone::Unparked, + Gone::Panicked, + ] { + assert_eq!(decode(gone.word()), State::Gone(gone)); + } + assert_eq!(decode(VACANT), State::Vacant); + assert_eq!(decode(IDLE), State::Idle); + assert_eq!(decode(WORK), State::Work); + assert_eq!(decode(CALL), State::Call); + assert_eq!(decode(QUIT), State::Quit); + // The probe's value is never a state a worker could be waiting in. + assert!(matches!(decode(NEVER), State::Gone(_))); + // A word this module never writes reads as gone: fail closed. + assert_eq!(decode(0x77), State::Gone(Gone::Panicked)); + } + + #[test] + fn a_bridge_has_one_host_at_a_time() { + static BRIDGE: Bridge = Bridge::new(0); + assert!(!BRIDGE.is_hosted()); + assert!(BRIDGE.claim()); + assert!(BRIDGE.is_hosted()); + assert!(!BRIDGE.claim(), "a held bridge is not claimed twice"); + BRIDGE.vacate(); + assert!(BRIDGE.claim(), "a vacated bridge is claimed again"); + } + + // A compiler-shaped engine: one item, many calls, then return. + struct Summer; + static SUMMER: Bridge = Bridge::new(Sums { + input: 0, + reply: 0, + total: 0, + calls: 0, + }); + static SUMMER_ENDS: TestEnds = TestEnds::worker(); + + impl Engine for Summer { + type Payload = Sums; + fn bridge() -> &'static Bridge { + &SUMMER + } + fn run(mut work: Work) { + let calls = work.with(|p| p.calls); + let mut total = 0; + for i in 0..u64::from(calls) { + let reply = SUMMER + .call_with(&SUMMER_ENDS, |p| p.input = i, |p| p.reply) + .expect("inside the item, on the worker"); + total += reply; + } + work.finish(|p| p.total = total); + } + } + + #[test] + fn calls_hand_the_cell_back_and_forth() { + let calls = if cfg!(miri) { 24 } else { 2000 }; + let worker = spawn::(&SUMMER_ENDS, |p| p.calls = calls); + let first = serve(&SUMMER, |p| p.reply = p.input * 3 + 1); + // The body finished, then returned: the host reads the item's end and + // then, or instead, the worker's. + assert!(matches!(first, Settled::Idle | Settled::Gone(Gone::Ended))); + worker.join().unwrap(); + assert_eq!( + serve(&SUMMER, |_| unreachable!()), + Settled::Gone(Gone::Ended) + ); + let expected: u64 = (0..u64::from(calls)).map(|i| i * 3 + 1).sum(); + assert_eq!(unsafe { SUMMER.host_payload() }.total, expected); + SUMMER.vacate(); + } + + // A REPL-shaped engine: items posted one by one until a quit. + struct Repl; + static REPL: Bridge> = Bridge::new(Vec::new()); + static REPL_ENDS: TestEnds = TestEnds::worker(); + + impl Engine for Repl { + type Payload = Vec; + fn bridge() -> &'static Bridge> { + &REPL + } + fn run(first: Work>) { + // The engine is built, and says it is ready. + first.finish(|p| p.push(0)); + loop { + match REPL.next_with(&REPL_ENDS) { + Ok(Next::Work(mut work)) => { + let item = work.with(|p| *p.last().unwrap()); + // Each item makes one call before it finishes. + let doubled = REPL + .call_with(&REPL_ENDS, |p| p.push(item), |p| *p.last().unwrap()) + .unwrap(); + work.finish(|p| p.push(doubled + 1)); + } + Ok(Next::Quit) => return, + other => panic!("unexpected {:?}", other.is_ok()), + } + } + } + } + + #[test] + fn items_are_posted_served_and_quit() { + let worker = spawn::(&REPL_ENDS, |p| p.clear()); + // The first item (the engine's construction) settles first. + assert_eq!(serve(&REPL, |_| unreachable!()), Settled::Idle); + let items = if cfg!(miri) { 8 } else { 300 }; + for item in 1..=items { + REPL.post_with(|p| p.push(item)).unwrap(); + let settled = serve(&REPL, |p| { + let asked = p.pop().unwrap(); + p.push(asked * 2); + }); + assert_eq!(settled, Settled::Idle); + assert_eq!(unsafe { REPL.host_payload() }.last(), Some(&(item * 2 + 1))); + } + assert!(matches!(REPL.ask_quit(), QuitAsk::Asked)); + assert_eq!(serve(&REPL, |_| unreachable!()), Settled::Gone(Gone::Quit)); + worker.join().unwrap(); + assert!(matches!(REPL.ask_quit(), QuitAsk::Leaving)); + REPL.vacate(); + } + + // An engine whose park the kernel refuses mid-call. + struct GivesUp; + static GIVES_UP: Bridge = Bridge::new(0); + static GIVES_UP_ENDS: TestEnds = TestEnds::parking_only(usize::MAX); + + impl Engine for GivesUp { + type Payload = u64; + fn bridge() -> &'static Bridge { + &GIVES_UP + } + fn run(_work: Work) { + // The probe passed; the next park is refused. + GIVES_UP_ENDS.parks.store(0, Ordering::Relaxed); + let _ = GIVES_UP.call_with(&GIVES_UP_ENDS, |p| *p = 7, |p| *p); + panic!("a worker that gave its call up returned into it"); + } + } + + #[test] + fn a_worker_that_cannot_park_gives_its_call_up_and_exits() { + let worker = spawn::(&GIVES_UP_ENDS, |p| *p = 0); + let error = worker.join().expect_err("the worker exits, never returns"); + assert!(error.is::()); + // The host had not answered: its answer now has no reader. + assert_eq!( + GIVES_UP.host_step(), + HostStep::Settled(Settled::Gone(Gone::Unparked)) + ); + assert!(!GIVES_UP.answer()); + GIVES_UP.vacate(); + } + + #[test] + fn a_give_up_and_an_answer_race_and_exactly_one_wins() { + static RACE: Bridge = Bridge::new(0); + let rounds = if cfg!(miri) { 9 } else { 600 }; + let mut answered = 0; + let mut abandoned = 0; + for round in 0..rounds { + assert!(RACE.claim()); + RACE.hand_over_first(); + // The worker posts a call, and its park is refused after a few + // turns. One round in three forces each order; the third races. + let (parks, order) = match round % 3 { + 0 => (0, Order::WorkerFirst), + 1 => (usize::MAX, Order::HostFirst), + _ => ((round % 5) as usize, Order::Race(round % 7)), + }; + let ends = TestEnds::parking_only(parks); + let (answer, joined) = thread::scope(|scope| { + let worker = scope.spawn(|| RACE.call_with(&ends, |p| *p = round, |p| *p).unwrap()); + order.wait(|| matches!(RACE.state(), State::Gone(_))); + // The host answers whenever it sees the call. + let answer = loop { + match RACE.host_step() { + HostStep::Serve => { + unsafe { *RACE.host_payload() = round + 1 }; + break RACE.answer(); + } + HostStep::Settled(Settled::Gone(Gone::Unparked)) => break false, + _ => thread::yield_now(), + } + }; + (answer, worker.join()) + }); + match order { + Order::WorkerFirst => assert!(!answer && joined.is_err()), + Order::HostFirst => assert!(answer && joined.is_ok()), + Order::Race(_) => {} + } + match joined { + Ok(reply) => { + assert!(answer, "a worker reads an answer only the host made"); + assert_eq!(reply, round + 1); + assert_eq!(RACE.state(), State::Work); + answered += 1; + } + Err(error) => { + assert!(error.is::()); + assert!(!answer, "an answered call is never also given up"); + assert_eq!(RACE.state(), State::Gone(Gone::Unparked)); + abandoned += 1; + } + } + RACE.vacate(); + } + assert_eq!(answered + abandoned, rounds); + assert!(answered > 0 && abandoned > 0, "both orders ran"); + } + + #[test] + fn a_post_and_a_leaving_worker_race_and_exactly_one_wins() { + static LEAVE: Bridge = Bridge::new(0); + let rounds = if cfg!(miri) { 9 } else { 600 }; + let (mut taken, mut refused) = (0, 0); + for round in 0..rounds { + assert!(LEAVE.claim()); + // The worker waits for an item, and its park is refused after a + // few turns, so it leaves out of the host-held `IDLE`. One round + // in three forces each order; the third races them. + let (parks, order) = match round % 3 { + 0 => (0, Order::WorkerFirst), + 1 => (usize::MAX, Order::HostFirst), + _ => ((round % 4) as usize, Order::Race(round % 7)), + }; + let ends = TestEnds::parking_only(parks); + let (posted, took) = thread::scope(|scope| { + let worker = scope.spawn(|| match LEAVE.next_with(&ends) { + Ok(Next::Work(work)) => { + // The post won: the worker holds the item and + // finishes it. + work.finish(|p| *p += 1); + true + } + Ok(Next::Gone(Gone::Unparked)) => false, + _ => panic!("unexpected next"), + }); + order.wait(|| matches!(LEAVE.state(), State::Gone(_))); + let posted = LEAVE.post_with(|p| *p = round).is_ok(); + (posted, worker.join().unwrap()) + }); + match order { + Order::WorkerFirst => assert!(!posted && !took), + Order::HostFirst => assert!(posted && took), + Order::Race(_) => {} + } + taken += usize::from(took); + refused += usize::from(!posted); + if took { + assert!(posted); + assert_eq!(LEAVE.host_step(), HostStep::Settled(Settled::Idle)); + assert_eq!(unsafe { *LEAVE.host_payload() }, round + 1); + } else { + // Whichever came first, the host now reads the worker gone and + // no item is left unread in a worker-held state. + assert_eq!( + LEAVE.host_step(), + HostStep::Settled(Settled::Gone(Gone::Unparked)) + ); + } + LEAVE.vacate(); + } + assert!(taken > 0 && refused > 0, "both orders ran"); + } + + /// Who moves first in a race round. + #[derive(Clone, Copy)] + enum Order { + /// The worker's move is waited for before the host's. + WorkerFirst, + /// The worker cannot move before the host does. + HostFirst, + /// Neither waits; the host yields this many turns first. + Race(u32), + } + + impl Order { + /// Hold the host back as the order asks; `moved` reads whether the + /// worker has made its move. + fn wait(self, moved: impl Fn() -> bool) { + match self { + Self::WorkerFirst => { + while !moved() { + thread::yield_now(); + } + } + Self::HostFirst => {} + Self::Race(turns) => { + for _ in 0..turns { + thread::yield_now(); + } + } + } + } + } + + // A one-item engine that finishes and returns while the host quits it. + struct OneShot; + static ONE_SHOT: Bridge = Bridge::new(0); + static ONE_SHOT_ENDS: TestEnds = TestEnds::worker(); + + impl Engine for OneShot { + type Payload = u32; + fn bridge() -> &'static Bridge { + &ONE_SHOT + } + fn run(work: Work) { + work.finish(|p| *p = 42); + } + } + + #[test] + fn a_quit_and_a_returning_body_race_and_the_worker_is_gone_either_way() { + let rounds = if cfg!(miri) { 4 } else { 200 }; + for _ in 0..rounds { + let worker = spawn::(&ONE_SHOT_ENDS, |p| *p = 0); + // Wait for the item's end, then quit while the body returns. + loop { + if let HostStep::Settled(_) = ONE_SHOT.host_step() { + break; + } + thread::yield_now(); + } + assert_eq!(unsafe { *ONE_SHOT.host_payload() }, 42); + let ask = ONE_SHOT.ask_quit(); + assert!(matches!(ask, QuitAsk::Asked | QuitAsk::Leaving)); + worker.join().unwrap(); + assert!(matches!( + ONE_SHOT.state(), + State::Gone(Gone::Quit | Gone::Ended) + )); + ONE_SHOT.vacate(); + } + } + + // An engine that cannot start. + struct NoStart; + static NO_START: Bridge<&'static str> = Bridge::new(""); + static NO_START_ENDS: TestEnds = TestEnds::worker(); + + impl Engine for NoStart { + type Payload = &'static str; + fn bridge() -> &'static Bridge<&'static str> { + &NO_START + } + fn run(work: Work<&'static str>) { + work.fail(|p| *p = "no engine"); + } + } + + #[test] + fn a_failed_item_leaves_the_worker_gone_with_its_reason_in_the_payload() { + let worker = spawn::(&NO_START_ENDS, |p| *p = ""); + worker.join().unwrap(); + assert_eq!( + serve(&NO_START, |_| unreachable!()), + Settled::Gone(Gone::Failed) + ); + assert_eq!(unsafe { *NO_START.host_payload() }, "no engine"); + // A post to a gone worker is refused with its reason. + assert_eq!(NO_START.post_with(|_| ()), Err(NotIdle::Gone(Gone::Failed))); + NO_START.vacate(); + } + + // A thread that cannot park at all never runs the body. + struct NeverRuns; + static NEVER_RUNS: Bridge = Bridge::new(false); + static NEVER_RUNS_ENDS: TestEnds = TestEnds::parking_only(0); + + impl Engine for NeverRuns { + type Payload = bool; + fn bridge() -> &'static Bridge { + &NEVER_RUNS + } + fn run(work: Work) { + work.finish(|p| *p = true); + } + } + + #[test] + fn the_probe_keeps_an_unparkable_worker_from_running() { + let worker = spawn::(&NEVER_RUNS_ENDS, |p| *p = false); + worker.join().unwrap(); + assert_eq!( + serve(&NEVER_RUNS, |_| unreachable!()), + Settled::Gone(Gone::Unparked) + ); + assert!(!unsafe { *NEVER_RUNS.host_payload() }); + NEVER_RUNS.vacate(); + } + + #[test] + fn worker_side_operations_refuse_everyone_but_the_worker_inside_an_item() { + static ROOM: Bridge = Bridge::new(0); + static OUTSIDER: TestEnds = TestEnds::outsider(); + static WORKER: TestEnds = TestEnds::worker(); + assert!(ROOM.claim()); + // Between items (`IDLE`): the cell is the host's. + assert_eq!(ROOM.with_work_as(&WORKER, |_| ()), Err(Refused::NoWork)); + assert_eq!( + ROOM.call_with(&WORKER, |_| (), |_| ()), + Err(Refused::NoWork) + ); + ROOM.hand_over_first(); + // Inside an item, only the worker. + assert_eq!( + ROOM.with_work_as(&OUTSIDER, |_| ()), + Err(Refused::NotWorker) + ); + assert_eq!( + ROOM.call_with(&OUTSIDER, |_| (), |_| ()), + Err(Refused::NotWorker) + ); + assert!(matches!(ROOM.next_with(&OUTSIDER), Err(Refused::NotWorker))); + assert!(!ROOM.leave_with(&OUTSIDER, Gone::Ended)); + // A borrow inside a borrow aliases the payload: refused, untouched. + let nested = ROOM.with_work_as(&WORKER, |p| { + *p = 5; + ROOM.call_with(&WORKER, |q| *q = 6, |_| ()) + }); + assert_eq!(nested, Ok(Err(Refused::Nested))); + assert_eq!(ROOM.state(), State::Work, "the refused call posted nothing"); + assert_eq!(ROOM.with_work_as(&WORKER, |p| *p), Ok(5)); + // The worker ends itself, holding its item. + let mut work = Work::new(&ROOM); + assert_eq!(work.with(|p| *p), 5); + assert!(ROOM.leave_with(&WORKER, Gone::Ended)); + assert!(!ROOM.leave_with(&WORKER, Gone::Ended), "once"); + assert_eq!(ROOM.state(), State::Gone(Gone::Ended)); + // The item is dead with it: the cell is the host's now, and its + // finish may not write the word back to `IDLE`. + let late = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + work.finish(|p| *p = 9); + })); + assert!(late.is_err()); + assert_eq!(ROOM.state(), State::Gone(Gone::Ended)); + assert_eq!(unsafe { *ROOM.host_payload() }, 5); + ROOM.vacate(); + } + + #[test] + fn a_panic_is_the_last_word_from_any_state_but_gone() { + for (from, expected) in [ + (IDLE, Some(Gone::Panicked)), + (WORK, Some(Gone::Panicked)), + (CALL, Some(Gone::Panicked)), + (QUIT, Some(Gone::Panicked)), + (Gone::Quit.word(), None), + (VACANT, None), + ] { + let word = AtomicU32::new(from); + assert_eq!(settle(&word, Gone::Panicked), expected.is_some()); + match expected { + Some(gone) => assert_eq!(decode(word.load(Ordering::Relaxed)), State::Gone(gone)), + None => assert_eq!(word.load(Ordering::Relaxed), from), + } + } + // A worker ending itself while asked to quit has answered the quit. + let word = AtomicU32::new(QUIT); + assert!(settle(&word, Gone::Ended)); + assert_eq!( + decode(word.load(Ordering::Relaxed)), + State::Gone(Gone::Quit) + ); + // Off metal no thread is a worker: the panic path leaves no word. + panicked(); + } + + #[test] + fn a_host_serves_only_calls_and_waits_out_the_workers_states() { + static STEPS: Bridge = Bridge::new(0); + for (word, step) in [ + (IDLE, HostStep::Settled(Settled::Idle)), + (WORK, HostStep::Wait(WORK)), + (QUIT, HostStep::Wait(QUIT)), + (CALL, HostStep::Serve), + ( + Gone::Failed.word(), + HostStep::Settled(Settled::Gone(Gone::Failed)), + ), + ] { + STEPS.word().store(word, Ordering::Relaxed); + assert_eq!(STEPS.host_step(), step); + } + // A post needs the idle cell, and quitting a busy worker is refused. + STEPS.word().store(WORK, Ordering::Relaxed); + assert_eq!(STEPS.post_with(|_| ()), Err(NotIdle::Busy)); + assert!(matches!(STEPS.ask_quit(), QuitAsk::Busy)); + STEPS.vacate(); + } +} diff --git a/native/crates/rt/src/crash.rs b/native/crates/rt/src/crash.rs index f61c060..34f4cd1 100644 --- a/native/crates/rt/src/crash.rs +++ b/native/crates/rt/src/crash.rs @@ -123,10 +123,14 @@ pub fn panic(info: &core::panic::PanicInfo<'_>) -> ! { /// end — a host that ends the whole process on a worker's panic tells the /// host only once the record is published, so the host's barrier cannot cut /// the report short. `true` when this call published the record. +/// +/// A [`bridge`](crate::bridge) worker's host is told here, after the record: +/// the worker's bridge reads `GONE`·`Panicked` and the host is kicked. pub fn report(info: &core::panic::PanicInfo<'_>) -> bool { let first = crate::entry::begin_panic(); if first { report_panic_info(info); + crate::bridge::panicked(); } first } diff --git a/native/crates/rt/src/entry.rs b/native/crates/rt/src/entry.rs index 00b9b80..77128a1 100644 --- a/native/crates/rt/src/entry.rs +++ b/native/crates/rt/src/entry.rs @@ -720,6 +720,15 @@ pub(crate) mod geometry { /// and its argument. Taken by the exit, so it runs once. exit_call: AtomicUsize, exit_argument: AtomicUsize, + /// The [`bridge`](crate::bridge) this thread is the worker of — the + /// address of its word — or `0` for every other thread. Written by + /// the worker's own entry before anything else runs on it, so a worker + /// panic from its first instruction reaches the host; read by the + /// worker's bridge calls as the proof that their caller is the worker, + /// and by the panic paths, which give the host the worker's last word. + /// Not inherited: a spawned thread's block is its own, and zero. Only + /// the owning thread reads or writes it. + hosted_bridge: AtomicUsize, } // The block must fit the founder's one-page window. @@ -821,6 +830,11 @@ pub(crate) mod geometry { block().panicking.swap(1, Ordering::AcqRel) == 0 } + /// This thread has begun a panic ([`begin_panic`]). + pub(crate) fn panicking() -> bool { + block().panicking.load(Ordering::Acquire) != 0 + } + /// This thread's C `errno` word (the block's `c_errno` field). /// /// Lawful wherever [`block`] is: C code runs only on an anchored @@ -890,6 +904,17 @@ pub(crate) mod geometry { Some((call, b.exit_argument.load(Ordering::Relaxed))) } + /// The bridge this thread is the worker of, by its word's address (the + /// block's `hosted_bridge` field); `0` when it is no bridge's worker. + pub(crate) fn hosted_bridge() -> usize { + block().hosted_bridge.load(Ordering::Relaxed) + } + + /// Name (or, with `0`, unname) the bridge this thread is the worker of. + pub(crate) fn set_hosted_bridge(word: usize) { + block().hosted_bridge.store(word, Ordering::Relaxed); + } + /// Where this thread's block lives — the value the thread pointer holds, /// recovered the same way every other accessor recovers it. pub(crate) fn block_va() -> usize { @@ -2123,6 +2148,8 @@ pub fn panic_refuse(info: &core::panic::PanicInfo<'_>) -> ! { refuse(); } crate::crash::report_panic_info(info); + // A bridge worker's host hears it is gone, now the record is published. + crate::bridge::panicked(); let payload = info .location() .map_or(RUNG_PANIC, |at| panic_payload(at.file(), at.line())); diff --git a/native/crates/rt/src/futex.rs b/native/crates/rt/src/futex.rs index bebb188..3a2db5d 100644 --- a/native/crates/rt/src/futex.rs +++ b/native/crates/rt/src/futex.rs @@ -2013,8 +2013,14 @@ mod tests { claim_wait(8, address_of(&a)).expect("the hub's own wait"); claim_wait(9, address_of(&a)).expect("a doorbell waiter"); claim_wait_on_word(10, address_of(&b)).expect("a word waiter"); + // The sweep reads every row, and the suite's other tests hold rows of + // their own meanwhile: only these four are this test's to judge. A + // kick of any other row is refused, so it is not counted either. let mut kicked = std::vec::Vec::new(); let n = sweep_with(8, |thread| { + if !(8..=11).contains(&thread) { + return false; + } kicked.push(thread); true }); diff --git a/native/crates/rt/src/lib.rs b/native/crates/rt/src/lib.rs index 3c5bb09..f398081 100644 --- a/native/crates/rt/src/lib.rs +++ b/native/crates/rt/src/lib.rs @@ -48,6 +48,11 @@ //! [`MemSource`](heap::MemSource) backing. kore-rt ships the engine, //! the seam, and the metal source (`heap::HeapSupplySource` — the //! founder-provisioned grow window every founded binary links). +//! - [`bridge`] — **a synchronous engine on a worker thread**, served by its +//! async host: one request cell and one word, whose states, park and kick +//! ordering, cell hand-over and worker-gone handling live here once. Lua, +//! TinyCC and an independent C program's `main` are the same exchange with +//! different payloads ([`bridge::Engine`], [`bridge::Bridge`]). //! - [`service::SystemService`] — the unified shape of a bootstrapped //! service (KRAPH §5.2's role bundle + a serve step): the endowment is a //! typed [`Bundle`](service::Bundle) resolved fail-closed from the @@ -84,6 +89,7 @@ extern crate std; pub mod barrier; pub mod block; +pub mod bridge; pub mod cap; pub mod chan; pub mod clock; diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index 7064582..8e7c3d5 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -1001,13 +1001,16 @@ pub(crate) const fn gang_founding_slots(workers: usize, tls: bool, endowments: u /// Minting one is the star's rule (`res/SPMD.md` §3) applied to a single /// peer. /// -/// This is the shape a synchronous engine on a worker thread needs — a +/// This is the transport a synchronous engine on a worker thread needs — a /// C interpreter whose `require` must hand a request to the executor /// and wait for the answer without spinning (`res/DAGFS.md` §4): the /// worker `futex::wait`s on a word and parks on its doorbell; the /// executor answers, stores, and [`kick`](Self::kick)s; the worker /// stores and [`kick_parent`]s; the executor parks on -/// [`wake_recv`](Self::wake_recv) through `next_wake_on`. +/// [`wake_recv`](Self::wake_recv) through `next_wake_on`. An engine does +/// not speak that protocol itself: [`bridge::WorkerHost`](crate::bridge) +/// founds the worker on this and owns the word, its ordering and the +/// worker's end, so every engine shares one copy of it. #[cfg(target_os = "none")] #[must_use] pub struct DoorbellWorker { @@ -2269,8 +2272,9 @@ pub fn exit(_code: ExitCode) -> ! { /// Have this thread's [`exit`] run `call(argument)` first, on this thread, /// before the exit commits — how the thread that reaps it learns it is /// ending without polling for it (the -/// [`thread_controller`](crate::thread_controller)'s children). The last hook -/// set stands. `call` runs after the thread's own code, so it must not block, +/// [`thread_controller`](crate::thread_controller)'s children; a +/// [`bridge`](crate::bridge) worker's host). The last hook set stands, so the +/// hook is the thread's entry's to set. `call` runs after the thread's own code, so it must not block, /// and must not end the thread itself. /// /// The kernel's exit commits after the hook returns, so a reaper woken by diff --git a/native/provenance.json b/native/provenance.json index bbbd22f..9a0583c 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "10c7b400aa96c7533be05054fb76e43850408b23", + "revision": "6a8c754bc959e9489a6fb18be642fcc881a65c48", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -209,6 +209,7 @@ "crates/rt/Cargo.toml": "0b472a92c92d74738202204ccafe53088ba9d303c3c8d4ac09a9dc29204a4d23", "crates/rt/src/barrier.rs": "c5bc3bef52f2bcc24edb1016fde955bd01e73074d4ce357d79cae9d3b63e6f3b", "crates/rt/src/block.rs": "70c2aef8bf149da2f32c7c995cd0f4c63cb492647896b107fdd5dbecd3d3042a", + "crates/rt/src/bridge.rs": "872333db41ba23c0aa699a2ca8ffdf782fbfabaf3d25e626a25d2a248593e89a", "crates/rt/src/cap/mapped.rs": "b29ce431638a29b01edfd72500348f5ae201afe97ec4684ea4449d01e75ec1e7", "crates/rt/src/cap/mod.rs": "2e7bd2965128172e4f0f700dd8b64b161236ba4091df13e87d753bb71a71bf33", "crates/rt/src/cap/record.rs": "640b0ff60052d4775c259489af5fbff5cd03164370aceb4775186be956817bcd", @@ -219,15 +220,15 @@ "crates/rt/src/clock/riscv64.rs": "17d4ecdb2b4f8152c0cf51f71ce28fe331d929a62405810a1b7eb332bb275c3d", "crates/rt/src/clock/x86_64.rs": "ca499d4dc51b7ed0eb196b831295b5fdecb09a1bd2df628a82e749e27011994a", "crates/rt/src/core_id.rs": "cadde39801a1d0ed05d70d0634a9e52d58043878737d790c4130183751b622cf", - "crates/rt/src/crash.rs": "b6463a490f80124e22fdb5e13652bbaa94461eb1c4aefcffeca2e7dc7e977611", - "crates/rt/src/entry.rs": "499fd1ac8c1161ceb1241f093f167d7fcb8a5ef0405e5b43dc8e7293942058f3", + "crates/rt/src/crash.rs": "43266939e7537774f0919776c5e07889e1a224be689d75ecd91735c91ceb3a9f", + "crates/rt/src/entry.rs": "d88aea5008e3d0dc93775242683d811bee51f5df14b4e4bfbc4f52c5322e887e", "crates/rt/src/event.rs": "709a5500231735ad5bdb9ab9ab2fcee7b8f72f93e1b4741afba72fdc8b7c54f3", "crates/rt/src/executor/mod.rs": "071e3dc66bca448e6dabf96976702600bc77fdc98933d58847662fb2de073cb8", "crates/rt/src/executor/task.rs": "b3017cb97f287b8796c61d245c49e5c1b0ebfabe8c535cc2b3842cdbc854dc29", "crates/rt/src/executor/time.rs": "62f9ca9e5e17b395ec2b19a52fc6ed1bb53fd4acc3554cfca81eaf20e48fc272", "crates/rt/src/exit.rs": "3bf7e8ce6dac63b4ce365bdf2e743a9fd171851f1c5e40f8afbb4548877cef1b", "crates/rt/src/fiber_stack.rs": "48bf4fedba07ea2826615368e22882301ae4e0bbce0d893d0d7754d86bd02ce6", - "crates/rt/src/futex.rs": "df5622a1bd811d7d97ec20dc44a568dba3a2c995254fc30405a60d41371e7390", + "crates/rt/src/futex.rs": "dd0c0e1bb279e5df5a7b767a59d8bed4cdccbddfd627a4e4990a729449367a88", "crates/rt/src/heap/mod.rs": "a9c39d3de56187a14d4633a229cbbb6e7ddb2c438d6f9f154b3b2551031b9f81", "crates/rt/src/heap/pool.rs": "67d42c6ffc7c7e176d21e97dd3179a3e4ff331f46dd457ed4d29914b1a404de8", "crates/rt/src/heap/shard.rs": "d97f91b3ecc71e3ec56fde0017f9d98cb99c5fbeeffe6ff79ddd153f5c6690fa", @@ -237,7 +238,7 @@ "crates/rt/src/lane/mod.rs": "6912c495dfbb3bec8e9ddbb6cf0e5458ca5318b6e90882ae3433c2f0649d2313", "crates/rt/src/lane/switch/mod.rs": "932f1e204f8abeae5614a9caea091e982875c7eea2db87a2a7a643f4a1ff577d", "crates/rt/src/lane/switch/x86_64.rs": "97c83af0e63523a220924f50fb2b058fcda650cc41445f0d010fa128ecc41366", - "crates/rt/src/lib.rs": "205cfacaf11ab1a1f883b24caf2abc2d221d8a0b97ccf501939a1d985e69952c", + "crates/rt/src/lib.rs": "d7debd32ceed05279e16e391ec22e7b9b52b5fc33198318dc08b5b5912e75777", "crates/rt/src/mem_source.rs": "41cbfb3200c222c2d2d0094b19445a9cf7a016e80f20241f8f5cf22872c5ce56", "crates/rt/src/pace.rs": "81217909cee01fa148df9e953e6281d7abf328eed7c6257aac8fbcebc824710c", "crates/rt/src/pinned.rs": "9f86a46572178965c3a111363eb20431fc19916eb9e06c2da50f8f918f3eccf8", @@ -264,7 +265,7 @@ "crates/rt/src/svc/serve.rs": "a22172638a94e3ab52b88095dff9d78c88a1d693c81c6242af4932881d268706", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "06d0eefa92c46fe31392db8b10e891e008e95c3ee4162f9f4034f7dec7d9b35c", + "crates/rt/src/thread.rs": "7d99177943316041c361b40d8f302b52e571b5e9b51ab9d136ac5b7685233239", "crates/rt/src/thread_controller.rs": "6ce535ded212d30b4838d15108cf9e9625b4f9e9fd83e0fecde51be833b1326d", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", @@ -315,13 +316,13 @@ "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", - "services/block/boot.toml": "00aa7a50eb6cea63d8da291572b941ef44136a57682715a73a811d0610055c6a", + "services/block/boot.toml": "33e5fbd50b82878632c4fa3fd5a36aebabc9ef1e313fd06c91db9f78bf064543", "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "25a46002f036013e4e1dd87927324713a4c184a3c73b5e40435aedb4615c0586", + "services/dagfs/boot.toml": "4202704fd02113c2e36ab588e6f0e00c7d96da31c746cd644a6d238e93fd52e6", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", @@ -363,7 +364,7 @@ "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", - "services/device/boot.toml": "3d5eaf0c17d8a4a976405c9d3182319a0be1d36993842b95cf3be6ce64cc721a", + "services/device/boot.toml": "22c2a78e9be0e6784260668c31b1f8e49f47bcab0c684b6ac1a8d02249c3cb90", "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", "services/device/src/service/mod.rs": "6038ba0b9375e3c6bbb25c49bf0849a5878a402f4a3a691dc9df1494849d252b", @@ -405,7 +406,7 @@ "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", - "services/mm/boot.toml": "294b4639deec93e2ef5188522da1d3f5ef0428e8a37e21616a697a2b6182fd24", + "services/mm/boot.toml": "6cc0fd1224b6387ef80b0146b8048b45f2f5d15c4b6f658d473556c71ebfde6d", "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", @@ -417,7 +418,7 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "4b6d4873ce4124a5d8ffa6bf93d15fdce0be213520e909ac9baeceec1ab2c120", + "services/swap/boot.toml": "7dd8ff4c10f9dc04ddb969cf9f3a12220023a2233c476fbde6bd776bb562206f", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml index b085cfb..2e00c72 100644 --- a/native/services/block/boot.toml +++ b/native/services/block/boot.toml @@ -12,5 +12,5 @@ chain = true endowment = "kore_svc_block::BlockEndowment" protocol = "kore_svc_block::BlockService" endows = ["DMA_POOL@1"] -digest_x86_64 = "blake3:a22f3a5b08f35a278ebf663d557e0b1ec490c8e732cdee0fa68151e349ddbc57" -digest_aarch64 = "blake3:93e621b55fbacf93391e4a32212cacfed71ede1797fb63935e2e971da15ab824" +digest_x86_64 = "blake3:6658f8f4c97b44cf5da083aabc866fca9066790c84104f8a2e2f5e980dbef60c" +digest_aarch64 = "blake3:1a549b72358df27ddf6d56fd804980e15dc2c9d7e9a392922b74516e3b4947d9" diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index ffb941d..a7765b2 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -47,5 +47,5 @@ exec_client = true # `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 -digest_x86_64 = "blake3:453a43f78c11f4fedf1715a141e4627a4d9858ab975fc8ad03c5d44b8814b7d9" -digest_aarch64 = "blake3:44ed7f47afd777ec49dc5b1193947e3d9cbf3ad3534f2b6edd162056abab35fe" +digest_x86_64 = "blake3:21134f67366457e67f9cb95a21642283ba04896df77e1dd0a745fbb5cd9f9e47" +digest_aarch64 = "blake3:64dd2f658d903d9c55ef36c3aa3488c7ebb545c5e83793b8823298eee05aa7d9" diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml index 021ef5d..81a3a77 100644 --- a/native/services/device/boot.toml +++ b/native/services/device/boot.toml @@ -47,5 +47,5 @@ protocol = "kore_svc_device::DeviceBroker" # row's own claimant band. instances = 8 endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] -digest_x86_64 = "blake3:78d5b044c7b3ad3467484859549f8404cffc61ece27dd793cdd21c8c2c0a930a" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:5632f4509b69316c1cac719cd0ddbf5a4ac0239db6fb42488ca6ec9811de4382" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:1c106650a75981ec2767aec603e78c0552a14484a1d64d848703a3d53a16102f" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:6ebaa69d4e354314c2c3fad76611aec9cde219b2d9f2afef993e91d9392f11cf" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml index eee0bbd..600d1fb 100644 --- a/native/services/mm/boot.toml +++ b/native/services/mm/boot.toml @@ -54,5 +54,5 @@ supply_quota_bytes = 27917287424 # 26 GiB endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirror; asserted === PROFILE.roles shell_client = true exec_client = true -digest_x86_64 = "blake3:d1ce3128c73f02293e323f87d79850b8558dfd82d882b65169767febb7baf6e4" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:901b68a3acc55e7424ee98c90758fa17d5675f50292a1bc8e277591b2f0f5803" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:b68f42add3eea14c162334ca3a26e5357f1777298d46a1a969ca94d96f9e4ca0" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:ccbc42fb7b4188f46196c88c56ec42be4e756e7f654f24617f31675a4ae0fcc1" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index 3c9cab3..be7578d 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:dddc0488ba218b48a691a1e10e5995e962296056daf64ef0408ab15ecbd3a67f" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:b18a607ce1dec8f172effb4ae331e3be712ee99c6d3951dd33e907c5491e5cba" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:5b7a9e35bb0732de36b6dd29d909025820bcb69df7438c7ae9eac9fef6ca7211" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:cbe8173eee0e76e47bc497fc2bcbf4cc9ee66710c6788cd15f610dd5eb3d57c3" # res/aarch64/; make boot-digests-aarch64