diff --git a/crates/ffi/src/futex.rs b/crates/ffi/src/futex.rs index 6e04454..43910dd 100644 --- a/crates/ffi/src/futex.rs +++ b/crates/ffi/src/futex.rs @@ -61,7 +61,8 @@ pub extern "C" fn kore_futex_available() -> u32 { /// all waits/wakes. All concurrent accesses must be compatible 32-bit atomic /// operations (C atomic builtins or Zig atomics); never mix ordinary accesses /// or differently sized atomics. Managed-thread waits are cancellation points: -/// they may exit without stack unwinding, as specified by kore_thread_testcancel. +/// they may exit without stack unwinding, as specified by kore_thread_testcancel, +/// and a cancellation requested while the thread waits wakes it to act. /// Publish the condition with a sequentially consistent atomic store BEFORE waking. /// No address-space sharing is implied. #[unsafe(no_mangle)] @@ -77,32 +78,51 @@ pub unsafe extern "C" fn kore_futex_wait( if !crate::thread_local::cancellable() { return futex::wait(word, expected, timeout).into(); } - // A bounded native park makes cancellation observable even without a peer - // kick capability. Recompute against one deadline; slices never restart it. unsafe { crate::kore_thread_testcancel() }; - let start = match kore::time::uptime() { - Ok(time) => time.as_nanos(), - Err(_) => return KoreFutexOutcome::Unsupported, + let outcome = match crate::thread::slice() { + // A cancellation's wake reaches this thread (kore_thread_cancel), and + // the wait re-reads the request once it is registered: it parks until + // a wake, the caller's deadline or that request, and never polls. + None => futex::wait_unless(word, expected, timeout, crate::thread_local::cancel_pending), + Some(slice) => unsafe { sliced(word, expected, timeout_ns, slice) }, }; - let deadline = timeout.map(|_| start.saturating_add(timeout_ns)); + unsafe { crate::kore_thread_testcancel() }; + outcome.into() +} + +/// A cancellable wait on a thread the controller cannot wake: no wake would +/// announce a cancellation, so it parks one `slice` at a time and acts on a +/// request between slices, against one deadline that slices never restart. +/// # Safety +/// A managed thread's cancellation point, as kore_futex_wait. +unsafe fn sliced( + word: &AtomicU32, + expected: u32, + timeout_ns: u64, + slice: Duration, +) -> futex::WaitOutcome { + let Ok(start) = kore::time::uptime() else { + return futex::WaitOutcome::Unsupported; + }; + let deadline = + (timeout_ns != KORE_FUTEX_INFINITE).then(|| start.as_nanos().saturating_add(timeout_ns)); + let slice = u64::try_from(slice.as_nanos()).unwrap_or(u64::MAX); loop { unsafe { crate::kore_thread_testcancel() }; - let now = match kore::time::uptime() { - Ok(time) => time.as_nanos(), - Err(_) => return KoreFutexOutcome::Unsupported, + let Ok(now) = kore::time::uptime() else { + return futex::WaitOutcome::Unsupported; }; - let slice = crate::thread_local::CANCEL_SLICE_NS; - let slice = deadline.map_or(slice, |end| end.saturating_sub(now).min(slice)); - let outcome = futex::wait(word, expected, Some(Duration::from_nanos(slice))); + let bound = deadline.map_or(slice, |end| end.saturating_sub(now.as_nanos()).min(slice)); + let outcome = futex::wait(word, expected, Some(Duration::from_nanos(bound))); unsafe { crate::kore_thread_testcancel() }; if outcome != futex::WaitOutcome::TimedOut { - return outcome.into(); + return outcome; } if let Some(end) = deadline { match kore::time::uptime() { Ok(time) if time.as_nanos() < end => {} - Ok(_) => return KoreFutexOutcome::TimedOut, - Err(_) => return KoreFutexOutcome::Unsupported, + Ok(_) => return futex::WaitOutcome::TimedOut, + Err(_) => return futex::WaitOutcome::Unsupported, } } } diff --git a/crates/ffi/src/thread.rs b/crates/ffi/src/thread.rs index 9e05906..5ddf34c 100644 --- a/crates/ffi/src/thread.rs +++ b/crates/ffi/src/thread.rs @@ -45,6 +45,10 @@ pub(crate) struct Control { outcome: AtomicU32, retired: AtomicU32, pub(crate) cancel: AtomicBool, + /// The native band the thread runs on, plus one; zero until it starts + /// (`futex::current_thread`). A canceller reads it after storing + /// `cancel`, to wake the thread out of a wait at a cancellation point. + band: AtomicU32, } impl Control { pub(crate) fn new() -> Self { @@ -53,13 +57,32 @@ impl Control { outcome: AtomicU32::new(0), retired: AtomicU32::new(0), cancel: AtomicBool::new(false), + band: AtomicU32::new(0), } } + /// Record the calling thread's band, before its callback can reach a + /// cancellation point. `SeqCst`, against the canceller's store of `cancel` + /// and then this load: a canceller that reads zero stored `cancel` before + /// the thread's first wait reads it. + fn start(&self) { + let band = kore::futex::current_thread().map_or(0, |band| band as u32 + 1); + self.band.store(band, Ordering::SeqCst); + } + fn band(&self) -> Option { + let band = self.band.load(Ordering::SeqCst); + band.checked_sub(1).map(|band| band as usize) + } pub(crate) fn finish(&self, value: *mut c_void, outcome: u32) { self.result.store(value, Ordering::Relaxed); self.outcome.store(outcome, Ordering::Release); } } +/// How long one wait of the calling thread may park ([`Controller::slice`]): +/// unbounded where the controller can wake it, and therefore where a +/// cancellation's wake reaches it; one bounded slice where nothing could. +pub(crate) fn slice() -> Option { + THREADS.slice() +} fn status(error: Error) -> i32 { match error { Error::Unavailable | Error::Full => KORE_THREAD_EAGAIN, @@ -86,6 +109,7 @@ fn invoke(argument: usize) -> usize { call_context(context) } fn call_context(context: &StartContext) -> usize { + context.control.start(); // SAFETY: forwarded foreign callback and argument lifetime contract. crate::thread_local::enter(&context.control); let value = unsafe { (context.call)(context.argument) }; @@ -208,12 +232,17 @@ pub unsafe extern "C" fn kore_thread_join( if control.retired.load(Ordering::SeqCst) != 0 { break; } - // A cancellation request is a flag, not a wake: a joiner that can be - // cancelled parks one slice at a time, as a cancellable futex wait - // does. Any other joiner parks until the controller retires the target. - let slice = crate::thread_local::cancellable() - .then(|| core::time::Duration::from_nanos(crate::thread_local::CANCEL_SLICE_NS)); - if kore::futex::wait(&control.retired, 0, slice) == kore::futex::WaitOutcome::Unsupported + // Park until the controller wakes `retired`, or this joiner's own + // cancellation is requested: its canceller stores the request, then + // wakes this thread (kore_thread_cancel), and the wait re-reads the + // request once it is registered. Only a joiner the controller cannot + // wake parks in bounded slices; nothing else would end its park. + if kore::futex::wait_unless( + &control.retired, + 0, + slice(), + crate::thread_local::cancel_pending, + ) == kore::futex::WaitOutcome::Unsupported // A native worker with no doorbell takes the deadline park, which // needs no wake capability at all. && !kore::runtime::pace::cede() @@ -270,7 +299,12 @@ pub unsafe extern "C" fn kore_thread_detach(thread: *mut KoreThread) -> i32 { } /// Request deferred cancellation. The target acts at testcancel, join or foreign -/// futex wait. A disabled target retains the request. No asynchronous kill occurs. +/// futex wait; a target waiting at one is woken to act. A disabled target +/// retains the request. No asynchronous kill occurs. EAGAIN when no wake the +/// calling thread sends can reach the target, because it neither started the +/// controller nor was created through it (create refuses it too). Then no +/// request is recorded, since a target parked at a cancellation point would +/// never see it. /// # Safety /// thread is a live handle. Borrowing cancel calls may run concurrently with one /// another, but must finish before a consuming join/detach can free the handle. @@ -279,12 +313,31 @@ pub unsafe extern "C" fn kore_thread_cancel(thread: *mut KoreThread) -> i32 { let Some(ticket) = NonNull::new(thread.cast::()) else { return KORE_THREAD_EINVAL; }; - let control = unsafe { &ticket.as_ref().control }; + // SAFETY: the caller supplies a live handle, borrowed for this call. + request_cancel(unsafe { &ticket.as_ref().control }) +} + +/// kore_thread_cancel's request against the target's shared control. +fn request_cancel(control: &Control) -> i32 { if control.outcome.load(Ordering::Acquire) != 0 || control.retired.load(Ordering::Acquire) != 0 { return KORE_THREAD_ESRCH; } - control.cancel.store(true, Ordering::Release); + // The request is a flag its wake announces: only the controller holds a + // send into a managed thread's doorbell, so the wake is relayed through + // it, and a thread that cannot relay through it is refused here rather + // than leaving a request behind a park nothing ends. + if !THREADS.wakes_threads() { + return KORE_THREAD_EAGAIN; + } + // Store, then wake (`futex::wake_thread`'s order). The wake reads the + // target's registration, and the target's wait reads the request once it + // is registered, so one of the two sees the other. A target that has not + // yet recorded its band reads the request at its first cancellation point. + control.cancel.store(true, Ordering::SeqCst); + if let Some(band) = control.band() { + let _ = kore::futex::wake_thread(band); + } KORE_THREAD_OK } @@ -313,6 +366,22 @@ mod tests { assert_eq!(Arc::strong_count(&control), 1); } #[test] + fn a_cancel_no_wake_can_announce_is_refused_and_never_recorded() { + // Off metal the controller never starts, so no thread's wake reaches + // its threads: the request is refused, and nothing a parked target + // would wait on unwoken is left behind. An ended target is gone first. + let control = Control::new(); + control.start(); + assert_eq!(control.band(), None, "off metal a thread has no band"); + assert_eq!(request_cancel(&control), KORE_THREAD_EAGAIN); + assert!(!control.cancel.load(Ordering::SeqCst)); + control.finish(ptr::null_mut(), 1); + assert_eq!(request_cancel(&control), KORE_THREAD_ESRCH); + assert!(!control.cancel.load(Ordering::SeqCst)); + // Every wait off metal is one the controller cannot end early. + assert!(slice().is_some()); + } + #[test] fn host_refusal_preserves_output_and_never_calls_foreign_code() { unsafe extern "C" fn never(_: *mut c_void) -> *mut c_void { panic!("host ran native callback") diff --git a/crates/ffi/src/thread_local.rs b/crates/ffi/src/thread_local.rs index 64bd577..21484c3 100644 --- a/crates/ffi/src/thread_local.rs +++ b/crates/ffi/src/thread_local.rs @@ -15,12 +15,6 @@ pub const KORE_THREAD_CANCEL_ENABLE: i32 = 0; pub const KORE_THREAD_CANCEL_DISABLE: i32 = 1; pub const KORE_THREAD_CANCEL_DEFERRED: i32 = 0; pub const KORE_THREAD_CANCEL_ASYNCHRONOUS: i32 = 1; -/// How long one park lasts at a cancellation point that waits: a request is -/// a flag no wake carries, so a cancellable wait re-checks it this often. -/// One millisecond is the cadence the native runtime parks an unreachable -/// waiter at (`kore-rt`'s `pace::CEDE_NS`): a cancellation is acted on within -/// one, and an idle cancellable waiter wakes no more often than that. -pub(crate) const CANCEL_SLICE_NS: u64 = 1_000_000; pub type KoreThreadKey = u64; pub type KoreThreadDestructor = Option; @@ -85,6 +79,16 @@ pub(crate) fn leave() { pub(crate) fn cancellable() -> bool { !CURRENT.get().is_null() && !DISABLED.get() && !FINISHING.get() } +/// An enabled cancellation request is pending for this managed thread: what +/// testcancel acts on, and the stop a cancellation point's wait re-reads once +/// it is registered (`futex::wait_unless`). `SeqCst`, as the canceller's store +/// is: the canceller stores, then reads this thread's registration before it +/// wakes it, so one of the two sees the other. +pub(crate) fn cancel_pending() -> bool { + // SAFETY: CURRENT is non-null while cancellable, and points at this + // managed thread's Control until `leave`. + cancellable() && unsafe { &*CURRENT.get() }.cancel.load(Ordering::SeqCst) +} fn active() -> bool { !CURRENT.get().is_null() } @@ -368,7 +372,7 @@ pub unsafe extern "C" fn kore_thread_setcanceltype(kind: i32, old_type: *mut i32 /// never cross live Rust RAII guards or executor callbacks at a cancellation point. #[unsafe(no_mangle)] pub unsafe extern "C" fn kore_thread_testcancel() { - if cancellable() && unsafe { &*CURRENT.get() }.cancel.load(Ordering::Acquire) { + if cancel_pending() { unsafe { kore_thread_exit(kore_thread_canceled()); } diff --git a/docs/provenance.md b/docs/provenance.md index 737162c..f94d2d6 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -351,3 +351,119 @@ C program (`docs/languages.md`). It is the first libkore package a C program on Kore links without a Rust host. Its shared object is a distribution's one-line `cdylib` over it. `tools/check-program.py` links that object for the three Kore ISAs and checks its shape and its exact exported ABI. + +## A cancellation is a wake + +The export records Kore `a4ab7234e2fe60b43a1df42f12be8f636255fc7a`, one commit +past MicroPerceptron/kore#1494's branch head `76fddb9f`. Two of the six +commits since `0018956a` reach the closure, which keeps its 26 packages and +382 files: + +- `kore-rt`'s futex gains a wait another thread can end, and the wake that + ends it (Kore `a4ab7234`). `futex::wait_unless` is `wait` with a caller + condition read once the waiter is registered, as the word is, and always + parks on the doorbell. `futex::wake_thread` kicks one thread's doorbell + rather than a word's waiters, directly or relayed through the caller's hub, + and reads the target's registration before it kicks. A condition stored + with `SeqCst` before that wake is therefore never missed, and no slice is + needed to notice it. Arm, register and check are one function that every + doorbell wait shares, and the host suite drives it with the arm injected to + pin that order. `futex::current_thread` is public: the band a thread + records for its wakers. +- Its thread controller exposes the rule its own joins used: + `Controller::slice` is unbounded where the controller can wake the calling + thread, and `UNREACHED_SLICE` where nothing could. `Controller::wakes_threads` + says whether a wake the calling thread sends reaches the threads it founded. +- The services' `boot.toml` digests are #1494's reseal over the parking + controller (`76fddb9f`). + +No exported manifest changed, so `native/Cargo.lock` did not. + +`kore-ffi`'s cancellation points no longer poll. A managed thread records its +band when it starts; `kore_thread_cancel` stores the request and wakes that +band; `kore_thread_join` and a managed thread's `kore_futex_wait` wait with +`wait_unless`, with no timeout of their own. An idle cancellable waiter now +sleeps until something happens rather than waking every millisecond. Only a +thread the controller cannot wake keeps the bounded slice, now the +controller's `UNREACHED_SLICE`. A canceller whose wake cannot reach the +controller's threads is refused with EAGAIN, as create refuses it, and records +nothing. A host test drives `request_cancel` to that refusal. The metal +evidence is Kore's `keytest-tcc-x86`, whose libkore tour cancels a thread +parked in `pthread_join` and one parked in `kore_futex_wait` with no timeout. + +The export then moved to Kore `40de25825eb1ea7816f7e7df86e116b3ad20d1fc`, +which closes a child's founding window. The controller sets a child's reach +bit only after `spawn_with_arg` returns, and the child ran its callback at +once. Its first wait could therefore start before the controller had +recorded that it can kick the child. A cancellable `kore_futex_wait` in that +window took the unreached slice and kept it until its first real wake. +`child_entry` now settles on its record until the founding is published, +which follows the reach bit, so a thread the controller founds with a +doorbell is reached from its first wait. A host test holds a founding back +and checks that the callback waits for it. The closure keeps its 26 packages +and 382 files. Besides `thread_controller.rs`, it carries the +`services/swap/boot.toml` digest of Kore's reseal over the cancellation wake +(`bfecfd17`). + +The export then moved to Kore `618772900e088b8b45ebafa8101d066edf4ffb3b`, +the merge of MicroPerceptron/kore#1494's newer branch into this change. That +branch brings three changes, which its own exports carried here up to Kore +`2d4998ba`: + +- threads that found take a founder's arena; +- a founding's staged space waits for the reap; +- Kore's C library (`kore-c-shim`). + +The closure is now 27 packages and 401 files. Against that branch's export, +only kore-rt's `futex.rs` and `thread_controller.rs` and the record differ. +The two sides' controller changes compose: the controller and its children +are founded with `spawn_founder_with_arg`, and a child still waits for its +founding before its callback runs. + +It then moved to Kore `4a56f8279c79fd4d65918e581fe382f0775dfb51`, the merge +of #1494's next round: kore-rt names the link window's free span once, and C +programs gain descriptors and mmap. The closure stays at 27 packages and 401 +files. Against that branch's export at `10c7b400`, only kore-rt's `futex.rs` +and `thread_controller.rs`, the record, and five services' `boot.toml` +digests differ. The digests come from #1494's own reseal (`f2644e5b`), which +followed its export. + +It then moved to Kore `eb59823a29cc9b0cfe0ea27a386c3fac266534ef`, the merge +of #1494's merge of Kore main `f2dc8309`. Main reaches the closure in four +places: + +- the seat clipboard and Cmd chords (#1469): kore-abi's insert into a held + input line and a window's clipboard chunks, and kore-window's clipboard, + key split and shortcuts; +- Files' live folder and sidebar (#1450, #1456): a DagFS watch plane in the + `dagfs` crate and service, and kore-rt's owed service wakes + (`SystemService::take_owed_wakes`), which the serve host delivers after + each unit of work; +- DagFS's cwd anchor and `MatchView`, `.` and `..` in `resolve`, and a dead + view reaped by asking the kernel; +- five services' `boot.toml` digests, from #1494's reseal over the merge + (`e6b108d1`). + +The closure is now 27 packages and 406 files. Against Kore `e6b108d1`, only +kore-rt's `futex.rs` and `thread_controller.rs` differ. No exported manifest +changed, so `native/Cargo.lock` did not. + +It then moved to Kore `5c09f155b03682e95f24f924cdd65188fc9ebc60`, the merge +of #1494's next round into this change. That round lets the kernel observe a +PCI function's live state and its BARs' shapes, and puts the name a program +was run by in its process page. libkore#32 exported it first, at Kore +`dfece644`. The closure is now 27 packages and 407 files. Against Kore +`4b236fbe`, only kore-rt's `futex.rs` and `thread_controller.rs` differ. +Against libkore#32's export, five services' `boot.toml` digests differ as +well, from #1494's reseals that followed it. No exported manifest changed, +so `native/Cargo.lock` did not. + +It then moved to Kore `7c0be6e84ac2e8ee67591b44c0f42d46ab596f83`, the merge +of #1494's next round into this change. That round has the kernel offer a +function to drivers only on positive evidence and name each claim's holder, +which kore-abi's device state and its new device classes carry. The closure +is now 27 packages and 408 files. Against Kore `fe9603e3`, only kore-rt's +`futex.rs` and `thread_controller.rs` differ, and `services/swap/boot.toml`: +swap links kore-rt's futex, so its digests are this change's reseal over +the ioctl round (`b04325e9`). No exported manifest changed, so +`native/Cargo.lock` did not. diff --git a/docs/runtime.md b/docs/runtime.md index fd1d88b..0c80f91 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -131,10 +131,13 @@ names on Kore targets; it never interposes host pthreads. The first create starts one process-lifetime controller. It owns all native join capabilities; portable handles contain identifiers, never arena slot authority. -Calls use bounded shared records and paced parks, including a 1 ms controller -scan while idle. They must run on native threads, not executor callbacks that -owe other futures progress. Capacity depends on native bands and the controller's -capability arena; ordinary slot exhaustion is refused before spending a band. +Calls use bounded shared records and futex parks: the controller parks until a +request, a thread's end or a relayed wake moves its word, and a waiting caller +parks until the controller wakes it. Only a thread the controller cannot wake +parks in bounded 1 ms slices. Calls must run on native threads, not executor +callbacks that owe other futures progress. Capacity depends on native bands and +the controller's capability arena; ordinary slot exhaustion is refused before +spending a band. Create uses the installed process allocator for its handle and callback context. On error the output is unchanged and the argument is not retained. On success, @@ -193,8 +196,22 @@ this does not implement dynamic DSO unloading or Rust/C++ stack unwinding. `kore_thread_testcancel`, foreign join and foreign futex waits. Disabled targets retain pending requests. `setcancelstate` enables/disables delivery; `setcanceltype` accepts deferred and refuses asynchronous with ENOTSUP, preserving -output on failure. Cancellation-aware waits use bounded 1 ms native parks and -one monotonic deadline. A canceled joiner leaves its target joinable. +output on failure. A canceled joiner leaves its target joinable. + +A request wakes a target parked at a cancellation point; no cancellation-aware +wait polls for one. The canceller stores the request, then wakes the target's +thread (`futex::wake_thread`), relayed through the controller, the one owner of +a send into each managed thread's doorbell. The target's wait reads the request +again once it is registered (`futex::wait_unless`), and the wake reads that +registration, so one of the two always sees the other. Only a target the +controller cannot wake waits in bounded 1 ms slices against one monotonic +deadline. A thread the controller founds with a doorbell is never one: it +runs its callback only once its founding is published, after the controller +can kick it. A thread whose wake cannot reach the controller's threads, because +it neither started the controller nor was created through it, is refused with +EAGAIN and records no request, as create refuses it: Kore's capability model +grants no wake without a send, and a request a parked target never sees would +be a silent loss. On explicit `kore_thread_exit` or cancellation, caller-owned cleanup nodes run LIFO, then compiler TLS destructors, then pthread key destructors. Normal callback @@ -237,11 +254,14 @@ there is no second wait registry. Use `wait_async` in Rust executor tasks. C exposes `kore_futex_available`, `kore_futex_wait`, `kore_futex_wake` and `kore_futex_wake_all` in ``; Zig exposes `kore.futex.*`. The foreign wait blocks a native thread and must not run inside an executor callback. -For managed threads it is also a deferred cancellation point. Independent -controller-created threads are not endowed with doorbells, so their native futex -waits report `Unsupported`; this adapter does not create a peer wake mesh. -Foreign join uses the controller's deadline-park helper and works on these -threads. Cancellation is checked even when a foreign futex wait is unsupported. +For managed threads it is also a deferred cancellation point, which a +cancellation request wakes. Controller-created threads are founded with +doorbells when the thread that started the controller holds a channel-space +grant, and park; a wake one of them cannot deliver directly is relayed through +the controller, which holds a send into each. Without the grant they have none: +their native futex waits report `Unsupported`, and foreign join takes the +deadline park, which needs no wake. Cancellation is checked even when a foreign +futex wait is unsupported. Timeouts are relative nanoseconds; zero is a deadline of zero and `KORE_FUTEX_INFINITE` requests an unbounded wait. Retry, spurious wake, timeout, unsupported context and invalid pointer shape remain distinct outcomes. diff --git a/extern/c/include/kore.h b/extern/c/include/kore.h index a9089a0..ba27a6d 100644 --- a/extern/c/include/kore.h +++ b/extern/c/include/kore.h @@ -690,7 +690,8 @@ uint32_t kore_futex_available(void); * all waits/wakes. All concurrent accesses must be compatible 32-bit atomic * operations (C atomic builtins or Zig atomics); never mix ordinary accesses * or differently sized atomics. Managed-thread waits are cancellation points: - * they may exit without stack unwinding, as specified by kore_thread_testcancel. + * they may exit without stack unwinding, as specified by kore_thread_testcancel, + * and a cancellation requested while the thread waits wakes it to act. * Publish the condition with a sequentially consistent atomic store BEFORE waking. * No address-space sharing is implied. */ @@ -935,7 +936,12 @@ int32_t kore_thread_detach(struct KoreThread *thread); /** * Request deferred cancellation. The target acts at testcancel, join or foreign - * futex wait. A disabled target retains the request. No asynchronous kill occurs. + * futex wait; a target waiting at one is woken to act. A disabled target + * retains the request. No asynchronous kill occurs. EAGAIN when no wake the + * calling thread sends can reach the target, because it neither started the + * controller nor was created through it (create refuses it too). Then no + * request is recorded, since a target parked at a cancellation point would + * never see it. * # Safety * thread is a live handle. Borrowing cancel calls may run concurrently with one * another, but must finish before a consuming join/detach can free the handle. diff --git a/native/crates/abi/src/device.rs b/native/crates/abi/src/device.rs index 463489e..f739f5e 100644 --- a/native/crates/abi/src/device.rs +++ b/native/crates/abi/src/device.rs @@ -20,9 +20,11 @@ //! route) — and teardown is an ordinary `Revoke` of the leaf. mod census; +mod class; mod i2c_admission; mod state; pub use census::*; +pub use class::*; pub use i2c_admission::*; pub use state::*; diff --git a/native/crates/abi/src/device/class.rs b/native/crates/abi/src/device/class.rs new file mode 100644 index 0000000..e43aa63 --- /dev/null +++ b/native/crates/abi/src/device/class.rs @@ -0,0 +1,153 @@ +//! What a PCI function's **class code** says it is to a driver. +//! +//! A function is offered to drivers on positive evidence only. The class +//! code is the one identification every vendor's function carries, so it is +//! the evidence asked first, and it is asked in one direction: a class named +//! here as a peripheral admits a function; no class, and no absence of one, +//! proves a function safe to hand out. Everything this table does not name +//! is [`Platform`](PciClassRole::Platform), the functions a machine keeps +//! for itself: its management engine, flash and SMBus controllers, power +//! and thermal telemetry, shared memory, and whatever a vendor adds next. +//! Those differ on every platform and their classes are mostly the +//! catch-all "other" subclasses, so they are not enumerated: they are what +//! is left. A new or unknown function is therefore kept, never offered. +//! +//! The kernel decides ([`DeviceRecord::DRIVER_ENDPOINT`]); a reader of a +//! census row uses the same table to say *why* a function is withheld. +//! Codes are the PCI-SIG's (PCI Code and ID Assignment Specification, +//! "Class Codes"). +//! +//! [`DeviceRecord::DRIVER_ENDPOINT`]: super::DeviceRecord::DRIVER_ENDPOINT + +/// What a class code alone admits a function as. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PciClassRole { + /// Bus fabric: a bridge of any kind (`06`), or an IOMMU (`08/06`). It + /// carries other functions' traffic and no driver takes it. + Fabric, + /// A display controller (`03`), or the legacy VGA-compatible spelling + /// (`00/01`). A peripheral, but one firmware may still be scanning out + /// of: the kernel also requires proof of its boot ownership. + Display, + /// A device with a standard programming model that a driver is written + /// against, and that a machine keeps running without. + Peripheral, + /// Anything else: a function of the platform itself. Not offered. + Platform, +} + +impl PciClassRole { + /// The role `class`/`subclass`/`progif` admit. + pub const fn of(class: u8, subclass: u8, progif: u8) -> Self { + match (class, subclass) { + (0x06, _) | (0x08, 0x06) => Self::Fabric, + (0x03, _) | (0x00, 0x01) => Self::Display, + // Mass storage, network, multimedia (audio, video, telephony), + // input, wireless and processing accelerators: every subclass + // is a device someone attaches or a driver brings up, including + // the vendor-specific "other" ones (a Wi-Fi radio is `02/80`). + (0x01 | 0x02 | 0x04 | 0x09 | 0x0d | 0x12, _) => Self::Peripheral, + // A UART with a named register model: 16450 through 16950. + // Programming interface 0 ("generic XT") names none, and is + // what a platform stamps on functions that are not UARTs at + // all (a sensor hub, a management engine's serial redirection). + (0x07, 0x00) if matches!(progif, 0x01..=0x06) => Self::Peripheral, + // An SD host controller: the one base-system peripheral that is + // a slot for removable media rather than part of the machine. + (0x08, 0x05) => Self::Peripheral, + // Host controllers of the external peripheral buses: IEEE 1394, + // USB (every interface, USB4 included), Fibre Channel and + // InfiniBand. SMBus (`0c/05`), IPMI (`0c/08`), I3C (`0c/0a`) + // and "other" (`0c/80`: a platform's I2C, SPI and flash + // controllers) are the machine's own buses. + (0x0c, 0x00 | 0x03 | 0x04 | 0x07) => Self::Peripheral, + _ => Self::Platform, + } + } +} + +#[cfg(test)] +mod tests { + use super::PciClassRole::{self, Display, Fabric, Peripheral, Platform}; + + #[test] + fn a_class_admits_a_peripheral_and_everything_unnamed_is_the_platforms() { + for (class, subclass, progif, role, what) in [ + (0x06, 0x00, 0x00, Fabric, "host bridge"), + (0x06, 0x01, 0x00, Fabric, "ISA/LPC bridge"), + (0x06, 0x04, 0x00, Fabric, "PCI-to-PCI bridge"), + (0x08, 0x06, 0x00, Fabric, "IOMMU"), + (0x03, 0x00, 0x00, Display, "VGA controller"), + (0x03, 0x80, 0x00, Display, "display, other"), + (0x00, 0x01, 0x00, Display, "legacy VGA-compatible"), + (0x01, 0x08, 0x02, Peripheral, "NVMe"), + (0x01, 0x06, 0x01, Peripheral, "AHCI"), + (0x02, 0x00, 0x00, Peripheral, "Ethernet"), + (0x02, 0x80, 0x00, Peripheral, "Wi-Fi radio"), + (0x04, 0x03, 0x00, Peripheral, "HD Audio"), + (0x09, 0x00, 0x00, Peripheral, "keyboard"), + (0x0d, 0x11, 0x00, Peripheral, "Bluetooth"), + (0x12, 0x00, 0x00, Peripheral, "neural accelerator"), + (0x07, 0x00, 0x02, Peripheral, "16550 UART"), + (0x08, 0x05, 0x01, Peripheral, "SD host controller"), + (0x0c, 0x03, 0x30, Peripheral, "xHCI"), + (0x0c, 0x03, 0x40, Peripheral, "USB4 host interface"), + // What a machine keeps. The identities are the ones one Lunar + // Lake laptop and one AMD laptop enumerate. + (0x05, 0x00, 0x00, Platform, "PMC shared SRAM"), + (0x07, 0x80, 0x00, Platform, "management engine HECI"), + ( + 0x07, + 0x00, + 0x00, + Platform, + "sensor hub stamped generic serial", + ), + ( + 0x0c, + 0x80, + 0x00, + Platform, + "SPI flash or LPSS I2C controller", + ), + (0x0c, 0x05, 0x00, Platform, "SMBus"), + (0x0c, 0x0a, 0x00, Platform, "I3C"), + (0x11, 0x80, 0x00, Platform, "thermal and power tuning"), + (0x10, 0x80, 0x00, Platform, "platform security processor"), + (0x13, 0x00, 0x00, Platform, "instrumentation"), + (0x08, 0x80, 0x00, Platform, "system peripheral, other"), + (0x00, 0x00, 0x00, Platform, "unclassified"), + (0x00, 0xff, 0x00, Platform, "unclassified, vendor"), + (0xff, 0x00, 0x00, Platform, "unassigned"), + ] { + assert_eq!( + PciClassRole::of(class, subclass, progif), + role, + "{what} ({class:02x}/{subclass:02x}.{progif:02x})" + ); + } + } + + /// The table only ever admits by naming: no class outside the ones it + /// names is a peripheral, whatever its subclass or interface. + #[test] + fn no_unnamed_class_is_ever_a_peripheral() { + for class in 0..=u8::MAX { + if matches!( + class, + 0x01 | 0x02 | 0x04 | 0x07 | 0x08 | 0x09 | 0x0c | 0x0d | 0x12 + ) { + continue; + } + for subclass in 0..=u8::MAX { + for progif in [0x00, 0x01, 0x02, 0x30, 0x80, 0xff] { + assert_ne!( + PciClassRole::of(class, subclass, progif), + Peripheral, + "{class:02x}/{subclass:02x}.{progif:02x}" + ); + } + } + } + } +} diff --git a/native/crates/abi/src/device/state.rs b/native/crates/abi/src/device/state.rs index 5dab25c..dacf681 100644 --- a/native/crates/abi/src/device/state.rs +++ b/native/crates/abi/src/device/state.rs @@ -3,7 +3,7 @@ //! //! The census is boot-frozen identity. This is what changes after boot, and //! what a driver writer asks before touching a device: whether a driver -//! claimed the function, which founded row drives its DMA, which BARs exist, +//! claimed the function, which founded row holds that claim, which BARs exist, //! which of them a driver may map and which are mapped now, and how the //! function can interrupt. A BAR is its index, its size and the length a //! driver may map; **no address crosses** (res/DEVICE_SERVICE.md §1.1), so @@ -33,7 +33,7 @@ use super::census::{DeviceCensus, DeviceCensusPage, MAX_DEVICE_CENSUS_ROWS}; pub const DEVICE_BARS: usize = 6; /// Every BAR's bit, for the three per-BAR bitmaps of a [`DeviceState`]. pub const DEVICE_BAR_MASK: u8 = (1 << DEVICE_BARS) - 1; -/// `DeviceState::driver` when no founded row drives the function's DMA. +/// `DeviceState::driver` when no running founded row holds the function. pub const DEVICE_ROW_NONE: u8 = 0xFF; /// States per vended page: 8 header bytes + 128 = 136 bytes, the census /// page's size, so a page rides the monitor and exec reply wires as one. @@ -77,11 +77,12 @@ pub const fn device_bar_of(operand: u16) -> Option<(u16, u8)> { pub struct DeviceState { /// [`Self::PRESENT`] and the bits below it. pub flags: u8, - /// The founded row (the founding record's index) whose task activated - /// DMA on this function, or [`DEVICE_ROW_NONE`]. The kernel's own - /// shutdown ledger, so it names who drives the device rather than who - /// claimed it: a function a row claimed and delegated reads the row it - /// was delegated to, and a driver that never activates DMA reads none. + /// The founded row (the founding record's index) whose task holds this + /// function's claim, or [`DEVICE_ROW_NONE`]. The kernel records whom a + /// claim's grant is sealed for when the claim commits, and follows that + /// exact grant if it is carried on, so this names the claimant whether + /// or not it does DMA. A claim whose task is not a running founded row + /// (it ended, or it is a program) reads none: the claim still stands. pub driver: u8, /// Bit `n`: BAR `n` exists as memory the kernel resolved. An I/O BAR is /// not one: no driver may reach port space through a BAR. diff --git a/native/crates/rt/src/futex.rs b/native/crates/rt/src/futex.rs index bebb188..73d9168 100644 --- a/native/crates/rt/src/futex.rs +++ b/native/crates/rt/src/futex.rs @@ -122,6 +122,30 @@ //! simply that async code uses the async form, and it is a correctness rule //! rather than a stylistic one. //! +//! # Waking a thread out of its wait +//! +//! A futex wakes the threads that wait on a word. Some waits must also end +//! for a reason that is not the word: a C thread's cancellation has to end +//! the `pthread_join` or `kore_futex_wait` it is parked in. [`wait_unless`] +//! is that wait — [`wait`] with a caller condition, `stop`, read *after* the +//! waiter registers, as the word is — and [`wake_thread`] is its wake: a kick +//! into one thread's doorbell rather than to the waiters of one word. +//! +//! The condition's writer stores it (`SeqCst`), then wakes the thread. That +//! is mechanism 2 again, with the condition in the word's place. The writer +//! stores, then reads the waiter's registration — [`wake_thread`] reads it +//! before it kicks or relays, and a hub's sweep reads it again before it +//! kicks — and the waiter registers, then reads the condition, so one of +//! the two sees the other. A writer that misses the registration is one whose +//! store the waiter's read follows: the waiter does not park. A writer that +//! sees it kicks a post that already stands, since the waiter armed before it +//! registered, and that kick resolves the park the waiter is in or about to +//! enter. A wait that can be ended therefore carries no timeout it does not +//! need: nothing re-reads, on a slice, a condition its writer announces. +//! +//! A stoppable wait parks on its doorbell even for a word in a noted window: +//! the kernel's word wait ends only on its word, and a stop arrives as a kick. +//! //! # What a caller must still do //! //! **Re-check in a loop.** A return from [`wait`] is never a promise that the @@ -306,9 +330,9 @@ const NOT_WAITING: usize = 0; /// speed while reporting that it was politely waiting. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum WaitOutcome { - /// The word did not hold `expected` — the caller's condition had already - /// changed, and no park was attempted. The common fast path under - /// contention, and not a failure. + /// The word did not hold `expected`, or a [`wait_unless`] stop held — the + /// caller's condition had already changed, and no park was attempted. + /// The common fast path under contention, and not a failure. Retry, /// The thread parked and was woken. **Says nothing about the word**: it /// may be a matching [`wake`], an unrelated kick on the shared doorbell, @@ -399,7 +423,32 @@ pub(crate) const OWN_KICK: u16 = RELAY + 1; /// `std`'s locks will be built from, the same disposition /// [`block_on`](crate::block::block_on) takes. pub fn wait(word: &AtomicU32, expected: u32, timeout: Option) -> WaitOutcome { - wait_inner(word, expected, timeout, true, LOCK_SPIN) + wait_inner(word, expected, timeout, true, LOCK_SPIN, None) +} + +/// [`wait`] that another thread can also end by publishing `stop` and then +/// waking **this thread** ([`wake_thread`]) — the wait a cancellation point +/// is, whose request is a flag rather than a word. +/// +/// `stop` is read after this waiter registers, as the word is, and the park +/// follows only if the word still holds `expected` and `stop` is still +/// `false`; otherwise this returns [`Retry`](WaitOutcome::Retry), having +/// parked nothing. The writer of the condition stores it with `SeqCst` and +/// then calls [`wake_thread`] with this thread's band, so neither can miss +/// the other (module docs, *Waking a thread out of its wait*): no timeout is +/// needed to notice a stop. +/// +/// The park is always the doorbell's, even for a word in a noted window — +/// the kernel's word wait cannot be ended by a kick. Everything [`wait`] +/// says about spurious wakes holds: re-check the word *and* `stop` after any +/// return. Not from inside a future, for [`wait`]'s reasons. +pub fn wait_unless( + word: &AtomicU32, + expected: u32, + timeout: Option, + stop: impl Fn() -> bool, +) -> WaitOutcome { + wait_inner(word, expected, timeout, true, LOCK_SPIN, Some(&stop)) } /// [`wait`] with the **barrier's** longer pre-park monitored budget. @@ -422,7 +471,7 @@ pub(crate) fn wait_barrier( expected: u32, timeout: Option, ) -> WaitOutcome { - wait_inner(word, expected, timeout, true, BARRIER_SPIN) + wait_inner(word, expected, timeout, true, BARRIER_SPIN, None) } /// [`wait_barrier`] with **no** jitter-window spin: straight to the park. @@ -432,7 +481,7 @@ pub(crate) fn wait_barrier( /// Nothing co-resident is about to store the word, so the spin would only /// occupy the core; counted like the barrier wait it stands in for. pub(crate) fn wait_idle(word: &AtomicU32, expected: u32, timeout: Option) -> WaitOutcome { - wait_inner(word, expected, timeout, true, NO_SPIN) + wait_inner(word, expected, timeout, true, NO_SPIN, None) } /// [`wait`] without the [`stats`] accounting — for the star's hub, @@ -443,7 +492,7 @@ pub(crate) fn wait_quiet( expected: u32, timeout: Option, ) -> WaitOutcome { - wait_inner(word, expected, timeout, false, LOCK_SPIN) + wait_inner(word, expected, timeout, false, LOCK_SPIN, None) } fn wait_inner( @@ -452,6 +501,7 @@ fn wait_inner( timeout: Option, counted: bool, spin: SpinBudget, + stop: Option<&dyn Fn() -> bool>, ) -> WaitOutcome { let Some(me) = current_thread() else { return WaitOutcome::Unsupported; @@ -463,35 +513,19 @@ fn wait_inner( // named to the kernel by that authority and parks on the word itself // (the kernel address-wait); every other word keeps the doorbell park. // Decided once, before the registry entry, because the entry records - // the decision for wakers. - let route = crate::cap::mapped::locate(word as *const AtomicU32 as *const u8); - - // ── 1 · Arm ───────────────────────────────────────────────────────── - // The standing post must exist before this thread is discoverable as a - // waiter, or a notice sent in the window between registering and parking - // would find no post and be dropped (`WouldBlock`, on the floor). Arming - // first makes "too early" indistinguishable from "on time". A word park - // needs no post: the mapping is its wait authority, which is what lets a - // doorbell-less worker wait here at all. - if route.is_none() && !imp::arm() { - return WaitOutcome::Unsupported; - } - - // ── 2 · Register, then check ──────────────────────────────────────── - // This store must be visible before the load below, and `SeqCst` on both - // is what forbids the store-load reordering that would let this thread - // and a waker miss each other (module docs, mechanism 2). - let slot = match route { - Some(_) => claim_wait_on_word(me, addr), - None => claim_wait(me, addr), + // the decision for wakers. A wait with a `stop` keeps the doorbell + // whatever the word: its stop arrives as a kick, which a word park + // never sees (`wait_unless`). + let route = match stop { + Some(_) => None, + None => crate::cap::mapped::locate(word as *const AtomicU32 as *const u8), }; - let Some(slot) = slot else { - return WaitOutcome::Unsupported; + + // ── 1, 2 · Arm, register, then check ──────────────────────────────── + let slot = match stand(me, word, expected, route.is_some(), stop, imp::arm) { + Ok(slot) => slot, + Err(outcome) => return outcome, }; - let _slot = WaitSlot { thread: me, slot }; - if word.load(Ordering::SeqCst) != expected { - return WaitOutcome::Retry; - } // ── 3 · Spin on the line, briefly ─────────────────────────────────── if let Some(end) = spin_briefly(word, expected, spin) { @@ -499,7 +533,7 @@ fn wait_inner( } // ── 4 · Park ──────────────────────────────────────────────────────── - // `_slot` releases on the way out: a waiter that is awake must not stay + // `slot` releases on the way out: a waiter that is awake must not stay // findable, or the next `wake` spends its kick on nobody and reports // having woken someone. if counted { @@ -515,7 +549,7 @@ fn wait_inner( // stored — SeqCst on both sides, one of us sees the other), // then arm the doorbell. Without the demotion a word-mode // waiter parked on its doorbell is unreachable by any waker. - demote_to_bell(me, slot, addr) + demote_to_bell(me, slot.slot, addr) && word.load(Ordering::SeqCst) == expected && imp::arm() }) @@ -532,6 +566,57 @@ fn wait_inner( outcome } +/// Steps 1 and 2 of every wait — **arm, register, then check** — with the +/// arm injected, so the host suite holds the order to account against the +/// registry the target uses. +/// +/// `Ok` is a waiter that must park: registered on `word` (in word mode when +/// `on_word`), its post standing unless it parks on the word, and the word +/// still `expected` and `stop` still `false`, both read after the +/// registration. `Err` is what to return instead, having parked nothing; the +/// registration is released with it. +fn stand( + me: usize, + word: &AtomicU32, + expected: u32, + on_word: bool, + stop: Option<&dyn Fn() -> bool>, + arm: impl FnOnce() -> bool, +) -> Result { + let addr = address_of(word); + + // ── 1 · Arm ───────────────────────────────────────────────────────── + // The standing post must exist before this thread is discoverable as a + // waiter, or a notice sent in the window between registering and parking + // would find no post and be dropped (`WouldBlock`, on the floor). Arming + // first makes "too early" indistinguishable from "on time". A word park + // needs no post: the mapping is its wait authority, which is what lets a + // doorbell-less worker wait here at all. + if !on_word && !arm() { + return Err(WaitOutcome::Unsupported); + } + + // ── 2 · Register, then check ──────────────────────────────────────── + // This store must be visible before the loads below, and `SeqCst` on all + // of them is what forbids the store-load reordering that would let this + // thread and a waker miss each other (module docs, mechanism 2). `stop` + // is the same argument with the caller's condition in the word's place: + // its writer stores, then reads this registration before it kicks. + let slot = if on_word { + claim_wait_on_word(me, addr) + } else { + claim_wait(me, addr) + }; + let Some(slot) = slot else { + return Err(WaitOutcome::Unsupported); + }; + let slot = WaitSlot { thread: me, slot }; + if word.load(Ordering::SeqCst) != expected || stop.is_some_and(|stop| stop()) { + return Err(WaitOutcome::Retry); + } + Ok(slot) +} + /// Cumulative process-wide wait/wake accounting — the terminal's window /// into behavior no serial can show (`res/SPMD.md` §11's doctrine). /// Relaxed counters: telemetry, never synchronization. `rode` is the @@ -952,6 +1037,52 @@ pub fn wake_all(word: &AtomicU32) -> usize { woken } +/// Wake **thread `band`** rather than the waiters of a word: kick its +/// doorbell, so a park it is in or about to enter returns. The wake a +/// [`wait_unless`] waiter's `stop` is announced by. +/// +/// Store the condition first, with `SeqCst`. This reads the target's +/// registration before it kicks, so a target that registers after that read +/// reads the condition after it (module docs, *Waking a thread out of its +/// wait*); a target with no doorbell waiter registered is running, and is not +/// kicked. A thread waking itself kicks nothing either: it is not parked. +/// +/// The kick goes directly when this thread holds a send into the target's +/// doorbell, and otherwise through this thread's relay: the hub it adopted +/// (the [`thread_controller`](crate::thread_controller) of the threads it +/// founds), whose sweep kicks every thread with a doorbell waiter +/// registered, or the star's hub. `true` when a kick or a relay went out. +/// `false` when no kick was needed or this thread has no route to the +/// target, which a caller that must reach it checks before it publishes. +pub fn wake_thread(band: usize) -> bool { + wake_thread_with(current_thread(), band, imp::kick, relay_missed) +} + +/// [`wake_thread`]'s selection, with the kick and the relay injected so the +/// host suite can hold it to account. +fn wake_thread_with( + me: Option, + band: usize, + kick: impl FnOnce(usize) -> bool, + relay: impl FnOnce() -> bool, +) -> bool { + if band >= MAX_THREADS as usize || me == Some(band) || !waits_on_bell(band) { + return false; + } + if kick(band) { + STAT_KICKS.fetch_add(1, Ordering::Relaxed); + return true; + } + // Refused: no send into that doorbell, or its post already spent — the + // relay covers both, as it does for `wake`. + STAT_MISSED.fetch_add(1, Ordering::Relaxed); + if relay() { + STAT_RELAYS.fetch_add(1, Ordering::Relaxed); + return true; + } + false +} + /// A **relay hub that is also a waiter**: the thread that founds a group of /// threads and holds a send into each of their doorbells — the /// [`thread_controller`](crate::thread_controller) — parked on its @@ -1020,6 +1151,13 @@ pub(crate) fn adopt_hub(hub: &'static Hub) -> bool { imp::adopt_hub(hub) } +/// Whether the hub this thread relays through is `hub`: a wake it cannot +/// deliver itself goes there, and from there to every thread `hub`'s thread +/// can kick. +pub(crate) fn relays_through(hub: &Hub) -> bool { + imp::hub().is_some_and(|adopted| core::ptr::eq(adopted, hub)) +} + /// Record that this thread holds `send`, a send into band `peer`'s /// doorbell its founding did not stamp — the channel a controller mints for /// each child it founds, or a starter for the controller it founds. From @@ -1080,7 +1218,6 @@ fn sweep_with(me: usize, mut kick: impl FnMut(usize) -> bool) -> usize { } /// Whether `thread` has any waiter a kick reaches — a doorbell-mode entry. -#[cfg_attr(not(target_os = "none"), allow(dead_code))] fn waits_on_bell(thread: usize) -> bool { WAITING_ON[thread].iter().any(|cell| { let entry = cell.load(Ordering::SeqCst); @@ -1114,16 +1251,10 @@ pub(crate) fn wait_or_end( return WaitOutcome::Unsupported; }; // Arm, register, check — `wait`'s order and for its reasons. - if !imp::arm() { - return WaitOutcome::Unsupported; - } - let Some(slot) = claim_wait(me, address_of(word)) else { - return WaitOutcome::Unsupported; + let _slot = match stand(me, word, expected, false, None, imp::arm) { + Ok(slot) => slot, + Err(outcome) => return outcome, }; - let _slot = WaitSlot { thread: me, slot }; - if word.load(Ordering::SeqCst) != expected { - return WaitOutcome::Retry; - } imp::park_or_end(timeout, task, report) } @@ -1158,7 +1289,8 @@ fn address_of(word: &AtomicU32) -> usize { word as *const AtomicU32 as usize } -/// This thread's band index — its identity in [`WAITING_ON`]. +/// This thread's band index — its identity in the wait registry +/// (`WAITING_ON`). /// /// The allocator's shard index, deliberately: it is already stamped at /// anchor, already proved injective across live threads (the `MAX_THREADS <= @@ -1168,7 +1300,12 @@ fn address_of(word: &AtomicU32) -> usize { /// The bound is re-checked rather than assumed: an out-of-band index would /// index the registry out of range, and a thread that cannot name itself must /// refuse to wait rather than borrow a neighbour's entry. -pub(crate) fn current_thread() -> Option { +/// +/// Also the name [`wake_thread`] reaches this thread by, which a thread +/// records where its wakers can read it. An identity, not an authority: a +/// waker still needs a send into the band's doorbell, or a relay that holds +/// one. `None` off metal, and on a thread outside the band. +pub fn current_thread() -> Option { let index = imp::thread_index()?; (index < MAX_THREADS as usize).then_some(index) } @@ -2014,9 +2151,15 @@ mod tests { claim_wait(9, address_of(&a)).expect("a doorbell waiter"); claim_wait_on_word(10, address_of(&b)).expect("a word waiter"); let mut kicked = std::vec::Vec::new(); + // The sweep scans every row, and tests in parallel register rows of + // their own: only this test's rows are delivered and recorded, and + // another's is a refused kick, which is not counted. let n = sweep_with(8, |thread| { - kicked.push(thread); - true + let ours = (8..=11).contains(&thread); + if ours { + kicked.push(thread); + } + ours }); assert_eq!(kicked, [9], "the sweep kicked a thread no kick can wake"); assert_eq!(n, 1); @@ -2027,6 +2170,202 @@ mod tests { } } + #[test] + fn a_stop_is_read_only_once_the_waiter_is_registered() { + // The order `wait_unless` rests on: arm, register, and only then read + // the stop. Its writer stores the stop and then reads the registry, so + // it misses this waiter only when this waiter's read follows its store. + // Each step asserts that the one before it already happened. + use core::cell::Cell; + const ROW: usize = 12; + reset_thread(ROW); + let word = AtomicU32::new(5); + let addr = address_of(&word); + let armed = Cell::new(false); + let arm = || { + assert!( + !thread_waits_on(ROW, addr), + "registered before the post stood" + ); + armed.set(true); + true + }; + let holds = Cell::new(true); + let read = Cell::new(0); + let stop = || { + assert!( + armed.get() && thread_waits_on(ROW, addr), + "the stop was read before the waiter registered" + ); + read.set(read.get() + 1); + holds.get() + }; + + // A stop that holds: nothing parks, and the registration goes too. + let out = stand(ROW, &word, 5, false, Some(&stop), arm); + assert_eq!(out.err(), Some(WaitOutcome::Retry)); + assert_eq!(read.get(), 1); + assert!(!thread_waits_on(ROW, addr), "a stopped waiter stayed found"); + + // One that does not: the waiter parks registered, and leaves with + // its slot. + holds.set(false); + armed.set(false); + let slot = stand(ROW, &word, 5, false, Some(&stop), arm) + .expect("a waiter whose stop does not hold parks"); + assert_eq!(read.get(), 2); + assert!(thread_waits_on(ROW, addr), "a parking waiter is not found"); + drop(slot); + assert!(!thread_waits_on(ROW, addr)); + + // An arm the kernel refuses registers nothing and reads nothing. + let out = stand(ROW, &word, 5, false, Some(&stop), || false); + assert_eq!(out.err(), Some(WaitOutcome::Unsupported)); + assert_eq!(read.get(), 2); + assert!(!thread_waits_on(ROW, addr)); + reset_thread(ROW); + } + + #[test] + fn a_stop_published_before_its_wake_is_never_missed() { + // Every point at which the stop's writer — store, then + // `wake_thread` — can run against a waiter's arm, registration, stop + // read and park. The waiter either reads the stop and parks nothing, + // or parks on a post the writer's kick consumed, which the kernel + // resolves at once. The simulated post is the kernel's rule: arming + // stands it, a kick consumes a standing one and drops otherwise, and + // a park on a consumed post returns. + use core::sync::atomic::AtomicBool; + const ROW: usize = 13; + #[derive(Clone, Copy, Debug, PartialEq)] + enum At { + BeforeArm, + BeforeRegistering, + BeforeStopRead, + // Just after the waiter read `false`: the window a stop read + // before the registration would leave open. + AfterStopRead, + BeforePark, + } + for at in [ + At::BeforeArm, + At::BeforeRegistering, + At::BeforeStopRead, + At::AfterStopRead, + At::BeforePark, + ] { + reset_thread(ROW); + let word = AtomicU32::new(0); + let stopped = AtomicBool::new(false); + let post = AtomicBool::new(false); + let writer = || { + stopped.store(true, Ordering::SeqCst); + let _ = wake_thread_with( + None, + ROW, + |band| { + assert_eq!(band, ROW); + post.compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + }, + || false, + ); + }; + if at == At::BeforeArm { + writer(); + } + let arm = || { + post.store(true, Ordering::SeqCst); + if at == At::BeforeRegistering { + writer(); + } + true + }; + let stop = || { + if at == At::BeforeStopRead { + writer(); + } + let stop = stopped.load(Ordering::SeqCst); + if at == At::AfterStopRead { + writer(); + } + stop + }; + match stand(ROW, &word, 0, false, Some(&stop), arm) { + Err(WaitOutcome::Retry) => { + assert!( + !matches!(at, At::AfterStopRead | At::BeforePark), + "{at:?}: a stop not yet written was read" + ); + } + Ok(slot) => { + assert!( + matches!(at, At::AfterStopRead | At::BeforePark), + "{at:?}: the stop went unread" + ); + if at == At::BeforePark { + writer(); + } + assert!( + !post.load(Ordering::SeqCst), + "the waiter parks on a post its stop's wake never consumed" + ); + drop(slot); + } + Err(other) => panic!("{at:?}: {other:?}"), + } + } + reset_thread(ROW); + } + + #[test] + fn a_thread_wake_kicks_only_a_registered_waiter_and_relays_a_refusal() { + const ROW: usize = 14; + reset_thread(ROW); + let word = AtomicU32::new(0); + let never = |_: usize| -> bool { panic!("kicked a thread no kick can wake") }; + let no_relay = || -> bool { panic!("relayed a wake that needed none") }; + // A running thread (nothing registered), a band past the registry, + // and the caller itself are not kicked. + let past = MAX_THREADS as usize; + assert!(!wake_thread_with(None, ROW, never, no_relay)); + assert!(!wake_thread_with(None, past, never, no_relay)); + let slot = claim_wait(ROW, address_of(&word)).expect("a free wait slot"); + let waiter = WaitSlot { thread: ROW, slot }; + assert!(!wake_thread_with(Some(ROW), ROW, never, no_relay)); + // A registered doorbell waiter: kicked directly, relayed when the + // kick refuses, and `false` only when neither went out. + assert!(wake_thread_with( + None, + ROW, + |band| { + assert_eq!(band, ROW); + true + }, + no_relay + )); + assert!(wake_thread_with(None, ROW, |_| false, || true)); + assert!(!wake_thread_with(None, ROW, |_| false, || false)); + drop(waiter); + // A word-parked waiter is the kernel's address-wait's to end. + let slot = claim_wait_on_word(ROW, address_of(&word)).expect("a free wait slot"); + let waiter = WaitSlot { thread: ROW, slot }; + assert!(!wake_thread_with(None, ROW, never, no_relay)); + drop(waiter); + reset_thread(ROW); + } + + #[test] + fn off_metal_a_stoppable_wait_refuses_before_reading_its_stop() { + reset_thread(7); + let word = AtomicU32::new(1); + assert_eq!( + wait_unless(&word, 1, None, || panic!("read a stop with no identity")), + WaitOutcome::Unsupported + ); + assert!(!wake_thread(0), "off metal no kick reaches a thread"); + } + #[test] fn the_own_kick_is_named_past_every_peer_and_the_relay() { // No peer arithmetic (`PEER_BASE + band`) and no relay lookup may diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index 764447a..dc4923a 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -27,6 +27,8 @@ //! the controller mints a doorbell, keeps the send (an adopted peer, withdrawn //! before the child's band can be reused), and founds the child holding it, a //! send into the controller, the starter's kick, the host's wake and the heap. +//! The child runs its callback once its founding is published, which follows +//! the adopted send, so the controller reaches every wait the callback makes. //! Children and the starter relay through the controller's hub, so a wake //! between two threads neither can kick directly still arrives: the controller //! sweeps ([`futex::sweep`]). @@ -42,9 +44,12 @@ //! A starter with no channel space founds a controller without a doorbell: //! neither it nor its children can park, and each of their waits takes the //! deadline park every thread can ([`pace::cede`]). A client the controller -//! cannot kick waits in bounded slices ([`UNREACHED_SLICE`]). A thread that +//! cannot kick waits in bounded slices ([`Controller::slice`]). A thread that //! cannot wake a parking controller is refused a create, which would -//! otherwise wait on a request nothing announced. +//! otherwise wait on a request nothing announced; one that cannot relay +//! through it cannot announce a request to its threads either +//! ([`Controller::wakes_threads`]), so a parked child is never left waiting +//! on a request nothing woke it for. //! //! [`thread::exit`]: crate::thread::exit //! [`pace::cede`]: crate::pace::cede @@ -88,7 +93,8 @@ const POLLING: u32 = 3; /// at before the controller parked (`pace::CEDE_NS`): a client the wake /// fabric cannot reach is no worse off than it was. Shorter would wake an /// unreachable joiner more often for nothing; longer would delay it. -const UNREACHED_SLICE: Duration = Duration::from_nanos(crate::pace::CEDE_NS); +/// [`Controller::slice`] says which waits take it. +pub const UNREACHED_SLICE: Duration = Duration::from_nanos(crate::pace::CEDE_NS); /// How long the controller rides its doorbell after a reap the kernel did /// not complete — a refused lifecycle observation or reclamation, which no @@ -142,7 +148,8 @@ fn empty(_: usize) -> usize { struct Record { state: AtomicU64, /// Bumped by the controller after each transition a client can wait - /// for; clients park on it. An event count: the state is the truth. + /// for; clients park on it, the child too until it is founded. An event + /// count: the state is the truth. event: AtomicU32, /// Set by the child's exit hook: it has stopped running its own code /// and is committing its end. @@ -203,7 +210,8 @@ pub struct Controller { #[cfg_attr(not(target_os = "none"), allow(dead_code))] starter: AtomicU32, /// The bands whose doorbells the controller can kick, a bit each: the - /// clients its wakes reach. A child's bit is set once it is founded and + /// clients its wakes reach. A child's bit is set before its founding is + /// published, which the child waits for before its callback runs, and /// cleared before its band can be reused. reach: AtomicU32, records: [Record; COUNT], @@ -336,6 +344,37 @@ impl Controller { .is_some_and(futex::reaches) } + /// Whether a wake this thread sends reaches the threads the controller + /// founded, so a request it announces to them by a wake — a cancellation, + /// stored and then [`futex::wake_thread`]ed — ends any park they are in. + /// + /// True when the controller parks and this thread both holds a send into + /// it and relays through its hub — the starter, or a thread it created — + /// since every thread the controller can kick is then one relay away. True + /// as well when it does not park, since neither do its threads, which need + /// no wake. False otherwise, and before the controller starts: a caller + /// refuses such a request, rather than leave it behind a park nothing ends. + pub fn wakes_threads(&self) -> bool { + match self.mode() { + None => false, + Some(PARKING) => self.wakes_controller() && futex::relays_through(&self.hub), + Some(_) => true, + } + } + + /// How long one wait of the calling thread may park: unbounded (`None`) + /// when the controller can kick it — it holds a send into the thread's + /// doorbell, so every wake it sends, and every wake relayed through it, + /// reaches the thread — and [`UNREACHED_SLICE`] when nothing would end + /// that park early. A thread the controller founds with a doorbell is + /// reached before its callback runs, so every wait of that callback is + /// unbounded from the first. + pub fn slice(&self) -> Option { + let reached = futex::current_thread() + .is_some_and(|band| self.reach.load(Ordering::SeqCst) & (1 << band) != 0); + (!reached).then_some(UNREACHED_SLICE) + } + /// Park until `record`'s state satisfies `done`, and return that state. fn settle(&self, record: &Record, done: impl Fn(u64) -> bool) -> u64 { loop { @@ -347,9 +386,7 @@ impl Controller { if done(state) { return state; } - let reached = futex::current_thread() - .is_some_and(|band| self.reach.load(Ordering::SeqCst) & (1 << band) != 0); - park(&record.event, seen, (!reached).then_some(UNREACHED_SLICE)); + park(&record.event, seen, self.slice()); } } @@ -486,7 +523,7 @@ fn pause() { } /// The managed thread's entry: announce its end however it comes, relay -/// through the controller, run the callback. +/// through the controller, wait to be founded, run the callback. fn child_entry(argument: usize) { // SAFETY: controller storage is static; record cannot be reused until join. let record = unsafe { &*(argument as *const Record) }; @@ -503,7 +540,14 @@ fn child_entry(argument: usize) { let _ = record.state.load(Ordering::Acquire); let start = unsafe { *record.start.get() }; // SAFETY: `home` names the static controller whose record this is. - let _ = futex::adopt_hub(&unsafe { &*(start.home as *const Controller) }.hub); + let home = unsafe { &*(start.home as *const Controller) }; + let _ = futex::adopt_hub(&home.hub); + // The callback runs once the founding is published, as `create` returns + // once it is. The controller publishes it only after it can kick this + // thread (`reach`), so no wait in the callback rides the unreached slice + // for want of a bit the controller had yet to set. A child that outran + // its founding parks here as its creator does, until the announcement. + home.settle(record, |state| state & MASK != REQUEST); record .value .store((start.call)(start.argument), Ordering::Release); @@ -1056,6 +1100,45 @@ mod tests { assert_eq!(record.state.load(Ordering::Acquire), 16); } #[test] + fn a_child_runs_its_callback_only_once_its_founding_is_published() { + // The child's half of its founding, with a host thread for the child + // and this one for the controller. The controller sets the child's + // reach bit before it publishes the founding, so a callback that ran + // first could find its own waits unreached, and bounded. + static SEEN: AtomicU64 = AtomicU64::new(u64::MAX); + fn callback(record: usize) -> usize { + // SAFETY: the record is leaked below, so it outlives this child. + let record = unsafe { &*(record as *const Record) }; + SEEN.store(record.state.load(Ordering::Acquire), Ordering::Release); + 7 + } + let controller: &'static Controller = + std::boxed::Box::leak(std::boxed::Box::new(Controller::new())); + let record = &controller.records[0]; + // SAFETY: no child runs yet; this thread is the BUILD claimant. + unsafe { + *record.start.get() = Start { + call: callback, + argument: record as *const Record as usize, + retire: None, + home: controller as *const Controller as usize, + }; + } + record.state.store(REQUEST, Ordering::Release); + std::thread::scope(|scope| { + let child = scope.spawn(|| child_entry(record as *const Record as usize)); + // Every chance for a child that does not wait to run first. + std::thread::sleep(Duration::from_millis(20)); + assert_eq!(record.returned.load(Ordering::Acquire), 0); + record.state.store(RUNNING, Ordering::Release); + record.announce(); + child.join().unwrap(); + }); + assert_eq!(SEEN.load(Ordering::Acquire), RUNNING); + assert_eq!(record.value.load(Ordering::Acquire), 7); + assert_eq!(record.returned.load(Ordering::Acquire), 1); + } + #[test] fn an_unstarted_controller_refuses_and_never_waits() { let controller: &'static Controller = std::boxed::Box::leak(std::boxed::Box::new(Controller::new())); @@ -1067,6 +1150,16 @@ mod tests { assert_eq!(controller.create(|_| 0, 0), Err(Error::Unavailable)); } #[test] + fn an_unstarted_controller_wakes_no_thread_and_bounds_every_wait() { + // Nothing relays through a controller that was never founded, so no + // request to its threads is announced by a wake; and a thread it + // cannot kick — off metal, every thread — parks one slice at a time. + let controller = Controller::new(); + assert!(!controller.wakes_threads()); + assert_eq!(controller.slice(), Some(UNREACHED_SLICE)); + assert_eq!(controller.started.load(Ordering::Acquire), IDLE); + } + #[test] fn records_own_their_interference_span() { assert_eq!(core::mem::align_of::(), 128); assert_eq!(core::mem::size_of::(), 128); diff --git a/native/provenance.json b/native/provenance.json index 6832942..f15b663 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "dfece6447cd331375a28b2bb290691cb8d5ab060", + "revision": "7c0be6e84ac2e8ee67591b44c0f42d46ab596f83", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -38,10 +38,11 @@ "crates/abi/src/clock.rs": "cce1c2839cba2377d3cc8696d60812f9195f071f422bd97bb4530ea0368972fc", "crates/abi/src/command.rs": "e994b1527c989cc106281aa127df9a54dc03e1c85f3b15c49ba70ad5e5d65721", "crates/abi/src/crash.rs": "f3c9e5a954e06743f6993b0a5eeb6937d2199c50af5fe95b6be446393ccf999b", - "crates/abi/src/device.rs": "06e8891917210870b64fa2ec3d2018067233ad333cf9d1bd13b3ce848fca97ed", + "crates/abi/src/device.rs": "9767bc5990be3e7e13c0542e1f96d738f8f0220d38dbf342742cb4dd3dd1a989", "crates/abi/src/device/census.rs": "6fb54ecaade94b116f41e045b098dc5926e8e1c75eeea33870dd4be01bd1ce23", + "crates/abi/src/device/class.rs": "dacb939e32c175cde259ec2b8ba05132b810583fe6af4615479a0843cc8cc2dc", "crates/abi/src/device/i2c_admission.rs": "1ea2fc04cdb0b15186d2f1d80bde53187860e2197263e225f330965d96e40876", - "crates/abi/src/device/state.rs": "8caf4b751b4cae54f3d78626440bd847a39f1138a4dbc6055d6e8a1a1576ac32", + "crates/abi/src/device/state.rs": "a67c721ce8280c77251eac6d63a3513c8503236e89d2e5ce9d1c9e9245440f9b", "crates/abi/src/execution.rs": "106d679a3f993c3f4e7ad4eb625d40f4399f5b9b50947cb0336ba44a1d6b51af", "crates/abi/src/founding.rs": "801b8e9771763ff9223c3537aabb5653713d1883d1df62ff673f43982aa3e802", "crates/abi/src/guest/aarch64/mod.rs": "1fd24be3fa6cbc344e9873a0e3e75fabd1b919326e477ec4ea4cf148c01b66f8", @@ -229,7 +230,7 @@ "crates/rt/src/executor/time.rs": "62f9ca9e5e17b395ec2b19a52fc6ed1bb53fd4acc3554cfca81eaf20e48fc272", "crates/rt/src/exit.rs": "3bf7e8ce6dac63b4ce365bdf2e743a9fd171851f1c5e40f8afbb4548877cef1b", "crates/rt/src/fiber_stack.rs": "48bf4fedba07ea2826615368e22882301ae4e0bbce0d893d0d7754d86bd02ce6", - "crates/rt/src/futex.rs": "df5622a1bd811d7d97ec20dc44a568dba3a2c995254fc30405a60d41371e7390", + "crates/rt/src/futex.rs": "a3947ed8217f8c006ec8ba69d8d13a02dc01fd2afc8d1fe80af07342db246cd0", "crates/rt/src/heap/mod.rs": "a9c39d3de56187a14d4633a229cbbb6e7ddb2c438d6f9f154b3b2551031b9f81", "crates/rt/src/heap/pool.rs": "67d42c6ffc7c7e176d21e97dd3179a3e4ff331f46dd457ed4d29914b1a404de8", "crates/rt/src/heap/shard.rs": "d97f91b3ecc71e3ec56fde0017f9d98cb99c5fbeeffe6ff79ddd153f5c6690fa", @@ -267,7 +268,7 @@ "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", "crates/rt/src/thread.rs": "06d0eefa92c46fe31392db8b10e891e008e95c3ee4162f9f4034f7dec7d9b35c", - "crates/rt/src/thread_controller.rs": "6ce535ded212d30b4838d15108cf9e9625b4f9e9fd83e0fecde51be833b1326d", + "crates/rt/src/thread_controller.rs": "ecd5a6f2853617e0f0d3190b33b69c305506a81de2f693145ed6347852f8cab4", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", @@ -320,13 +321,13 @@ "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", - "services/block/boot.toml": "99615c152634c9eb62532eb87c917aa3da99fc26d31a1409c4d269043ceace00", + "services/block/boot.toml": "087654a22a409b8cbf867705d54156f67fcda00200f099535d997d343f6a6f23", "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "58eda46ef6e6f45bba43df5b72bd7b1c5f45fb039957f9ddedb5d333f10b6b4d", + "services/dagfs/boot.toml": "181aac86795574d05213b43763e9df1adff23563867071186127c955c5732e30", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", @@ -369,7 +370,7 @@ "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", - "services/device/boot.toml": "4a0c84d22eb06e749e5d0803d51b659e3f8af81c52acb14491f58cec4246e752", + "services/device/boot.toml": "7e8294056f29a80924cdd412dafb0818a1d0f080d2b4f2ba819a9e2e32ca8430", "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", "services/device/src/service/mod.rs": "6038ba0b9375e3c6bbb25c49bf0849a5878a402f4a3a691dc9df1494849d252b", @@ -411,7 +412,7 @@ "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", - "services/mm/boot.toml": "8753f22f227455e5fff90674c7295a933b1d035ee6f3c27b7474ca9b99e149c4", + "services/mm/boot.toml": "cee05d68c32d3d0b5dd038ac5e42a3333f6ba84313810d02aab5005aa1a3a254", "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", @@ -423,7 +424,7 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "bba1d75a7f4418ffa066453fa0b753eebc5ae9d7e69ac5dca2b2a0ed694479eb", + "services/swap/boot.toml": "4a459ffb0365de3f761f768d393fcca67fe649a96591b4d8561a79bb46235a89", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml index 65fdc1b..761c672 100644 --- a/native/services/block/boot.toml +++ b/native/services/block/boot.toml @@ -12,5 +12,5 @@ chain = true endowment = "kore_svc_block::BlockEndowment" protocol = "kore_svc_block::BlockService" endows = ["DMA_POOL@1"] -digest_x86_64 = "blake3:d9f8829568372dbcdccc3028bd4b302850c814eb4d44233290c5bdb165c67cef" -digest_aarch64 = "blake3:d004acf08fb53d2d42656f0df24bf54318e3eda87ec79a6778c2e186b61ade79" +digest_x86_64 = "blake3:953c5f723e822fa6f3566527afaf7a07923f8d8deba73802840510d7b253a0c5" +digest_aarch64 = "blake3:6550c0af897f1c1950e93eae75e5a4bf29b9f975e7d0403b64669614f51fd3e5" diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index 37866c1..ad3081f 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -47,5 +47,5 @@ exec_client = true # `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 -digest_x86_64 = "blake3:dc2fe387db38e271eac4f5d93ac4b5d815060a82cee8cdc81c16bd363d6e9eed" -digest_aarch64 = "blake3:d27bf4cc470afee6a1b5c7c80127dc7e0f8366569b56b74b09c75719e45a3e23" +digest_x86_64 = "blake3:9c9406900318ce9e1d661fa26407472b498530e75cd46bfa8343355b0f341d69" +digest_aarch64 = "blake3:cf6d6be92acb3ad1ea814f43e44e8933e4abd38bb9140aeb52aa093eb5f979c5" diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml index 96c7f02..c5f9443 100644 --- a/native/services/device/boot.toml +++ b/native/services/device/boot.toml @@ -47,5 +47,5 @@ protocol = "kore_svc_device::DeviceBroker" # row's own claimant band. instances = 8 endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] -digest_x86_64 = "blake3:5a3ff2252bef814e79727f873c89de62565ab491f8065599ada4db348c829223" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:f00fd8b3aad6308541c51f785e3651dc793b60652a821ffee9aa7d79b49eb615" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:912b7d5675e1f60eacca9e346ad7f521a12b9e0c94a2a0c3d561c6d72785b291" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:613862ec39ea738e4ab0b379af45528fc80dd54ef6a8d953363e9a73c4b7ff7c" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml index b6412e0..4a0c382 100644 --- a/native/services/mm/boot.toml +++ b/native/services/mm/boot.toml @@ -54,5 +54,5 @@ supply_quota_bytes = 27917287424 # 26 GiB endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirror; asserted === PROFILE.roles shell_client = true exec_client = true -digest_x86_64 = "blake3:041a33515a21fff195c4cab320e9c8e64cbf775f04f01b7dbe40fa90be5a31a7" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:e17d59e57476b1db9554bc889640ffd3f02b6e2c98d509d93dea1c7efadb2a2a" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:3b6d33ce11bcfbda7eba8b730c42a6f0a4f119bbd0c71c8fa98e96aed7d93d07" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:b18acfa0b8a739b7afb6657506820dbaec7c6dd2bbbfc99ec143011a0ad83e14" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index 0a1028a..117cdb7 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:bc3ff6fbb123b3b44645e959346e320519b9ab5f43b5c999d0685ee9c7fed141" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:6495b930db0d4e4ca6ce41b0d19580ea1d816748358cc9c8bceef150ec0ccc30" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:c64857d1f04b3d2fcf5fcb547fc4a64eddd367df7e326341adb64e7798488dce" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:bfdbd6b573e15ad82edb567ef3cfab5112e298fbdccd284418b9c29809bdf4de" # res/aarch64/; make boot-digests-aarch64 diff --git a/tests/guest/thread_lifecycle.rs b/tests/guest/thread_lifecycle.rs index caad0e8..e447dfd 100644 --- a/tests/guest/thread_lifecycle.rs +++ b/tests/guest/thread_lifecycle.rs @@ -70,10 +70,12 @@ unsafe extern "C" fn worker(mode: *mut c_void) -> *mut c_void { 3 => { READY.store(1, Ordering::Release); loop { - // Every foreign wait entry checks cancellation. Independent - // controller workers have no doorbell: preserve Unsupported - // and park via the native deadline helper between retries. - // This witnesses cancellation delivery, not futex wake routing. + // Nothing wakes WORD: only the cancellation ends this wait. + // A worker with a doorbell parks until the request's wake, + // relayed through the controller, arrives. One founded + // without (a starter with no channel space) is refused the + // park: preserve Unsupported and take the native deadline + // park between retries, whose entry checks cancellation. if kore_futex_wait(WORD.as_ptr(), 0, KORE_FUTEX_INFINITE) == KoreFutexOutcome::Unsupported {