diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index c29b9984f..c1e628da6 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -81,7 +81,7 @@ jobs: - name: Record supervisor lifecycle traces env: LIFECYCLE_TRACE_DIR: ${{ github.workspace }}/lifecycle-traces - run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace --test coordinator_trace + run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test disposal_trace --test lifecycle_trace --test coordinator_trace - name: Validate traces and negative controls run: nix run .#check-tla -- --traces-only lifecycle-traces --negative-controls # Which model actions, projected states and transitions the recorded diff --git a/CHANGELOG.md b/CHANGELOG.md index eb8fa7fe4..880f55fe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,26 @@ (the selection trace missed it in 24 of 30 runs). New `lifecycle_trace::recorded_with` test helper. +- Keep user code off the shared deadline-service thread (the known 0.8 + limitation). The thread still decides expiry and returns credits, but the + wakers and payload destructors an expiry releases (a caller's waker, a + discarded scheduled emit, a displaced job result, a `done` closure and its + captures) now run on an elastic pool of disposal workers, one item at a + time, so a blocking destructor or waker no longer delays other actors' + deadlines or other callers of the same actor. A payload handed off still + waits for its owner's death point (R6). No cost + on requests that do not expire. New regression tests (tasks and threads), + a Loom case for the handed-off wake-up, an `InlineDisposal` negative + control in `DeferredReplies.tla`, and a new trace-validated model, + `Disposal.tla` (expiry and hand-off, per-item queueing, the pool's start + rule, idle count and cap, worker wake-ups after the transition, two-step + payload drops held until the death point), validated against traces that + record every hand-off and worker step (`disposal_trace`, with + `disposal.item`, `disposal.keep_alive` and `disposal.batch_serial` + failpoints; the last records a control trace from code with the + shared-batch bug, which must be rejected) and five negative controls. + Empty payloads are no longer handed off, and a due entry is dropped in + place unless it may hold an unread reply. - Regenerate the 11 planted-bug patches (04, 10-17, 19, 20) that no longer applied, each planting the same fault where the code now lives, and check in the required `model-sync` job that every planted patch still applies. diff --git a/ROADMAP.md b/ROADMAP.md index a07acdace..054bb0760 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -39,13 +39,13 @@ handlers and the existing default behavior. New policies are explicit and opt-in performance campaign against 0.7.3, paired four-hour soaks and the full Lifecycle models (evidence attached to the GitHub release). Ethrex and Ethlambda pin 0.8.0 in their draft PRs and run it on 24-hour canaries. -- **Deadline-service follow-up (known 0.8 limit):** separate user payload - disposal from shared deadline bookkeeping, with no default-path cost. - Acceptance: block a job result destructor in actor A while actor B's - unpolled token/raw deadline expires and returns its credit. Keep - foreign-waker progress assumptions explicit; never make exit wait for a - pre-exit destructor that can wait for the owner's callers. The current - scheduling limit is documented in the guide. +- **Deadline-service follow-up (done, unreleased):** the shared deadline + thread only decides expiry; wakers and payload destructors go to an elastic + disposal pool, with no default-path cost. A blocking destructor or waker in + actor A no longer delays actor B's token, raw or job deadlines + (`concurrency/tests/deadline_service.rs`, tasks and threads). The remaining + assumption (the owner's own executor wake runs on the service thread) is in + `docs/design/deferred-replies.md` §2. - **Post-0.8 cleanups (in progress):** experiment features moved to `benchmarks/`; the large actor and supervision files split; a lint and a `model-sync` mapping that keep the Loom-checked cores tied to their Loom tests. diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index ba32a6dd7..16ebc8b0e 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -79,6 +79,28 @@ pub const SITES: &[Site] = &[ refused, and its payload is destroyed only after every caller is \ answered (tasks and threads).", }, + Site { + name: "disposal.item", + has_error: false, + description: "A disposal worker is about to run an item the deadline service \ + handed off (a wake-up or a payload drop). Delay lets the owner's \ + exit begin first: a payload handed off before it is held until the \ + death point, not dropped by the worker.", + }, + Site { + name: "disposal.batch_serial", + has_error: true, + description: "Error: the deadline service's handed-off batch runs as one entry, \ + serially on one worker (the shared-batch bug fixed in PR #282). A \ + negative control: the disposal trace it records must be rejected by \ + Disposal.tla.", + }, + Site { + name: "disposal.keep_alive", + has_error: true, + description: "Error: an idle disposal worker waits 5 ms instead of the keep-alive \ + before it retires, so a test sees workers retire and new ones start.", + }, Site { name: "job.native_worker_unused", has_error: false, diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index b4eeaa555..f0fd3d46e 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -180,6 +180,36 @@ impl ReplyEvent { } } +/// One action of `verification/tla/Disposal.tla`: the deadline service's +/// expiry and hand-off, the disposal pool's steps, and the owner's payload +/// hold. On the service thread: `Expire` (a slot's transition; `r` the +/// request, `id` its wake-up item, `x`: `wake` or `none`), `Defer` (an item +/// handed off; `x`: `drop` or `other`) and `Submit` (the visit's items +/// queued; `x`: `spawn` if a worker start was decided). In the pool (`w` the +/// worker): `Begin`, `Take` (`id`, `x` as for `Submit`), `Run` and `Done` +/// (an item starts and returns), `Settle` (a payload's hold check; `x`: +/// `held` or `dropped`), `Wait`, `Woke` and `Retire`. The owner: `Exit` +/// (its hold begins) and `Death` (the hold is released at the death point). +/// Numbers are fresh and normalized by the checker. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DisposalEvent { + pub action: &'static str, + pub w: u64, + pub id: u64, + pub r: u64, + pub x: &'static str, +} + +impl DisposalEvent { + /// JSON object understood by `verification/tla/trace_check.py`. + pub fn to_json(&self) -> String { + format!( + r#"{{"a":"{}","w":{},"id":{},"r":{},"x":"{}"}}"#, + self.action, self.w, self.id, self.r, self.x + ) + } +} + /// One base-lifecycle action of `verification/tla/Lifecycle.tla`, recorded /// for the actors spawned while `record_lifecycle` is on. `t` is the actor /// the action belongs to (the target of a signal or message, the caller of @@ -486,6 +516,94 @@ mod imp { }) } + // Deadline-service disposal (`Disposal.tla`): one process-wide recording + // at a time. + static DISPOSAL_ON: AtomicBool = AtomicBool::new(false); + static DISPOSAL: Mutex> = Mutex::new(Vec::new()); + + /// Start recording disposal events until [`take_disposal`]. Tests that + /// record must not run concurrently with each other or with other + /// deadline expiries or actor exits in the process. + /// + /// The first event, `Pool`, is the pool's state when recording starts + /// (`w`: idle workers, `r`: workers, `x`: `starting` if a start is in + /// flight); the checker starts the model from it. + pub fn record_disposal() { + crate::reply::disposal::with_pool_state(|idle, workers, starting| { + atomic(|| { + let mut events = lock(&DISPOSAL); + events.clear(); + events.push(super::DisposalEvent { + action: "Pool", + w: idle as u64, + id: 0, + r: workers as u64, + x: if starting { "starting" } else { "" }, + }); + DISPOSAL_ON.store(true, Ordering::Release); + }); + }); + } + + /// Stop recording and return the events in their recorded order. + pub fn take_disposal() -> Vec { + atomic(|| { + DISPOSAL_ON.store(false, Ordering::Release); + std::mem::take(&mut *lock(&DISPOSAL)) + }) + } + + fn disposal_on() -> bool { + DISPOSAL_ON.load(Ordering::Acquire) + } + + /// Run `f` as one atomic step of the recorded linearization while + /// disposal events are recorded (and as is otherwise). + pub(crate) fn disposal_section(f: impl FnOnce() -> R) -> R { + if disposal_on() { + atomic(f) + } else { + f() + } + } + + /// A fresh slot or payload number for a disposal event (0 when off). + pub(crate) fn disposal_id() -> u64 { + static NEXT: AtomicU64 = AtomicU64::new(1); + if disposal_on() { + NEXT.fetch_add(1, Ordering::Relaxed) + } else { + 0 + } + } + + /// Record a disposal action (inside the section of the step it names). + /// An item or request numbered 0 (taken while recording was off), or a + /// worker step off a worker, is not recorded. + pub(crate) fn disposal(action: &'static str, w: u64, id: u64, r: u64, x: &'static str) { + if !disposal_on() { + return; + } + let recorded = match action { + "Expire" => r != 0, + "Defer" => id != 0, + "Take" | "Run" | "Done" | "Settle" => id != 0 && w != 0, + "Begin" | "Wait" | "Woke" | "Retire" => w != 0, + _ => true, + }; + if recorded { + atomic(|| { + lock(&DISPOSAL).push(super::DisposalEvent { + action, + w, + id, + r, + x, + }); + }); + } + } + /// Whether actor `id` is recorded. pub(crate) fn life_traced(id: u64) -> bool { LIFE_ON.load(Ordering::Acquire) && lock(&MEMBERS).contains(&id) @@ -1082,10 +1200,22 @@ mod imp { pub(crate) fn in_actor(_: u64, turn: T) -> T { turn } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn disposal(_: &'static str, _: u64, _: u64, _: u64, _: &'static str) {} + #[inline(always)] + pub(crate) fn disposal_section(f: impl FnOnce() -> R) -> R { + f() + } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn disposal_id() -> u64 { + 0 + } } #[cfg(feature = "lifecycle-trace")] pub use imp::{ - record_lifecycle, recorded, recorded_with, take, take_jobs, take_lifecycle, take_replies, - take_tree, watch_deferred, + record_disposal, record_lifecycle, recorded, recorded_with, take, take_disposal, take_jobs, + take_lifecycle, take_replies, take_tree, watch_deferred, }; diff --git a/concurrency/src/mailbox.rs b/concurrency/src/mailbox.rs index 8c153872f..6d473e27d 100644 --- a/concurrency/src/mailbox.rs +++ b/concurrency/src/mailbox.rs @@ -31,27 +31,85 @@ impl DropFailures { } /// The owner's exit began: hold every later drop until `release`. pub(crate) fn hold(&self) { - self.held - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get_or_insert_with(Vec::new); + // Disposal.tla `Exit`, linearized with the hold checks below. + crate::lifecycle_trace::disposal_section(|| { + let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); + if held.is_none() { + *held = Some(Vec::new()); + crate::lifecycle_trace::disposal("Exit", 0, 0, 0, ""); + } + }); } /// Every caller was answered: drop what was held (contained), and drop /// at once from now on. Idempotent. pub(crate) fn release(&self) { - let held = self.held.lock().unwrap_or_else(|p| p.into_inner()).take(); + // Disposal.tla `Death`: the hold ends. + let held = crate::lifecycle_trace::disposal_section(|| { + let held = self.held.lock().unwrap_or_else(|p| p.into_inner()).take(); + if held.is_some() { + crate::lifecycle_trace::disposal("Death", 0, 0, 0, ""); + } + held + }); for value in held.into_iter().flatten() { self.drop_contained(value); } } - pub(crate) fn discard(&self, value: T) { - let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); - if let Some(held) = held.as_mut() { - // Moved, not dropped: no user code under the lock. - held.push(Box::new(value)); - return; + /// On the deadline service thread the drop goes to a disposal worker, + /// which settles it there ([`settle`](Self::settle)): the hold is + /// re-checked, so a drop deferred before the owner's exit still waits for + /// its death point. + pub(crate) fn discard(self: &Arc, value: T) { + let mut value = Some(value); + // One step of Disposal.tla with `Exit` and `Death`: the hold check + // and, on the service thread, `Release` (the hand-off, recorded as + // `Defer` with `x: drop`). + let inline = crate::lifecycle_trace::disposal_section(|| { + let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(held) = held.as_mut() { + // Moved, not dropped: no user code under the lock. + held.push(Box::new(value.take())); + return false; + } + drop(held); + if crate::reply::disposal::collecting() { + let failures = self.clone(); + let value = value.take(); + crate::reply::disposal::defer_as("drop", move || failures.settle(value)); + return false; + } + true + }); + if inline { + self.drop_contained(value); + } + } + /// [`discard`](Self::discard) of an optional payload: nothing to dispose + /// of (and no hand-off) when it is `None`. + #[inline] + pub(crate) fn discard_some(self: &Arc, value: Option) { + if let Some(value) = value { + self.discard(value); } - drop(held); + } + /// A disposal worker's step for a payload the service handed off + /// (Disposal.tla `Settle`, then the destructor until `Done`): held if + /// the owner's exit began meanwhile (dropped at its death point), else + /// dropped now. A drop decided before the exit is pre-exit work and may + /// still run while the exit proceeds; the exit never waits for it + /// (deferred-replies.md §5). + fn settle(&self, value: T) { + let mut value = Some(value); + let (w, item) = crate::reply::disposal::current(); + crate::lifecycle_trace::disposal_section(|| { + let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(held) = held.as_mut() { + held.push(Box::new(value.take())); + crate::lifecycle_trace::disposal("Settle", w, item, 0, "held"); + } else { + crate::lifecycle_trace::disposal("Settle", w, item, 0, "dropped"); + } + }); self.drop_contained(value); } /// Drop at once (contained): the mailbox's own items, destroyed after diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 7fd354d88..21abd4bac 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -12,7 +12,9 @@ //! due entries even when no caller polls, so credits return on time: //! //! - real time: one process-wide service thread with its own timer, so -//! per-test Tokio runtimes cannot break it; +//! per-test Tokio runtimes cannot break it. What an expiry releases +//! (wakers, payload destructors) goes to a disposal worker +//! (`super::disposal`, which lists what still runs inline); //! - under `spawned_sim`: a task of the simulated runtime per actor heap, so //! clock domains never mix and virtual time drives it. @@ -25,8 +27,14 @@ use std::sync::{Arc, Weak}; pub(crate) trait Expirable: Send + Sync { /// The current deadline while the item is pending, else `None`. fn pending_deadline(&self) -> Option; - /// The deadline passed (`now >= deadline`): expire the item. - fn expire(&self, now: u64); + /// Service a popped entry: if `due` (its deadline passed, `now >= + /// deadline`), expire it. Returns whether dropping the last reference + /// could still run user code (an unread reply): the service hands such a + /// drop off ([`super::disposal`]) and drops the others in place, with no + /// hand-off. Job entries route their user payloads through the owner's + /// drop sink (deferred there), so they return `false`. One method, not + /// two: each is instantiated per reply type in the caller's crate. + fn expire(&self, now: u64, due: bool) -> bool; } struct Entry { @@ -304,15 +312,22 @@ mod thread_service { state.actors.remove(&key); drop(state); // Entries are contained one by one; this also covers the - // visit itself and dropping the upgraded registry, so the - // one process-wide service never stops. - super::super::registry::contain_entry(|| { - if let Some(registry) = weak.upgrade() { - if let Some(next) = registry.service_deadlines(now) { - schedule(®istry, next); + // visit itself, so the one process-wide service never + // stops. Whatever the visit releases that can run user + // code (wakers, payloads, the upgraded registry) goes to + // a disposal worker: this thread runs none of it. + let released = super::super::disposal::collect(|| { + super::super::registry::contain_entry(|| { + if let Some(registry) = weak.upgrade() { + if let Some(next) = registry.service_deadlines(now) { + schedule(®istry, next); + } + super::super::disposal::defer_drop_arc(registry); } - } + }); }); + // Disposal.tla `Submit` (recorded there). + super::super::disposal::submit(released); state = lock(service); } Some(Reverse((at, _))) => { diff --git a/concurrency/src/reply/disposal.rs b/concurrency/src/reply/disposal.rs new file mode 100644 index 000000000..90c6106ff --- /dev/null +++ b/concurrency/src/reply/disposal.rs @@ -0,0 +1,599 @@ +//! Disposal off the deadline service thread +//! (docs/design/deferred-replies.md §2, "Callback progress"; modelled in +//! verification/tla/Disposal.tla, whose actions the comments name). +//! +//! The process-wide deadline thread decides expiry: it pops due entries, +//! makes their terminal transitions and returns their deferral credits +//! (atomics and framework locks only). What an expiry releases that can run +//! user code (a caller's waker, a payload destructor routed to the owner's +//! failure sink, the last reference to a job or registry, a slot holding an +//! unread reply) is collected while the thread services a visit +//! ([`collect`]) and handed to the disposal workers ([`submit`]) one item at +//! a time. A worker that blocks in user code holds only the item it runs: a +//! queued item that finds no idle worker starts another one, so, while fewer +//! than [`MAX_WORKERS`] callbacks are blocked at once, a blocking destructor +//! or waker delays no other expiry and no other wake-up, of another actor or +//! of the same one. Past the cap, later items queue (expiry does not). +//! +//! The service thread still runs, inline: the owner's own executor wake +//! (enqueueing a job's message), dropping (not waking) a job's stored +//! wakers, the `tracing` subscriber for a contained panic or the cap +//! warning, a panic payload's destructor (`dispose_panic`), and, only if no +//! disposal thread exists and none can be spawned, the work itself. +//! +//! Outside a [`collect`] scope, [`defer`] and [`defer_drop`] run inline, so +//! every other thread (owner turns, jobs, the simulated service task) keeps +//! its behaviour. The check is one thread-local read, made only on paths +//! that already have something to wake or drop. +//! +//! A burst of quick items can start workers that are then idle: at most one +//! start is in flight, and idle workers exit after [`KEEP_ALIVE`]. + +use std::cell::RefCell; +use std::collections::VecDeque; +use std::sync::{Arc, Condvar, Mutex, OnceLock}; +use std::time::Duration; + +pub(crate) type Work = Box; + +/// One handed-off item. With `lifecycle-trace`, its number in disposal +/// traces (0: not recorded). +pub(crate) struct Item { + work: Work, + #[cfg(feature = "lifecycle-trace")] + id: u64, +} + +impl Item { + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + fn new(work: Work, id: u64) -> Self { + Self { + work, + #[cfg(feature = "lifecycle-trace")] + id, + } + } + fn id(&self) -> u64 { + #[cfg(feature = "lifecycle-trace")] + return self.id; + #[cfg(not(feature = "lifecycle-trace"))] + 0 + } +} + +impl From for Item { + fn from(work: Work) -> Self { + Self::new(work, 0) + } +} + +thread_local! { + /// `Some` while the deadline thread services a visit. + static BATCH: RefCell>> = const { RefCell::new(None) }; +} + +#[cfg(feature = "lifecycle-trace")] +thread_local! { + /// This disposal worker's number in traces (0: not a worker), and the + /// item it runs. + static WORKER: std::cell::Cell<(u64, u64)> = const { std::cell::Cell::new((0, 0)) }; +} + +/// The current worker and item numbers (0, 0 outside a worker's item). +pub(crate) fn current() -> (u64, u64) { + #[cfg(feature = "lifecycle-trace")] + return WORKER.try_with(|w| w.get()).unwrap_or((0, 0)); + #[cfg(not(feature = "lifecycle-trace"))] + (0, 0) +} + +/// Run `work` now, or, on the deadline thread inside [`collect`], defer it +/// to a disposal worker. Runs inline when this thread's batch is gone (a +/// thread-local destructor dropping a payload at thread exit) or borrowed. +#[inline] +pub(crate) fn defer(work: impl FnOnce() + Send + 'static) { + defer_as("other", work); +} + +/// [`defer`] an item of `kind` (`wake`, `drop` or `other`). Returns its +/// number in disposal traces (0 if it ran inline or nothing is recorded). +/// Records `Defer` (Disposal.tla `Release`/`Other`) except for a wake-up, +/// whose hand-off is part of `Expire`. +#[inline] +pub(crate) fn defer_as(kind: &'static str, work: impl FnOnce() + Send + 'static) -> u64 { + let mut work = Some(work); + let mut id = 0; + let _ = BATCH.try_with(|batch| { + if let Ok(mut batch) = batch.try_borrow_mut() { + if let Some(batch) = batch.as_mut() { + id = crate::lifecycle_trace::disposal_id(); + if kind != "wake" { + crate::lifecycle_trace::disposal("Defer", 0, id, 0, kind); + } + let work: Work = Box::new(work.take().expect("not yet taken")); + batch.push(Item::new(work, id)); + } + } + }); + if let Some(work) = work { + work(); + } + id +} + +/// Drop `value` now, or defer its drop (see [`defer`]). +#[inline] +pub(crate) fn defer_drop(value: T) { + defer(move || drop(value)); +} + +/// Drop `arc` now, or, when it is the last reference, defer dropping its +/// value (see [`defer`]). A reference that is not the last runs no user +/// code, so it costs no hand-off. +#[inline] +pub(crate) fn defer_drop_arc(arc: Arc) { + if let Some(value) = Arc::into_inner(arc) { + defer_drop(value); + } +} + +/// Whether this thread is collecting (the deadline thread in a visit). +#[inline] +pub(crate) fn collecting() -> bool { + BATCH + .try_with(|batch| batch.try_borrow().is_ok_and(|batch| batch.is_some())) + .unwrap_or(false) +} + +/// Run `step` with deferral on, and return what it deferred. If `step` +/// unwinds, what it deferred so far is submitted, not dropped here. +pub(crate) fn collect(step: impl FnOnce()) -> Vec { + struct Scope; + impl Scope { + fn take() -> Vec { + BATCH + .try_with(|batch| batch.borrow_mut().take().unwrap_or_default()) + .unwrap_or_default() + } + } + impl Drop for Scope { + fn drop(&mut self) { + // Only on unwind: the normal path took the batch already. + submit(Self::take()); + } + } + BATCH.with(|batch| *batch.borrow_mut() = Some(Vec::new())); + let scope = Scope; + step(); + let work = Scope::take(); + drop(scope); + work +} + +// --------------------------------------------------------------------------- +// Workers +// --------------------------------------------------------------------------- + +/// Workers beyond this many wait for a busy one: a bound on threads, reached +/// only when that many callbacks are blocked at once. +pub(crate) const MAX_WORKERS: usize = 64; +/// An idle worker exits after this long without work. +#[cfg(not(test))] +const KEEP_ALIVE: Duration = Duration::from_secs(10); +/// Short in tests, so submissions race idle exits. +#[cfg(test)] +const KEEP_ALIVE: Duration = Duration::from_millis(2); + +#[derive(Default)] +struct State { + queue: VecDeque, + /// Workers waiting for work (or woken and not yet back under the lock). + idle: usize, + workers: usize, + /// A worker is starting: it will take an item when it runs, so no other + /// starts meanwhile (a burst of quick items does not start one each). + starting: bool, + /// The cap was reported; reset once below it again. + capped: bool, +} + +impl State { + /// `NeedsWorker`: queued work needs another worker (more items than idle + /// workers that could take them, none starting, below the cap). + fn needs_worker(&mut self) -> Start { + if self.queue.len() <= self.idle || self.starting { + return Start::No; + } + if self.workers >= MAX_WORKERS { + if std::mem::replace(&mut self.capped, true) { + return Start::No; + } + return Start::Capped; + } + self.workers += 1; + self.starting = true; + Start::Yes + } +} + +enum Start { + No, + Yes, + /// At the cap for the first time since below it: report it. + Capped, +} + +impl Start { + /// The decision as a disposal trace's `x`. + #[allow(dead_code)] // recorded only with `lifecycle-trace` + fn x(&self) -> &'static str { + match self { + Start::Yes => "spawn", + _ => "", + } + } +} + +#[derive(Default)] +struct Pool { + state: Mutex, + ready: Condvar, +} + +fn pool() -> &'static Arc { + static POOL: OnceLock> = OnceLock::new(); + POOL.get_or_init(Default::default) +} + +fn lock(pool: &Pool) -> std::sync::MutexGuard<'_, State> { + pool.state.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// Run `f` with the pool's (idle, workers, starting) under its lock: a +/// recording starts from a consistent pool state (lock order: pool, then +/// the trace section, as everywhere). +#[cfg(feature = "lifecycle-trace")] +pub(crate) fn with_pool_state(f: impl FnOnce(usize, usize, bool) -> R) -> R { + let state = lock(pool()); + f(state.idle, state.workers, state.starting) +} + +/// `Submit`: hand deferred work to the disposal workers, one queue entry per +/// item, so a blocking item delays none of the others. When queued items +/// outnumber idle workers a new worker starts (one at a time, up to +/// [`MAX_WORKERS`]); each worker re-checks when it takes an item. +pub(crate) fn submit(work: Vec) { + if work.is_empty() { + return; + } + #[allow(unused_mut)] + let mut work = work; + // Test-only: run the batch serially on one worker (the shared-batch + // bug), so trace validation can be shown to reject it. + fail_point!("disposal.batch_serial" => { + let items = std::mem::take(&mut work); + let serial: Work = Box::new(move || items.into_iter().for_each(run_item)); + work = vec![Item::from(serial)]; + }); + let pool = pool(); + let mut state = lock(pool); + let wake = work.len().min(state.idle); + state.queue.extend(work); + let start = state.needs_worker(); + crate::lifecycle_trace::disposal("Submit", 0, 0, 0, start.x()); + drop(state); + for _ in 0..wake { + pool.ready.notify_one(); + } + start_worker(pool, start); +} + +fn start_worker(pool: &Arc, start: Start) { + match start { + Start::No => {} + Start::Capped => crate::actor_cleanup::log_safely(|| { + tracing::warn!( + event = "disposal_workers_capped", + max = MAX_WORKERS, + "every deadline disposal worker is busy: later wake-ups and \ + payload drops queue until one returns" + ); + }), + Start::Yes => { + let worker = pool.clone(); + let spawned = std::thread::Builder::new() + .name("spawned-disposal".into()) + .spawn(move || run(&worker)); + if spawned.is_err() { + // No new thread: an existing worker takes the work. With + // none at all, run it here rather than lose a wake-up. + let orphaned = { + let mut state = lock(pool); + state.workers -= 1; + state.starting = false; + if state.workers == 0 { + std::mem::take(&mut state.queue) + } else { + VecDeque::new() + } + }; + orphaned.into_iter().for_each(run_item); + } + } + } +} + +/// A worker's loop (Disposal.tla `Begin`, `Take`, `Wait`, `Woke`, `Retire`; +/// each recorded under the pool lock, with the start decision on `Take`). +fn run(pool: &Arc) { + #[cfg(feature = "lifecycle-trace")] + let w = { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + let w = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let _ = WORKER.try_with(|c| c.set((w, 0))); + w + }; + #[cfg(not(feature = "lifecycle-trace"))] + let w = 0; + let mut state = lock(pool); + state.starting = false; + crate::lifecycle_trace::disposal("Begin", w, 0, 0, ""); + loop { + if let Some(item) = state.queue.pop_front() { + // Whatever is still queued must not wait for this item. + if !state.queue.is_empty() && state.idle > 0 { + pool.ready.notify_one(); + } + let start = state.needs_worker(); + crate::lifecycle_trace::disposal("Take", w, item.id(), 0, start.x()); + drop(state); + start_worker(pool, start); + run_item(item); + state = lock(pool); + continue; + } + state.idle += 1; + crate::lifecycle_trace::disposal("Wait", w, 0, 0, ""); + let keep_alive = keep_alive(); + let (next, timeout) = pool + .ready + .wait_timeout(state, keep_alive) + .unwrap_or_else(|p| p.into_inner()); + state = next; + state.idle -= 1; + crate::lifecycle_trace::disposal("Woke", w, 0, 0, ""); + if timeout.timed_out() && state.queue.is_empty() { + state.workers -= 1; + state.capped = false; + crate::lifecycle_trace::disposal("Retire", w, 0, 0, ""); + return; + } + } +} + +/// [`KEEP_ALIVE`]; the `disposal.keep_alive` failpoint (Error) shortens it so +/// a test sees workers retire. +fn keep_alive() -> Duration { + fail_point!("disposal.keep_alive" => return Duration::from_millis(5)); + KEEP_ALIVE +} + +/// `Run` then `Done`: contained, so a failing callback ends neither its +/// worker nor the others. +fn run_item(item: Item) { + fail_point!("disposal.item"); + let (w, _) = current(); + let id = item.id(); + #[cfg(feature = "lifecycle-trace")] + let _ = WORKER.try_with(|c| c.set((w, id))); + crate::lifecycle_trace::disposal("Run", w, id, 0, ""); + super::registry::contain_entry(item.work); + crate::lifecycle_trace::disposal("Done", w, id, 0, ""); + #[cfg(feature = "lifecycle-trace")] + let _ = WORKER.try_with(|c| c.set((w, 0))); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::mpsc; + + /// Submit plain work (untraced items). + fn submit(work: Vec) { + super::submit(work.into_iter().map(Item::from).collect()); + } + + /// Generous: sanitizer builds are slow, and a panic's first report there + /// can take seconds. + const WAIT: Duration = Duration::from_secs(60); + + #[test] + fn defer_runs_inline_outside_a_scope() { + let (tx, rx) = mpsc::channel(); + defer(move || tx.send(()).unwrap()); + rx.try_recv().expect("ran inline"); + assert!(!collecting()); + } + + #[test] + fn collect_defers_and_ends_its_scope() { + let (tx, rx) = mpsc::channel(); + let work = collect(|| { + assert!(collecting()); + let sender = tx.clone(); + defer(move || sender.send(1).unwrap()); + defer_drop(tx); + }); + assert!(!collecting()); + assert!(rx.try_recv().is_err(), "deferred, not run"); + assert_eq!(work.len(), 2); + super::submit(work); + assert_eq!(rx.recv_timeout(WAIT), Ok(1)); + } + + #[test] + fn an_unwinding_scope_submits_what_it_collected() { + let (tx, rx) = mpsc::channel(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = collect(|| { + defer(move || tx.send(()).unwrap()); + panic!("injected visit panic"); + }); + })); + crate::mailbox::dispose_panic(result.unwrap_err()); + assert!(!collecting()); + assert!(rx.recv_timeout(WAIT).is_ok(), "collected work was lost"); + } + + #[test] + fn last_reference_drops_are_deferred_others_are_not() { + let shared = Arc::new(()); + let work = collect(|| { + defer_drop_arc(shared.clone()); // not the last: nothing deferred + defer_drop_arc(Arc::new(())); // the last: deferred + }); + assert_eq!(work.len(), 1); + assert_eq!(Arc::strong_count(&shared), 1); + } + + #[test] + fn a_blocked_item_does_not_hold_back_the_next_of_its_batch() { + let (release_tx, release_rx) = mpsc::channel::<()>(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (tx, rx) = mpsc::channel(); + // One visit's work: a blocking callback, then another caller's wake. + submit(vec![ + Box::new(move || { + entered_tx.send(()).unwrap(); + let _ = release_rx.recv_timeout(WAIT); + }), + Box::new(move || tx.send(()).unwrap()), + ]); + entered_rx + .recv_timeout(WAIT) + .expect("the first item started"); + assert!( + rx.recv_timeout(WAIT).is_ok(), + "a blocked item held back the next one of its batch" + ); + release_tx.send(()).unwrap(); + } + + #[test] + fn a_blocked_batch_does_not_hold_back_the_next() { + let (release_tx, release_rx) = mpsc::channel::<()>(); + let (entered_tx, entered_rx) = mpsc::channel(); + submit(vec![Box::new(move || { + entered_tx.send(()).unwrap(); + let _ = release_rx.recv_timeout(WAIT); + })]); + entered_rx + .recv_timeout(WAIT) + .expect("the first batch started"); + let (tx, rx) = mpsc::channel(); + submit(vec![Box::new(move || tx.send(()).unwrap())]); + assert!( + rx.recv_timeout(WAIT).is_ok(), + "a blocked worker held back another batch" + ); + release_tx.send(()).unwrap(); + } + + #[test] + fn a_panicking_item_is_contained() { + let (tx, rx) = mpsc::channel(); + submit(vec![ + Box::new(|| panic!("injected disposal panic")), + Box::new(move || tx.send(()).unwrap()), + ]); + assert!(rx.recv_timeout(WAIT).is_ok()); + } + + /// Submissions race workers going idle and exiting (`KEEP_ALIVE` is 2 ms + /// in tests), with some items blocked meanwhile: every item runs. + #[test] + fn submissions_racing_idle_exits_lose_no_item() { + let (release_tx, release_rx) = mpsc::channel::<()>(); + let release_rx = Arc::new(Mutex::new(release_rx)); + let (tx, rx) = mpsc::channel::(); + let mut expected = 0; + let submitters: Vec<_> = (0..4) + .map(|t| { + let tx = tx.clone(); + let release_rx = release_rx.clone(); + std::thread::spawn(move || { + let mut sent = 0; + for round in 0..150usize { + let mut work: Vec = Vec::new(); + for _ in 0..(round % 3 + 1) { + let tx = tx.clone(); + work.push(Box::new(move || tx.send(1).unwrap())); + sent += 1; + } + if round % 50 == t { + // A blocked callback, released at the end. + let release_rx = release_rx.clone(); + work.insert( + 0, + Box::new(move || { + let _ = release_rx.lock().unwrap().recv_timeout(WAIT); + }), + ); + } + submit(work); + // Around the keep-alive: idle workers time out + // while later submissions arrive. + std::thread::sleep(Duration::from_micros(500 * (round as u64 % 6))); + } + sent + }) + }) + .collect(); + for submitter in submitters { + expected += submitter.join().unwrap(); + } + let mut ran = 0; + while ran < expected { + match rx.recv_timeout(WAIT) { + Ok(n) => ran += n, + Err(_) => break, + } + } + for _ in 0..16 { + let _ = release_tx.send(()); + } + assert_eq!( + ran, expected, + "an item was lost or held behind a blocked one" + ); + } + + /// A thread-local destructor that drops a payload after this thread's + /// batch is destroyed runs it inline instead of aborting. + #[test] + fn deferral_after_thread_local_teardown_runs_inline() { + struct DropsLater(Option>); + impl Drop for DropsLater { + fn drop(&mut self) { + let tx = self.0.take().unwrap(); + let collecting = collecting(); + let sender = tx.clone(); + defer(move || sender.send(true).unwrap()); + let _ = tx.send(collecting); + } + } + thread_local! { + static LATER: RefCell> = const { RefCell::new(None) }; + } + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + // Registered first, so destroyed after the batch (reverse order). + LATER.with(|later| *later.borrow_mut() = Some(DropsLater(Some(tx)))); + let _ = collect(|| {}); + }) + .join() + .expect("a deferral during thread-local teardown must not abort"); + let got: Vec = rx.try_iter().collect(); + assert_eq!(got, [true, false], "ran inline, not collecting"); + } +} diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 79ee101f8..4bea803c1 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -257,6 +257,22 @@ pub(crate) struct JobState { /// Payloads an owner's exit took from a job, destroyed after its death point. type ExitReleased = (Option, Vec>); +impl Drop for JobState { + /// The last reference can go on the deadline service (a job deadline or + /// emit entry): what exit released and nobody disposed of yet is user + /// payload, so it goes to a disposal worker there (inline elsewhere). + fn drop(&mut self) { + let released = self + .exit_released + .get_mut() + .unwrap_or_else(|p| p.into_inner()) + .take(); + if let Some(released) = released { + super::disposal::defer_drop(released); + } + } +} + fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().unwrap_or_else(|p| p.into_inner()) } @@ -415,9 +431,15 @@ impl JobState { #[cfg(feature = "async")] fn wake_emitters(&self) { let wakers = std::mem::take(&mut *lock(&self.emit_wakers)); - for waker in wakers { - waker.wake(); + if wakers.is_empty() { + return; } + // Off the deadline service thread when a deadline or emit fires. + super::disposal::defer(move || { + for waker in wakers { + waker.wake(); + } + }); } /// Take the terminal notification and run `enqueue` under the job's @@ -456,8 +478,11 @@ impl JobState { (taken, unsent, held) }); // A held result the deadline displaced (user payload): dropped - // outside the trace section, after JobDeadline is recorded. - drop(held); + // outside the trace section, after JobDeadline is recorded, and off + // the deadline service thread. + if let Some(held) = held { + super::disposal::defer_drop(held); + } #[cfg(feature = "async")] if taken { self.wake_emitters(); @@ -987,11 +1012,14 @@ impl Expirable for JobDeadline { fn pending_deadline(&self) -> Option { self.state.core.is_pending().then_some(self.at) } - fn expire(&self, _now: u64) { - let notify = lock(&self.notify).take(); - if let Some(notify) = notify { - notify(); + fn expire(&self, _now: u64, due: bool) -> bool { + if due { + let notify = lock(&self.notify).take(); + if let Some(notify) = notify { + notify(); + } } + false } } @@ -1047,7 +1075,7 @@ impl Drop for ScheduledEmit { .get_mut() .unwrap_or_else(|p| p.into_inner()) .take(); - self.failures.discard(send); + self.failures.discard_some(send); } } @@ -1083,8 +1111,11 @@ impl ScheduledEmit { let delivered = result.is_ok(); // Whatever was not enqueued (the refused envelope, the uncalled send // or the discarded one, each with its credit guard) drops here, - // unlocked and contained. - self.failures.discard((result, unused, discarded)); + // unlocked and contained. A delivered emit leaves nothing to drop: + // no hand-off then. + if result.is_err() || unused.is_some() || discarded.is_some() { + self.failures.discard((result, unused, discarded)); + } lock(&job.scheduled).retain(|emit| !std::ptr::eq(Arc::as_ptr(emit), self)); if let Some(held) = held { held(); @@ -1103,11 +1134,17 @@ impl Expirable for ScheduledEmit { fn pending_deadline(&self) -> Option { lock(&self.send).is_some().then_some(self.at) } - fn expire(&self, _now: u64) { + fn expire(&self, _now: u64, due: bool) -> bool { + if !due { + return false; + } fail_point!("job.emit_due"); if let Some(job) = self.job.upgrade() { self.settle(&job, true); + // Possibly the last reference (its other scheduled emits). + super::disposal::defer_drop_arc(job); } + false } } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index db81bbb0f..31b263140 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -7,6 +7,7 @@ //! gets exactly one terminal discharge. pub(crate) mod deadline; +pub(crate) mod disposal; pub(crate) mod job_core; pub mod jobs; mod pending; diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index cf4503b1f..3b5d5a43a 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -59,11 +59,56 @@ impl Expirable for Slot { let deadline = self.deadline(); (!self.is_terminal() && deadline != NEVER).then_some(deadline) } - fn expire(&self, now: u64) { - self.check_deadline(&|| now); + /// The transition and credit return happen here; a parked blocking + /// receiver is unparked here too (no user code); the wakers are handed + /// to a disposal worker when this runs on the deadline thread. Only a + /// `REPLIED` slot can hold a value (an unread reply, user code when + /// destroyed), or one not yet terminal (a reply still `WRITING` may + /// store one); any other terminal state is final and holds none. + fn expire(&self, now: u64, due: bool) -> bool { + if !due { + return may_hold_value(self); + } + if let Some(mut wake) = self.expire_collect(now) { + wake.notify_parked(); + // Disposal.tla `Expire`: after the transition and its credit + // return, the wake-up (if any) is handed off. Compiled out + // without `lifecycle-trace`; recorded only on the service. + let request = if cfg!(feature = "lifecycle-trace") && super::disposal::collecting() { + crate::lifecycle_trace::disposal_id() + } else { + 0 + }; + let (item, kind) = if wake.is_empty() { + (0, "none") + } else { + ( + super::disposal::defer_as("wake", move || wake.run()), + "wake", + ) + }; + crate::lifecycle_trace::disposal("Expire", 0, item, request, kind); + } + may_hold_value(self) } } +/// Whether dropping `slot`'s last reference could run user code: it is +/// `REPLIED` (an unread value) or not yet terminal. +/// Not generic (one copy, in this crate): only the slot's word is read. +#[inline(never)] +fn may_hold_word(word: u64) -> bool { + let state = super::slot::state(word); + match Terminal::of(state) { + None => true, + Some(terminal) => terminal == Terminal::Replied, + } +} + +fn may_hold_value(slot: &Slot) -> bool { + may_hold_word(slot.word()) +} + impl Closeable for Slot { fn close_collect(&self) -> Option { Slot::close_collect(self) @@ -292,19 +337,25 @@ impl Registry { /// The service visits: expire every due entry outside the lock, and /// return the next earliest deadline. /// - /// Entries run framework code that can reach user code (a waker, a - /// payload destructor the entry did not route to its failure sink, the - /// last reference to a job's `done`). Each runs contained, so one - /// failing entry neither skips the others nor ends the shared service. + /// Entries make their terminal transitions and return credits here. + /// What they release that can run user code (a waker, a payload + /// destructor, the last reference to an entry or a job's `done`) is + /// deferred: on the deadline thread it goes to a disposal worker + /// ([`disposal`](super::disposal)); elsewhere it runs inline. Each entry + /// runs contained, so one failing entry neither skips the others nor + /// ends the shared service. pub(crate) fn service_deadlines(&self, now: u64) -> Option { let due = lock(&self.lists).deadlines.due(now); for item in due { contain_entry(|| { - if item.pending_deadline().is_some_and(|d| d <= now) { - item.expire(now); + let due = item.pending_deadline().is_some_and(|d| d <= now); + // Possibly the last reference: handed off only if dropping + // it could run user code (see `Expirable::expire`). + if item.expire(now, due) { + super::disposal::defer_drop(item); + } else { + drop(item); } - // Possibly the last reference: dropped inside the guard. - drop(item); }); } lock(&self.lists).deadlines.earliest() @@ -840,8 +891,11 @@ mod tests { fn pending_deadline(&self) -> Option { Some(self.0) } - fn expire(&self, _: u64) { - let _ = self.1.send(()); + fn expire(&self, _: u64, due: bool) -> bool { + if due { + let _ = self.1.send(()); + } + false } } assert!(spawned_rt::tasks::Handle::try_current().is_err()); diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 6c5b63132..f1ce7be0b 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -336,15 +336,19 @@ impl Wake { }; /// Wake a parked blocking receiver (no user code). Done first, so a /// parked `recv()` never waits behind an arbitrary waker. - fn notify_parked(&mut self) { + pub(crate) fn notify_parked(&mut self) { if std::mem::take(&mut self.parked) { if let Some(waker) = self.waker.take() { waker.wake(); } } } + /// Nothing left to run: no waker and no watchers. + pub(crate) fn is_empty(&self) -> bool { + self.waker.is_none() && self.watchers.is_empty() + } #[inline] - fn run(self) { + pub(crate) fn run(self) { if let Some(waker) = self.waker { waker.wake(); } @@ -545,10 +549,15 @@ impl Slot { /// [`run_wake`](Self::run_wake) once a whole batch is closed, so no /// waker (executor or user code) runs between two closes. pub(crate) fn close_collect(&self) -> Option { + self.terminate_collect(STOPPED) + } + + /// `OPEN -> target` with its credit return; the wake-up is returned. + fn terminate_collect(&self, target: u64) -> Option { let (old, new) = self .cas::<()>(None, |w| { if state(w) == OPEN { - Ok(with_state(w, STOPPED)) + Ok(with_state(w, target)) } else { Err(()) } @@ -613,6 +622,19 @@ impl Slot { } } + /// The deadline service's `[deadline]` check: the same `OPEN -> TIMED_OUT` + /// transition and credit return as [`check_deadline`](Self::check_deadline), + /// but the wake-up (a caller's waker, the token's watchers: executor or + /// user code) is returned for the caller to run elsewhere. `None` when + /// the deadline has not passed or another transition won. + pub(crate) fn expire_collect(&self, now: u64) -> Option { + let deadline = self.deadline(); + if deadline == NEVER || now < deadline { + return None; + } + self.terminate_collect(TIMED_OUT) + } + /// `is_closed()`: not `OPEN`, or past its deadline (then expired). pub(crate) fn is_closed(&self, now: &impl Fn() -> u64) -> bool { state(self.word()) != OPEN || self.check_deadline(now) diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index ef691fd7e..fa83ae99a 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -151,7 +151,7 @@ where let (_, unsent) = self.state.notify(end, || self.enqueue_terminal(outcome)); // A refused terminal message (`done` and the outcome) may drop on // the deadline service thread: contained. - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); } fn enqueue_terminal( @@ -196,10 +196,13 @@ where if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { run.notify(outcome); } + // Possibly the last reference (`done` and its captures): + // off the deadline service thread. + crate::reply::disposal::defer_drop_arc(run); }) }, ); - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); } } @@ -212,7 +215,7 @@ impl Drop for Run { .get_mut() .unwrap_or_else(|p| p.into_inner()) .take(); - self.ctx.shared.sender.failures().discard(done); + self.ctx.shared.sender.failures().discard_some(done); } } @@ -437,7 +440,7 @@ impl JobCtx { self.ctx.shared.sender.send(envelope) }) .map_err(|(gone, unsent)| { - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); gone }) } @@ -717,13 +720,15 @@ impl Context { notify: Mutex::new(Some(Box::new(move || { if let Some(run) = weak.upgrade() { run.notify(Err(JobError::DeadlineExceeded)); + // Possibly the last reference: off the deadline thread. + crate::reply::disposal::defer_drop_arc(run); } }))), }); // The run owns it; the heap only refers to it. let _ = run.deadline.set(deadline.clone()); if at <= now_ticks() { - deadline.expire(at); + deadline.expire(at, true); } else { let item: Weak = Arc::downgrade(&deadline) as Weak; self.shared.replies.track(at, item); diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 45cc2972f..19a85441f 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -130,7 +130,7 @@ where let (_, unsent) = self.state.notify(end, || self.enqueue_terminal(outcome)); // A refused terminal message (`done` and the outcome) may drop on // the deadline service thread: contained. - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); } fn enqueue_terminal(&self, outcome: Contained>) -> Option> { @@ -178,10 +178,13 @@ where if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { run.notify(outcome); } + // Possibly the last reference (`done` and its captures): + // off the deadline service thread. + crate::reply::disposal::defer_drop_arc(run); }) }, ); - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); } } @@ -194,7 +197,7 @@ impl Drop for Run { .get_mut() .unwrap_or_else(|p| p.into_inner()) .take(); - self.ctx.shared.sender.failures().discard(done); + self.ctx.shared.sender.failures().discard_some(done); } } @@ -320,7 +323,7 @@ impl JobCtx { self.ctx.shared.sender.send(MailboxItem::Message(envelope)) }) .map_err(|(gone, unsent)| { - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); gone }) } @@ -488,13 +491,15 @@ impl Context { notify: Mutex::new(Some(Box::new(move || { if let Some(run) = weak.upgrade() { run.notify(Err(JobError::DeadlineExceeded)); + // Possibly the last reference: off the deadline thread. + crate::reply::disposal::defer_drop_arc(run); } }))), }); // The run owns it; the heap only refers to it. let _ = run.deadline.set(deadline.clone()); if at <= now_ticks() { - deadline.expire(at); + deadline.expire(at, true); } else { let item: Weak = Arc::downgrade(&deadline) as Weak; self.shared.replies.track(at, item); diff --git a/concurrency/tests/deadline_service.rs b/concurrency/tests/deadline_service.rs index 4e31c9908..63d0c6f70 100644 --- a/concurrency/tests/deadline_service.rs +++ b/concurrency/tests/deadline_service.rs @@ -133,3 +133,351 @@ fn deadline_service_child() { }); println!("deadline service survived"); } + +// --------------------------------------------------------------------------- +// Disposal isolation: user code that expiry releases (a payload destructor, a +// caller's waker) never runs on the shared deadline thread, so a blocking one +// in actor A does not delay actor B's deadlines. +// --------------------------------------------------------------------------- + +/// A one-shot latch with bounded waits. +#[derive(Default)] +struct Latch(std::sync::Mutex, std::sync::Condvar); +impl Latch { + fn set(&self) { + *self.0.lock().unwrap() = true; + self.1.notify_all(); + } + fn wait(&self, limit: Duration) -> bool { + let guard = self.0.lock().unwrap(); + let (guard, _) = self + .1 + .wait_timeout_while(guard, limit, |set| !*set) + .unwrap(); + *guard + } +} + +/// Blocking user code: signals `entered`, then waits (bounded) for `release`. +#[derive(Default)] +struct Blocker { + entered: Latch, + release: Latch, +} +impl Blocker { + fn block(&self) { + self.entered.set(); + self.release.wait(Duration::from_secs(10)); + } +} + +/// A message whose destructor blocks while armed. +struct Stall(Option>); +impl Message for Stall { + type Result = (); +} +impl Drop for Stall { + fn drop(&mut self) { + if let Some(blocker) = self.0.take() { + blocker.block(); + } + } +} +impl Handler for Owner { + async fn handle(&mut self, mut msg: Stall, _: PendingReply<'_, ()>, _: &Context) { + msg.0 = None; // delivered: not the scenario under test + } +} + +/// A waker whose `wake` blocks. +struct BlockingWake(std::sync::Arc); +impl std::task::Wake for BlockingWake { + fn wake(self: std::sync::Arc) { + self.0.block(); + } +} + +#[test] +fn deadline_service_isolates_a_blocking_payload_destructor() { + subprocess::run( + "deadline_service_blocking_destructor_child", + "blocking destructor isolated", + ); +} + +#[test] +fn deadline_service_isolates_a_blocking_waker() { + subprocess::run( + "deadline_service_blocking_waker_child", + "blocking waker isolated", + ); +} + +#[test] +fn deadline_service_isolates_a_blocking_destructor_threads() { + subprocess::run( + "deadline_service_blocking_destructor_threads_child", + "threads blocking destructor isolated", + ); +} + +/// Another actor's raw request and job deadlines, serviced by the heap. +async fn other_actor_deadlines_fire(blocker: &Blocker) -> Result<(), String> { + let other = Owner::default().start(); + let parked = other + .send_request(Park, Deadline::after(Duration::from_millis(50))) + .unwrap(); + let result = match tokio::time::timeout(Duration::from_secs(3), parked).await { + Ok(Err(ActorError::RequestTimeout)) => Ok(()), + Ok(other) => Err(format!("unexpected outcome {other:?}")), + Err(_) => Err("a blocked disposal delayed another actor's request deadline".into()), + }; + let fired = spawned_concurrency::__private::in_turn(&other, |_, ctx| { + let (tx, rx) = std::sync::mpsc::channel(); + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + |_| std::thread::sleep(Duration::from_millis(300)), + move |outcome: JobOutcome<()>| { + let _ = tx.send(outcome.is_err()); + Done + }, + ) + .unwrap(); + rx + }) + .unwrap() + .await + .unwrap(); + let job = tokio::task::spawn_blocking(move || fired.recv_timeout(Duration::from_secs(3))) + .await + .unwrap(); + // Release the blocked disposal before reporting, so the child exits. + blocker.release.set(); + let result = result.and(match job { + Ok(true) => Ok(()), + other => Err(format!( + "a blocked disposal delayed another actor's job deadline: {other:?}" + )), + }); + other.context().stop(); + other.join().await; + result +} + +fn tokio_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap() +} + +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_blocking_destructor_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let blocker = std::sync::Arc::new(Blocker::default()); + tokio_runtime().block_on(async { + let owner = Owner::default().start(); + let armed = blocker.clone(); + spawned_concurrency::__private::in_turn(&owner, move |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + move |job| { + // Discarded at 40 ms (the deadline took the notification + // at 20 ms): its destructor blocks. + let at = Instant::now() + Duration::from_millis(40); + job.emit_at_blocking(at, Stall(Some(armed))).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + }) + .unwrap() + .await + .unwrap(); + assert!( + blocker.entered.wait(Duration::from_secs(5)), + "the scheduled emit was never discarded" + ); + let result = other_actor_deadlines_fire(&blocker).await; + owner.context().stop(); + owner.join().await; + result.unwrap(); + }); + println!("blocking destructor isolated"); +} + +#[test] +fn deadline_service_isolates_callers_of_one_actor() { + subprocess::run( + "deadline_service_same_actor_child", + "same-actor callers isolated", + ); +} + +/// Two callers of one actor with the same deadline, expired in the same +/// service visit: the first caller's waker blocks, the second caller must +/// still be woken with its timeout. +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_same_actor_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let blocker = std::sync::Arc::new(Blocker::default()); + tokio_runtime().block_on(async { + let owner = Owner::default().start(); + let deadline = Deadline::after(Duration::from_millis(100)); + let first = owner.send_request(Park, deadline).unwrap(); + let second = owner.send_request(Park, deadline).unwrap(); + // The first is polled (and put in the heap) first, so its wake-up is + // collected first in the visit. + let mut first = Box::pin(std::future::IntoFuture::into_future(first)); + let waker = std::task::Waker::from(std::sync::Arc::new(BlockingWake(blocker.clone()))); + let mut cx = std::task::Context::from_waker(&waker); + assert!(std::future::Future::poll(first.as_mut(), &mut cx).is_pending()); + // The wake-up must come with the expiry: a final poll at the outer + // timeout would find the slot expired anyway, so time it. + let started = std::time::Instant::now(); + let outcome = tokio::time::timeout(Duration::from_secs(3), second).await; + let woken_after = started.elapsed(); + let entered = blocker.entered.wait(Duration::from_secs(5)); + blocker.release.set(); + drop(first); + owner.context().stop(); + owner.join().await; + assert!(entered, "the first caller's waker never ran"); + assert!( + matches!(outcome, Ok(Err(ActorError::RequestTimeout))), + "unexpected outcome {outcome:?}" + ); + assert!( + woken_after < Duration::from_millis(1500), + "a blocking waker delayed another caller of the same actor ({woken_after:?})" + ); + }); + println!("same-actor callers isolated"); +} + +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_blocking_waker_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let blocker = std::sync::Arc::new(Blocker::default()); + tokio_runtime().block_on(async { + let owner = Owner::default().start(); + // A raw receiver polled once with a waker that blocks: its deadline + // goes to the owner's heap, whose expiry wakes it. + let receiver = owner + .send_request(Park, Deadline::after(Duration::from_millis(30))) + .unwrap(); + let mut receiver = Box::pin(std::future::IntoFuture::into_future(receiver)); + let waker = std::task::Waker::from(std::sync::Arc::new(BlockingWake(blocker.clone()))); + let mut cx = std::task::Context::from_waker(&waker); + assert!(std::future::Future::poll(receiver.as_mut(), &mut cx).is_pending()); + assert!( + blocker.entered.wait(Duration::from_secs(5)), + "the receiver's waker never ran" + ); + let result = other_actor_deadlines_fire(&blocker).await; + drop(receiver); + owner.context().stop(); + owner.join().await; + result.unwrap(); + }); + println!("blocking waker isolated"); +} + +mod threads_backend { + use super::*; + use spawned_concurrency::threads::{ + Actor, ActorStart, Context, Handler, JobOptions, JobOutcome, + }; + + pub struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + fn handle(&mut self, mut msg: Stall, _: PendingReply<'_, ()>, _: &Context) { + msg.0 = None; + } + } + impl Handler for Owner { + fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + + pub fn run(blocker: &std::sync::Arc) -> Result<(), String> { + let owner = Owner.start(); + let armed = blocker.clone(); + spawned_concurrency::__private::in_turn_threads(&owner, move |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + move |job| { + let at = Instant::now() + Duration::from_millis(40); + job.emit_at_blocking(at, Stall(Some(armed))).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + }) + .unwrap() + .recv() + .unwrap(); + assert!( + blocker.entered.wait(Duration::from_secs(5)), + "the scheduled emit was never discarded" + ); + // Another actor's job deadline, serviced by the same deadline service. + let other = Owner.start(); + let fired = spawned_concurrency::__private::in_turn_threads(&other, |_, ctx| { + let (tx, rx) = std::sync::mpsc::channel(); + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + |_| std::thread::sleep(Duration::from_millis(300)), + move |outcome: JobOutcome<()>| { + let _ = tx.send(outcome.is_err()); + Done + }, + ) + .unwrap(); + rx + }) + .unwrap() + .recv() + .unwrap(); + let job = fired.recv_timeout(Duration::from_secs(3)); + blocker.release.set(); + let result = match job { + Ok(true) => Ok(()), + other => Err(format!( + "a blocked disposal delayed another actor's job deadline: {other:?}" + )), + }; + other.context().stop(); + other.join(); + owner.context().stop(); + owner.join(); + result + } +} + +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_blocking_destructor_threads_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let blocker = std::sync::Arc::new(Blocker::default()); + threads_backend::run(&blocker).unwrap(); + println!("threads blocking destructor isolated"); +} diff --git a/concurrency/tests/disposal_trace.rs b/concurrency/tests/disposal_trace.rs new file mode 100644 index 000000000..da19314ba --- /dev/null +++ b/concurrency/tests/disposal_trace.rs @@ -0,0 +1,307 @@ +//! Records the deadline service's hand-off to the disposal workers from real +//! actors, for TLA+ trace validation against `verification/tla/Disposal.tla` +//! (verification/tla/README.md). Each scenario writes `disposal_.json` +//! to `$LIFECYCLE_TRACE_DIR` (default `target/lifecycle-traces`); +//! `verification/tla/trace_check.py` checks them (schema `disposal-v1`). A +//! negative control recorded from deliberately broken code (the +//! `disposal.batch_serial` failpoint) goes to `controls/` there, and +//! `negative_checks.py` requires it to be rejected. +//! +//! The recording is process-wide (the service and the pool are), so +//! scenarios run one at a time, start with the pool quiescent and wait for +//! their workers before taking the trace. +#![cfg(all(feature = "lifecycle-trace", feature = "async"))] + +use spawned_concurrency::{ + lifecycle_trace, + message::Message, + tasks::{Actor, ActorRef, ActorStart, Context, Handler, Instant, JobOptions, JobOutcome}, + ActorError, Deadline, PendingReply, ReplyTo, +}; +use std::{ + path::PathBuf, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +/// One recording at a time (held across awaits). +static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn trace_dir() -> PathBuf { + std::env::var_os("LIFECYCLE_TRACE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/lifecycle-traces") + }) +} + +/// Let the pool settle (every worker waiting), then record. +async fn record() { + tokio::time::sleep(Duration::from_millis(100)).await; + lifecycle_trace::record_disposal(); +} + +/// Let the workers finish, stop recording and write the trace to `dir`; +/// returns the events as `Action:x`. +async fn write_to(dir: PathBuf, name: &str) -> Vec { + tokio::time::sleep(Duration::from_millis(100)).await; + let events = lifecycle_trace::take_disposal(); + let json = format!( + "{{\"name\":\"disposal_{name}\",\"schema\":\"disposal-v1\",\"events\":[\n{}\n]}}\n", + events + .iter() + .map(|e| e.to_json()) + .collect::>() + .join(",\n") + ); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("disposal_{name}.json")), json).unwrap(); + events + .iter() + .map(|e| { + if e.x.is_empty() { + e.action.to_string() + } else { + format!("{}:{}", e.action, e.x) + } + }) + .collect() +} + +async fn write(name: &str) -> Vec { + write_to(trace_dir(), name).await +} + +fn count(events: &[String], event: &str) -> usize { + events.iter().filter(|e| *e == event).count() +} + +/// A payload whose destructor is counted. +struct Payload(Arc); +impl Message for Payload { + type Result = (); +} +impl Drop for Payload { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +struct Done; +impl Message for Done { + type Result = (); +} + +/// Parks requests (deferred, never answered); `stopped()` waits for `gate`. +#[derive(Default)] +struct Owner { + parked: Vec>, + gate: Option<(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>)>, +} +impl Actor for Owner { + async fn stopped(&mut self, _: &Context) { + if let Some((gate, stopping)) = self.gate.take() { + let _ = stopping.send(()); + let _ = gate.recv_timeout(Duration::from_secs(10)); + } + } +} +struct Park; +impl Message for Park { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Park, reply: PendingReply<'_, ()>, _: &Context) { + self.parked.push(reply.defer().unwrap()); + } +} +impl Handler for Owner { + async fn handle(&mut self, _: Payload, _: PendingReply<'_, ()>, _: &Context) {} +} +impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} +} + +/// A job whose deadline (20 ms) takes its notification while the work runs; +/// its emit scheduled at 40 ms is then discarded on the deadline service. +async fn discard_an_emit(owner: &ActorRef, drops: &Arc) { + let payload = Payload(drops.clone()); + spawned_concurrency::__private::in_turn(owner, move |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + move |job| { + let at = Instant::now() + Duration::from_millis(40); + job.emit_at_blocking(at, payload).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + }) + .unwrap() + .await + .unwrap(); +} + +/// Two callers waiting with the same deadline (expired in one visit) and +/// one that never polls. +async fn expire_callers(owner: &ActorRef) { + let deadline = Deadline::after(Duration::from_millis(50)); + let first = owner.send_request(Park, deadline).unwrap(); + let second = owner.send_request(Park, deadline).unwrap(); + let _unpolled = owner.send_request(Park, deadline).unwrap(); + let (a, b) = tokio::join!(first, second); + assert!(matches!(a, Err(ActorError::RequestTimeout))); + assert!(matches!(b, Err(ActorError::RequestTimeout))); +} + +async fn stop(owner: ActorRef) { + owner.context().stop(); + owner.join().await; +} + +/// Expired requests: two waiting callers are woken by workers, a third +/// (never polled) has nothing to wake. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn expired_callers_are_woken_by_workers() { + let _serial = SERIAL.lock().await; + let owner = Owner::default().start(); + record().await; + expire_callers(&owner).await; + tokio::time::sleep(Duration::from_millis(100)).await; + stop(owner).await; + let events = write("expired_callers").await; + assert_eq!(count(&events, "Expire:wake"), 2, "{events:?}"); + assert_eq!(count(&events, "Expire:none"), 1, "{events:?}"); + assert!(count(&events, "Run") >= 2, "{events:?}"); + assert_eq!(count(&events, "Run"), count(&events, "Done"), "{events:?}"); +} + +/// Payloads released on the deadline service are dropped by workers while +/// the owner runs. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn released_payloads_are_dropped_by_workers() { + let _serial = SERIAL.lock().await; + let owner = Owner::default().start(); + let drops = Arc::new(AtomicUsize::new(0)); + record().await; + discard_an_emit(&owner, &drops).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + drops.load(Ordering::SeqCst), + 1, + "the discarded emit was dropped" + ); + stop(owner).await; + let events = write("released_payloads").await; + assert!(count(&events, "Defer:drop") >= 1, "{events:?}"); + assert_eq!( + count(&events, "Settle:dropped"), + count(&events, "Defer:drop"), + "{events:?}" + ); + assert_eq!(count(&events, "Settle:held"), 0, "{events:?}"); +} + +#[cfg(feature = "failpoints")] +mod with_failpoints { + use super::*; + use spawned_concurrency::failpoints::{self, Action, FailPoint, FailScenario}; + + /// A payload handed off before the owner's exit, reaching its worker + /// after the exit began (the worker is delayed at `disposal.item`), is + /// held and dropped only at the death point. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_payload_handed_off_before_exit_waits_for_the_death_point() { + let _serial = SERIAL.lock().await; + let _scenario = FailScenario::setup(); + let (release, gate) = std::sync::mpsc::channel(); + let (stopping_tx, stopping) = std::sync::mpsc::channel(); + let owner = Owner { + gate: Some((gate, stopping_tx)), + ..Owner::default() + } + .start(); + let drops = Arc::new(AtomicUsize::new(0)); + record().await; + failpoints::configure( + "disposal.item", + FailPoint::new(Action::Delay(Duration::from_millis(400))), + ) + .unwrap(); + discard_an_emit(&owner, &drops).await; + // The emit is discarded at 40 ms; its worker waits until ~440 ms. + tokio::time::sleep(Duration::from_millis(100)).await; + owner.context().stop(); + tokio::task::spawn_blocking(move || stopping.recv_timeout(Duration::from_secs(5))) + .await + .unwrap() + .expect("stopped() began"); + // The worker settles the payload while stopped() still runs. + tokio::time::sleep(Duration::from_millis(600)).await; + let before_death = drops.load(Ordering::SeqCst); + release.send(()).unwrap(); + owner.join().await; + failpoints::remove("disposal.item"); + let events = write("held_until_death").await; + assert_eq!( + before_death, 0, + "a payload handed off before the exit was dropped before the death point" + ); + assert_eq!( + drops.load(Ordering::SeqCst), + 1, + "dropped at the death point" + ); + let held = events.iter().position(|e| e == "Settle:held"); + let exit = events.iter().position(|e| e == "Exit"); + let death = events.iter().position(|e| e == "Death"); + assert!(exit < held && held < death && held.is_some(), "{events:?}"); + } + + /// Idle workers retire (keep-alive shortened) and later work starts new + /// ones. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn idle_workers_retire_and_new_ones_start() { + let _serial = SERIAL.lock().await; + let _scenario = FailScenario::setup(); + failpoints::configure("disposal.keep_alive", FailPoint::new(Action::Error)).unwrap(); + // Let the workers left by earlier scenarios retire first. + tokio::time::sleep(Duration::from_millis(100)).await; + let owner = Owner::default().start(); + record().await; + expire_callers(&owner).await; + tokio::time::sleep(Duration::from_millis(100)).await; + expire_callers(&owner).await; + tokio::time::sleep(Duration::from_millis(100)).await; + stop(owner).await; + let events = write("workers_retire").await; + failpoints::remove("disposal.keep_alive"); + assert!(count(&events, "Retire") >= 1, "{events:?}"); + assert!(count(&events, "Begin") >= 2, "{events:?}"); + } + + /// Negative control recorded from broken code: with + /// `disposal.batch_serial` the batch runs serially on one worker. The + /// callers are still woken (nothing blocks here), but the trace must be + /// rejected: a worker runs an item it never took. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn batch_serial_control() { + let _serial = SERIAL.lock().await; + let _scenario = FailScenario::setup(); + let owner = Owner::default().start(); + record().await; + failpoints::configure("disposal.batch_serial", FailPoint::new(Action::Error)).unwrap(); + expire_callers(&owner).await; + tokio::time::sleep(Duration::from_millis(100)).await; + failpoints::remove("disposal.batch_serial"); + stop(owner).await; + let events = write_to(trace_dir().join("controls"), "batch_serial").await; + assert_eq!(count(&events, "Expire:wake"), 2, "{events:?}"); + assert!(count(&events, "Run") > count(&events, "Take"), "{events:?}"); + } +} diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 2681172de..00226a2c2 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -352,6 +352,54 @@ fn drop_vs_close_vs_expiry() { }); } +/// The deadline service's expiry (`expire_collect`) versus a reply, with the +/// expiry's wake-up handed off to another thread (a disposal worker) and the +/// caller polling throughout: exactly one wins, the credit returns with the +/// transition (before the wake-up runs), and the polling caller always gets +/// the outcome (the handed-off wake-up is never lost). +#[test] +fn service_expiry_with_handed_off_wake_vs_reply() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 10); + let service = { + let (slot, pool) = (slot.clone(), pool.clone()); + loom::thread::spawn(move || { + let wake = slot.expire_collect(10); + if wake.is_some() { + assert_eq!(pool.used(), 0, "credit returned before the wake-up"); + } + wake + }) + }; + let disposal = loom::thread::spawn(move || { + if let Some(mut wake) = service.join().unwrap() { + wake.notify_parked(); + wake.run(); + true + } else { + false + } + }); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(1).0) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + let expired = disposal.join().unwrap(); + let delivery = replier.join().unwrap(); + if expired { + assert_eq!(got, Err(Terminal::TimedOut)); + assert_eq!(delivery, Delivery::Expired); + } else { + assert_eq!(got, Ok(1)); + assert_eq!(delivery, Delivery::Delivered); + } + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + /// `defer()` versus the caller abandoning: a slot that went terminal never /// acquires a credit, and a deferred one releases it on abandonment. #[test] diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 6f53c07b2..63a1548ae 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -665,12 +665,15 @@ Rules (the [design](design/deferred-replies.md) has the full contract): added later can be answered). **Deadline-service progress.** Managed and unmanaged actors share a service -thread for token/job deadlines and scheduled job emits. Expiry can run user -wakers and payload destructors there; a blocking callback delays other -actors' heap-driven expiry too. Keep these callbacks short and nonblocking. -Ordinary `request()`/`request_with_timeout()`/`Response` timers and the -`send_after`/`send_interval` drivers are separate. This is a scheduling -limitation, not a hard real-time deadline guarantee. +thread for token/job deadlines and scheduled job emits. It only decides +expiry and returns credits; the wakers and payload destructors an expiry +releases (a caller's waker, a discarded emit, a displaced job result, a +`done` closure's captures) run on separate disposal workers, so a blocking +destructor or waker in one actor does not delay another actor's deadlines. +They still occupy a worker (at most 64), so keep them short. Ordinary +`request()`/`request_with_timeout()`/`Response` timers and the +`send_after`/`send_interval` drivers are separate. Deadlines are not a hard +real-time guarantee. **Answering with a reason.** The framework's errors (`ReplyDropped`, `ActorStopped`, `Overloaded`, `RequestTimeout`) say what happened to the diff --git a/docs/TESTING.md b/docs/TESTING.md index 4adf4741d..1cb677fa7 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1882,12 +1882,16 @@ and prints the seed when a test fails. Configuration is process-global. | `link.before_install` | no | Peer exiting while a link is installed (tasks and threads) | | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | | `job.emit_due` | no | A scheduled `emit_at` coming due during the owner's exit: its payload is held until every caller is answered (`emit_due_during_exit_*`) | +| `disposal.item` | no | A disposal worker delayed before an item the deadline service handed off, so the owner's exit begins first: the payload is held until the death point (`disposal_trace`: `a_payload_handed_off_before_exit_waits_for_the_death_point`) | +| `disposal.batch_serial` | yes | Error: the deadline service's batch runs serially on one worker (the shared-batch bug); its recorded trace must be rejected (`disposal_trace`, negative control) | +| `disposal.keep_alive` | yes | Error: idle disposal workers retire after 5 ms, so the trace covers `Retire` and later starts (`disposal_trace`) | | `job.native_worker_unused` | no | Threads mode: the parked thread of a dropped, unused blocking-job permit held at its exit; the native slot is already back (`an_unused_permit_returns_its_slot_without_its_thread`) | | `reply.close_upgraded` | no | The death point's close holding an already-answered slot while its caller drops the receiver: the unread reply is destroyed only after every caller is answered (tasks and threads) | | `reply.recv_parked` | no | Handshake: a blocking `recv()` is parked, so only a notification wakes it (discharge notifies parked receivers before any waker; tasks and threads) | | `monitor.down_before_dispatch` | no | `demonitor` on another thread returning while the monitor's `Down` is in dispatch: it is flushed (tasks and threads) | -`tests/failpoints.rs` drives every site and checks invariants through public +`tests/failpoints.rs` drives every site (the `disposal.*` sites are driven by +`tests/disposal_trace.rs`) and checks invariants through public APIs: restarts follow the budget (generation count equals injected failures plus started actors), a stop during a restart or publication window starts or publishes nothing, shutdown reports at its deadline and later completes, diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 6f1e8bdeb..66625905c 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -288,10 +288,73 @@ handles a call even after the caller timed out, and the late reply is dropped. Shortening or a new earliest entry re-registers and wakes it. A dead owner is forgotten at registry close. It is independent of individual Tokio runtime lifetimes. -- **Callback progress.** Expiry may run user wakers and destroy job payloads - or completion captures on that thread. A blocking callback delays other - actors' heap-driven deadlines and credit return. Callbacks must return - promptly; the service provides no isolation from non-returning callbacks. +- **Callback progress (disposal off the service thread).** The service + thread only decides expiry: it pops due entries, makes the terminal + transition (`OPEN -> TIMED_OUT`, a job's `JobDeadline` notification or a + scheduled emit's fire) and returns the deferral credit, all under + framework locks. It also unparks a parked blocking receiver and enqueues a + job's message on its owner's mailbox (the runtime wake of the owner's own + task). Anything an expiry releases that can run user code is collected + during the visit and handed to the disposal workers (`reply/disposal.rs`): + the caller's waker and the token's watchers, payloads routed to the + owner's failure sink (a held or refused job result, a discarded emit, + `done` and its captures), a due slot that may hold an unread reply, and + the last references to jobs, runs and the registry (a reference that is + not the last, or an entry with nothing of the user's left, is dropped in + place; empty payloads, the usual case, are not handed off at all). Each + item is queued on its own, so a destructor or waker that blocks holds only + the worker running it: while fewer than 64 callbacks are blocked at once, + no other expiry, credit return or wake-up waits for it, of another actor + or of the same one (two callers of one actor expiring in one visit are + woken independently). Disposal workers form an elastic pool: when queued items + outnumber idle workers one more starts (one start at a time, up to 64; + idle ones exit after 10 s), and each worker re-checks when it takes an + item. Reaching the cap logs `disposal_workers_capped` once. Modelled in + `verification/tla/Disposal.tla`, including the pool's start rule, idle + count and cap (trace-validated against `concurrency/tests/disposal_trace.rs`, + which records every hand-off and every worker step): no wake-up before the + transition and credit return, no payload drop decided while held, and no + expiry or wake-up waiting for a blocked item below the cap; its controls + include the shared-batch bug, as a model fault and as a trace recorded + from the code with that bug switched on (`disposal.batch_serial`). + - *Why a pool and not the owner.* Deferring disposal to the owning actor + would tie a caller's wake-up to the owner's next turn (a wedged owner + would hang its callers past their deadline), and deferring it to the + caller has no thread to run on for an unpolled token. A single disposal + thread would only move the stall: a caller's waker queued behind + another actor's blocking destructor still waits. + - *Order kept.* The death point's rules hold: the transition precedes the + wake-up as before (a caller that polls in between sees the terminal + state); a payload handed off is discarded again on the worker, where the + owner's hold is re-checked, so a drop deferred before the owner's exit + still waits for its death point (R6, callers before payloads). If a + visit unwinds, what it collected is still submitted, so no wake-up is + lost. A deferral from a thread-local destructor after this thread's + batch is gone runs inline instead of aborting. + - *Credits.* Deferral credits (`MAX_OUTSTANDING`) return on the service + thread with the transition. A scheduled emit's per-job emit credit + travels inside its message; when the emit is discarded or refused at its + instant, that credit returns when the message is dropped on a worker. + - *Cost.* None on a request or reply that does not expire: the hand-off is + in the service's own path, and the shared helpers test a thread-local + only when they already have a waker or a payload to dispose of. + - *Pre-exit drops.* A worker decides a payload's fate under the hold + lock (`DropFailures::settle`) and runs the destructor after releasing + it, like an inline discard: a drop decided just before the owner's exit + began may still be running while the exit proceeds. It is pre-exit work + (§5); the exit never waits for it, since such a destructor could itself + wait for this actor's callers. The guarantee is that no drop is decided + after the hold began (the model's `NoDropWhileHeld`, on the decision). + - *What still runs on the service thread.* The owner's own executor wake + (enqueueing a job's message: a runtime waker or condvar), dropping (not + waking) a job's stored wakers when its last reference goes, the `tracing` + subscriber for a contained panic or the cap warning, a panic payload's + destructor (`dispose_panic`), and, only if no disposal thread exists and + none can be spawned, the deferred work itself. If an unwinding job + notification drops its run there, that drop is inline too. + - *Limits.* With 64 callbacks blocked at once, later items queue behind + them. A burst of quick items can start workers that then sit idle (one + start in flight at a time; idle workers exit after 10 s). Ordinary request/Response runtime timers and send_after/send_interval drivers are separate (see semantics.md, section 10). - **Simulation.** Under `spawned_sim` the service is a task of the simulated @@ -1123,7 +1186,9 @@ semantics above. service thread (not a task on the `ActorSystem` control runtime); a dead owner is forgotten at its registry close. Under `spawned_sim` each owner's heap is served by a task of the simulated runtime, so clock domains never - mix. + mix. The wakers and payload destructors an expiry releases go to an + elastic disposal pool, not the thread (§2, "Callback progress", which + lists what still runs inline). - **Cleanup deadline.** Armed when exit begins: the supervisor's `shutdown_timeout` for a supervised child, the remaining system grace during `ActorSystem::shutdown`, otherwise `DEFAULT_STOP_DEADLINE` (5 s). diff --git a/docs/design/semantics.md b/docs/design/semantics.md index da6925d0b..8711e3da3 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -160,7 +160,13 @@ the expiry runs is delivered (`Delivery::Delivered`, and the caller gets the value), as an OTP reply already in the caller's mailbox when its `after` clause would fire is received. Timer resolution and when the runtime or deadline service can process expiry determine that window; this is not a -hard real-time bound. +hard real-time bound. The deadline service hands the wakers and payload +destructors an expiry releases to disposal workers, one item at a time, so +while fewer than 64 such callbacks are blocked at once, a blocking +destructor or waker (of another actor, or of another caller of the same +actor) does not delay an expiry, its deferral-credit return or its caller's +wake-up ([`deadline_service`][deadline_service]). What still runs on the +service thread is listed in deferred-replies.md §2. Which clock expires a waiting caller is part of the contract: `request`, `request_with_timeout` and `Response::from(rx)` arm the runtime's timer, so they follow paused Tokio time; a `ReplyReceiver` awaited directly @@ -763,16 +769,18 @@ stopped after `Shutdown`), - **Preemption.** A running handler, a blocking call or a non-returning hook is never interrupted; stop, shutdown deadlines and cleanup timeouts only signal and report. Rust cannot kill a thread. -- **Deadline-service callback isolation.** Outside simulation, managed and - unmanaged actors share one deadline-service thread. Expiry can invoke user - wakers and destroy job payloads or completion captures there. A callback - that blocks delays heap-driven expiry, credit return and scheduled job - emits for other actors too. These callbacks must return promptly. Ordinary - `request`/`request_with_timeout`/`Response` waits use their runtime timer; - `send_after` and `send_interval` use separate tasks or threads. No callback - is preempted. Separating disposal from deadline bookkeeping is follow-up - work; expiry authority, terminal outcomes and R6 caller-before-payload - ordering are unchanged. +- **Deadline-service isolation past 64 blocked callbacks, and its inline + remainder.** Wakers and payload destructors an expiry releases go to a + pool of at most 64 disposal workers (section 3). With that many callbacks + blocked at once, later disposals (and so their callers' wake-ups) queue + behind them; expiry and deferral-credit return do not. A discarded + scheduled emit's per-job emit credit returns with its message's disposal, + so it can queue too. A few things still run on the service thread: the + owner's executor wake, dropping a job's stored wakers, `tracing` + subscriber calls, a panic payload's destructor, and the work itself if no + disposal thread can be spawned (deferred-replies.md §2). A payload drop + decided just before the owner's exit may still run during it (pre-exit + work; the exit never waits for it). No callback is preempted. - **Bounded memory.** Mailboxes are unbounded (D5); there are no per-actor memory limits (rank 18a). - **Deadlock detection.** A request cycle, or a handler requesting its own @@ -855,6 +863,7 @@ remaining differences of the exit-signal design are listed in See the [ranked backlog](../../ROADMAP.md#ranked-backlog). [deferred_replies]: ../../concurrency/tests/deferred_replies.rs +[deadline_service]: ../../concurrency/tests/deadline_service.rs [deferred_jobs]: ../../concurrency/tests/deferred_jobs.rs [actor_scenarios]: ../../concurrency/tests/actor_scenarios.rs [cancellation_safety]: ../../concurrency/tests/cancellation_safety.rs diff --git a/verification/model-map.toml b/verification/model-map.toml index c9f8d5b90..a485265c0 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -137,6 +137,27 @@ code = [ ] coverage_floor = 14 # of 14 action keys (TraceReplies arms) +[models.Disposal] +# The deadline service's hand-off of user code to the disposal workers: +# expiry and credit return on the service thread, per-item queueing, the +# worker steps (wake after the transition, payload drops re-checking the +# owner's hold), the elastic pool, and the owner's hold until death. +files = [ + "verification/tla/Disposal.tla", + "verification/tla/MCDisposal*", + "verification/tla/TraceDisposal.tla", +] +code = [ + "concurrency/src/reply/disposal.rs", + "concurrency/src/reply/deadline.rs", + "concurrency/src/reply/registry.rs", + "concurrency/src/mailbox.rs", + "concurrency/src/tasks/jobs.rs", # job runs' last references (`Run`) + "concurrency/src/threads/jobs.rs", + "concurrency/src/lifecycle_trace.rs", +] +coverage_floor = 13 # of 13 action keys (TraceDisposal arms) + [models.ReplyTimer] # Receiver-side refinement: the shared runtime deadline and shortening mark. files = ["verification/tla/ReplyTimer.tla", "verification/tla/MCReplyTimer*"] @@ -323,6 +344,15 @@ specs = ["verification/tla/TraceJobs.tla"] [trace.unlogged] Restart = "a new owner instance reading the old one's slot-addressed mailbox: the code has no such mailbox (a job's messages go to its owner instance's own mailbox, discarded at that instance's death), so the model's cross-instance delivery is a conservative over-approximation (NoLateResult holds a fortiori); a trace is one instance" +[[trace]] +struct = "DisposalEvent" +emitters = [ + { call = '\blifecycle_trace::disposal\(', arg = 0 }, +] +specs = ["verification/tla/TraceDisposal.tla"] +[trace.unlogged] +Commit = "fault device only (Fault = WakeBeforeTransition: the transition committed after the hand-off); the code commits in Expire" + [[trace]] struct = "ReplyEvent" emitters = [ diff --git a/verification/tla/DeferredReplies.tla b/verification/tla/DeferredReplies.tla index 9572989db..85ea83dd6 100644 --- a/verification/tla/DeferredReplies.tla +++ b/verification/tla/DeferredReplies.tla @@ -75,6 +75,9 @@ (* NoReplyFence (NoReplyAfterDeath) a reply is admitted after D *) (* CallbackBeforeDischarge (WokenAfterDischarge) an answer's wake-up *) (* runs before the last answer *) +(* InlineDisposal (TimelyExpiry) the shared deadline thread runs *) +(* an expiry's user callback inline *) +(* (0.8.0), which never returns *) (***************************************************************************) EXTENDS Naturals, FiniteSets CONSTANTS N, BoundA, BoundB, MaxTime, MaxWait, Fault @@ -82,7 +85,8 @@ ASSUME Fault \in {"None", "NoAutoReply", "NoAtomicExpiry", "LateSuccess", "LoopDeadline", "Lengthen", "Unbounded", "NoDestAdmission", "ExitStateOnly", "UnwindDropped", "ShareRunning", "AttachClosed", "LeakDownstream", "EarlyClose", - "EarlyStopped", "NoReplyFence", "CallbackBeforeDischarge"} + "EarlyStopped", "NoReplyFence", "CallbackBeforeDischarge", + "InlineDisposal"} Req == 1..N \* qA queued; turnA PendingReply in a turn; localA ReplyTo in a turn; @@ -394,8 +398,21 @@ BReply(r) == ------------------------------------------------------------------------------ \* Deadlines -Blocked(r) == Fault = "LoopDeadline" /\ turnA # 0 - /\ holder[r] \in {"qA", "turnA", "localA", "dropA", "stateA", "doneA"} +\* One process-wide thread services every owner's deadline heap (the +\* tokens A holds). It only decides expiry; what an expiry releases that can +\* run user code (the caller's waker, a payload or completion destructor) is +\* handed to a disposal worker, so Expire(r) never waits for another +\* request's callback. InlineDisposal is the 0.8.0 behaviour: the thread ran +\* those callbacks itself, and one that never returns (here: the one an +\* earlier expiry released) stops it from expiring any other token. +ServiceStuck(r) == + /\ Fault = "InlineDisposal" + /\ holder[r] \in {"localA", "dropA", "stateA", "doneA"} + /\ \E q \in Req \ {r} : slot[q] = "timeout" + +Blocked(r) == \/ /\ Fault = "LoopDeadline" /\ turnA # 0 + /\ holder[r] \in {"qA", "turnA", "localA", "dropA", "stateA", "doneA"} + \/ ServiceStuck(r) Shorten(r) == /\ Open(r) /\ holder[r] \in {"localA", "stateA", "doneA"} diff --git a/verification/tla/Disposal.tla b/verification/tla/Disposal.tla new file mode 100644 index 000000000..185b1527c --- /dev/null +++ b/verification/tla/Disposal.tla @@ -0,0 +1,374 @@ +------------------------------ MODULE Disposal ------------------------------ +(***************************************************************************) +(* The deadline service's hand-off of user code to disposal workers *) +(* (concurrency/src/reply/disposal.rs, deadline.rs, registry.rs, and *) +(* DropFailures in mailbox.rs; docs/design/deferred-replies.md §2, *) +(* "Callback progress"). A companion of DeferredReplies.tla, which models *) +(* the reply slots themselves and takes the service's expiry as one step. *) +(* *) +(* One owner's deadline heap holds N requests (deferred tokens or awaited *) +(* raw receivers), each with a deadline fixed at Init and possibly a *) +(* waiting caller (a waker). The one process-wide service thread visits *) +(* the heap and collects what it releases in the visit's batch: *) +(* Expire(r) the slot's OPEN -> TIMED_OUT transition and its credit *) +(* return, in one step; a waiting caller's wake-up item *) +(* Release(p) a payload (a discarded scheduled emit, a displaced job *) +(* result, `done` and its captures); one released while the *) +(* owner's hold is on is held at once and is no step *) +(* Other(o) anything else whose drop or call may run user code (the *) +(* last reference to a job or registry, a job's emitter *) +(* wakers, a slot holding an unread reply) *) +(* Submit the batch goes to the pool, one queue entry per item; *) +(* then the start rule *) +(* The pool mirrors run()/submit(): a worker is "none" (not started), *) +(* "active" (in its loop, holding the lock), "idle" (waiting: counted in *) +(* `idle`), "taken" (holds an entry), "in" (runs its next item) or *) +(* "settled" (a payload's hold check is done, its destructor runs). The *) +(* start rule (`needs_worker`): when queued entries outnumber idle *) +(* workers, none is starting and fewer than Workers exist (the cap), one *) +(* starts; it is checked after Submit and after every Take. A payload's *) +(* run is two steps, as `DropFailures::settle`: Settle decides under the *) +(* hold lock (held if the owner's exit began, else dropped) and the *) +(* destructor then runs until Done. Items in Blocking never reach Done (a *) +(* blocking waker or destructor): their worker is lost. *) +(* *) +(* The owner: Exit (DropFailures::hold, its exit begins) and Death (the *) +(* death point; DropFailures::release drops what was held). A destructor *) +(* whose drop was decided before Exit may still run during the exit: it *) +(* is pre-exit work and Exit never waits for it (deferred-replies.md §5); *) +(* NoDropWhileHeld is about the decision. *) +(* *) +(* The clock: Tick only when no framework step is pending (an expiry due, *) +(* a batch not submitted, a worker that can start, take or run), so a *) +(* property "within the tick" says the step does not wait for a blocked *) +(* item. *) +(* *) +(* Fault selects a negative control; negative_checks.py requires each to *) +(* violate the named invariant: *) +(* InlineDisposal (TimelyExpiry) the service runs the batch *) +(* itself (0.8.0) *) +(* WakeBeforeTransition (WakeAfterTransition) the wake-up is handed off *) +(* before the transition commits *) +(* DropWhileHeld (NoDropWhileHeld) the worker drops a payload *) +(* without re-checking the hold *) +(* BatchSerial (TimelyWake) a visit's batch is one entry *) +(* run serially (the shared-batch *) +(* bug of PR #282's first rev.) *) +(* NoStartOnTake (TimelyWake) the start rule is checked only *) +(* at Submit: a queued item can be *) +(* stranded behind blocked workers *) +(***************************************************************************) +EXTENDS Naturals, Sequences, FiniteSets + +CONSTANTS N, P, O, MaxTime, Workers, Blocking, Fault, InitIdle +ASSUME Fault \in {"None", "InlineDisposal", "WakeBeforeTransition", + "DropWhileHeld", "BatchSerial", "NoStartOnTake"} + +Req == 1..N +Pay == 1..P +Oth == 1..O +W == 1..Workers +ASSUME InitIdle \subseteq W +WakeItem(r) == [k |-> "wake", id |-> r] +DropItem(p) == [k |-> "drop", id |-> p] +OtherItem(o) == [k |-> "other", id |-> o] +Items == {WakeItem(r) : r \in Req} \cup {DropItem(p) : p \in Pay} + \cup {OtherItem(o) : o \in Oth} +ASSUME Blocking \subseteq Items + +VARIABLES + now, dl, waiting, \* the clock; each request's deadline and caller + slot, credit, expAt, \* "open"/"timeout"; its credit; when it expired + woken, \* the caller's wake-up ran + cas, \* WakeBeforeTransition: handed off, not committed + payload, \* "none", "out", "held", "dropping", "dropped" + other, \* "none", "out", "done" + batch, \* the current visit's collected items + queue, \* queued entries (each a sequence of items) + ws, \* each worker's state (see above) + cur, \* the entry a worker holds (<<>> if none) + starting, \* a worker start was decided, not yet begun + svc, \* "idle", or "stuck" in a blocking item (InlineDisposal) + phase, \* the owner: "run", "exiting" (hold on), "dead" + early \* a payload's drop was decided while the hold was on + +vars == <> +pool == <> + +Init == + /\ now = 0 + /\ dl \in [Req -> 1..MaxTime] + /\ waiting \in [Req -> BOOLEAN] + /\ slot = [r \in Req |-> "open"] + /\ credit = [r \in Req |-> TRUE] + /\ expAt = [r \in Req |-> 0] + /\ woken = [r \in Req |-> FALSE] + /\ cas = {} + /\ payload = [p \in Pay |-> "none"] + /\ other = [o \in Oth |-> "none"] + /\ batch = <<>> + /\ queue = <<>> + /\ ws = [w \in W |-> IF w \in InitIdle THEN "idle" ELSE "none"] + /\ cur = [w \in W |-> <<>>] + /\ starting = FALSE + /\ svc = "idle" + /\ phase = "run" + /\ early = FALSE + +------------------------------------------------------------------------------ +\* The pool's bookkeeping (disposal.rs `State`) + +IdleCount == Cardinality({w \in W : ws[w] = "idle"}) +Started == {w \in W : ws[w] # "none"} +\* `needs_worker` on queue `q` with `idle` idle workers. +NeedsWorker(q, idle) == + /\ Len(q) > idle /\ ~starting + /\ Cardinality(Started) < Workers + +------------------------------------------------------------------------------ +\* The service thread + +RunInline(item) == IF item \in Blocking THEN svc' = "stuck" ELSE UNCHANGED svc + +Due(r) == slot[r] = "open" /\ r \notin cas /\ now >= dl[r] + +Expire(r) == + /\ svc = "idle" /\ Due(r) + /\ IF Fault = "WakeBeforeTransition" + THEN /\ cas' = cas \cup {r} + /\ batch' = IF waiting[r] THEN Append(batch, WakeItem(r)) ELSE batch + /\ UNCHANGED <> + ELSE /\ slot' = [slot EXCEPT ![r] = "timeout"] + /\ credit' = [credit EXCEPT ![r] = FALSE] + /\ expAt' = [expAt EXCEPT ![r] = now] + /\ UNCHANGED cas + /\ IF Fault = "InlineDisposal" + THEN /\ UNCHANGED batch + /\ IF waiting[r] /\ WakeItem(r) \notin Blocking + THEN woken' = [woken EXCEPT ![r] = TRUE] ELSE UNCHANGED woken + /\ IF waiting[r] THEN RunInline(WakeItem(r)) ELSE UNCHANGED svc + ELSE /\ batch' = IF waiting[r] THEN Append(batch, WakeItem(r)) ELSE batch + /\ UNCHANGED <> + /\ UNCHANGED <> + +\* WakeBeforeTransition only: the transition commits after the hand-off. +Commit(r) == + /\ r \in cas + /\ cas' = cas \ {r} + /\ slot' = [slot EXCEPT ![r] = "timeout"] + /\ credit' = [credit EXCEPT ![r] = FALSE] + /\ expAt' = [expAt EXCEPT ![r] = now] + /\ UNCHANGED <> + +Release(p) == + /\ svc = "idle" /\ payload[p] = "none" /\ phase # "exiting" + /\ IF Fault = "InlineDisposal" + THEN /\ payload' = [payload EXCEPT ![p] = IF DropItem(p) \in Blocking THEN "dropping" ELSE "dropped"] + /\ RunInline(DropItem(p)) + /\ UNCHANGED batch + ELSE /\ payload' = [payload EXCEPT ![p] = "out"] + /\ batch' = Append(batch, DropItem(p)) + /\ UNCHANGED svc + /\ UNCHANGED <> + +Other(o) == + /\ svc = "idle" /\ other[o] = "none" + /\ IF Fault = "InlineDisposal" + THEN /\ other' = [other EXCEPT ![o] = IF OtherItem(o) \in Blocking THEN "out" ELSE "done"] + /\ RunInline(OtherItem(o)) + /\ UNCHANGED batch + ELSE /\ other' = [other EXCEPT ![o] = "out"] + /\ batch' = Append(batch, OtherItem(o)) + /\ UNCHANGED svc + /\ UNCHANGED <> + +\* One queue entry per item; BatchSerial: the whole batch as one entry. +Entries(b) == IF Fault = "BatchSerial" THEN <> ELSE [i \in 1..Len(b) |-> <>] + +\* `spawn`: whether the code decided to start a worker (the trace's x). +SubmitDecide(spawn) == + /\ batch # <<>> + /\ queue' = queue \o Entries(batch) + /\ batch' = <<>> + /\ starting' = (starting \/ NeedsWorker(queue', IdleCount)) + /\ spawn = (starting' /\ ~starting) + /\ UNCHANGED <> +Submit == \E spawn \in BOOLEAN : SubmitDecide(spawn) + +------------------------------------------------------------------------------ +\* Disposal workers (disposal.rs `run`) + +\* A started thread enters its loop (`starting = false`). +Begin(w) == + /\ ws[w] = "none" /\ starting + /\ ws' = [ws EXCEPT ![w] = "active"] /\ starting' = FALSE + /\ UNCHANGED <> + +\* Pop an entry; then the start rule for what is still queued. +TakeDecide(w, spawn) == + /\ ws[w] = "active" /\ queue # <<>> + /\ ws' = [ws EXCEPT ![w] = "taken"] + /\ cur' = [cur EXCEPT ![w] = Head(queue)] + /\ queue' = Tail(queue) + /\ starting' = (starting \/ (Fault # "NoStartOnTake" /\ NeedsWorker(Tail(queue), IdleCount))) + /\ spawn = (starting' /\ ~starting) + /\ UNCHANGED <> +Take(w) == \E spawn \in BOOLEAN : TakeDecide(w, spawn) + +\* Nothing queued: wait (counted idle). +Wait(w) == + /\ ws[w] = "active" /\ queue = <<>> + /\ ws' = [ws EXCEPT ![w] = "idle"] + /\ UNCHANGED <> + +\* Back under the lock after a notification, a time-out or spuriously. +Woke(w) == + /\ ws[w] = "idle" + /\ ws' = [ws EXCEPT ![w] = "active"] + /\ UNCHANGED <> + +Retire(w) == + /\ ws[w] = "active" /\ queue = <<>> + /\ ws' = [ws EXCEPT ![w] = "none"] + /\ UNCHANGED <> + +\* Start the entry's next item: a wake-up wakes its caller here. +Run(w) == + /\ ws[w] = "taken" + /\ ws' = [ws EXCEPT ![w] = "in"] + /\ LET item == Head(cur[w]) IN + IF item.k = "wake" THEN woken' = [woken EXCEPT ![item.id] = TRUE] + ELSE UNCHANGED woken + /\ UNCHANGED <> + +\* A payload's hold check (`DropFailures::settle`, under the hold lock). +SettleTo(w, x) == + /\ ws[w] = "in" /\ Head(cur[w]).k = "drop" + /\ LET p == Head(cur[w]).id + held == phase = "exiting" /\ Fault # "DropWhileHeld" IN + /\ x = IF held THEN "held" ELSE "dropped" + /\ payload' = [payload EXCEPT ![p] = IF held THEN "held" ELSE "dropping"] + /\ early' = (early \/ (~held /\ phase = "exiting")) + /\ ws' = [ws EXCEPT ![w] = "settled"] + /\ UNCHANGED <> +Settle(w) == \E x \in {"held", "dropped"} : SettleTo(w, x) + +\* The item returns (a blocking one never does). +Done(w) == + /\ \/ ws[w] = "in" /\ Head(cur[w]).k # "drop" + \/ ws[w] = "settled" + /\ LET item == Head(cur[w]) IN + /\ item \notin Blocking + /\ IF item.k = "drop" /\ payload[item.id] = "dropping" + THEN payload' = [payload EXCEPT ![item.id] = "dropped"] ELSE UNCHANGED payload + /\ IF item.k = "other" THEN other' = [other EXCEPT ![item.id] = "done"] ELSE UNCHANGED other + /\ cur' = [cur EXCEPT ![w] = Tail(cur[w])] + /\ ws' = [ws EXCEPT ![w] = IF Tail(cur[w]) = <<>> THEN "active" ELSE "taken"] + /\ UNCHANGED <> + +------------------------------------------------------------------------------ +\* The owner + +Exit == + /\ phase = "run" /\ phase' = "exiting" + /\ UNCHANGED <> + +\* The death point: what was held is dropped. +Death == + /\ phase = "exiting" /\ phase' = "dead" + /\ payload' = [p \in Pay |-> IF payload[p] = "held" THEN "dropped" ELSE payload[p]] + /\ UNCHANGED <> + +------------------------------------------------------------------------------ +\* The clock + +Stuck(w) == ws[w] \in {"in", "settled"} /\ Head(cur[w]) \in Blocking + +\* A framework step is pending (it never waits for user code). +Pending == + \/ svc = "idle" /\ \E r \in Req : Due(r) + \/ cas # {} + \/ batch # <<>> + \/ starting + \/ \E w \in W : ws[w] \in {"active", "taken"} + \/ \E w \in W : ws[w] \in {"in", "settled"} /\ ~Stuck(w) + \/ queue # <<>> /\ \E w \in W : ws[w] = "idle" + +Tick == + /\ now < MaxTime /\ ~Pending + /\ now' = now + 1 + /\ UNCHANGED <> + +Next == + \/ Tick \/ Submit \/ Exit \/ Death + \/ \E r \in Req : Expire(r) \/ Commit(r) + \/ \E p \in Pay : Release(p) + \/ \E o \in Oth : Other(o) + \/ \E w \in W : Begin(w) \/ Take(w) \/ Wait(w) \/ Woke(w) \/ Retire(w) + \/ Run(w) \/ Settle(w) \/ Done(w) + +Spec == Init /\ [][Next]_vars + +\* Fairness for the clock, the service and the workers (framework code; an +\* idle worker is notified when work is queued); none for the owner, for +\* Retire, or for a blocking item (Done is not enabled for it). +FairSpec == + /\ Spec /\ WF_vars(Tick) /\ WF_vars(Submit) + /\ \A r \in Req : WF_vars(Expire(r)) /\ WF_vars(Commit(r)) + /\ \A w \in W : /\ WF_vars(Begin(w)) /\ WF_vars(Take(w)) /\ WF_vars(Wait(w)) + /\ WF_vars(Woke(w) /\ queue # <<>>) + /\ WF_vars(Run(w)) /\ WF_vars(Settle(w)) /\ WF_vars(Done(w)) + +------------------------------------------------------------------------------ +\* Properties + +TypeOK == + /\ now \in 0..MaxTime + /\ slot \in [Req -> {"open", "timeout"}] + /\ credit \in [Req -> BOOLEAN] /\ woken \in [Req -> BOOLEAN] + /\ payload \in [Pay -> {"none", "out", "held", "dropping", "dropped"}] + /\ other \in [Oth -> {"none", "out", "done"}] + /\ ws \in [W -> {"none", "active", "idle", "taken", "in", "settled"}] + /\ cas \subseteq Req /\ starting \in BOOLEAN + /\ phase \in {"run", "exiting", "dead"} /\ svc \in {"idle", "stuck"} + +\* A deferral credit is held exactly while the slot is open: it returns with +\* the transition, on the service thread, never with a disposal. +CreditWithTransition == \A r \in Req : credit[r] = (slot[r] = "open") +\* No wake-up runs before its slot's transition and credit return. +WakeAfterTransition == \A r \in Req : woken[r] => (slot[r] = "timeout" /\ ~credit[r]) +\* No handed-off payload's drop is decided while the owner's hold is on: it +\* is held and dropped at the death point (R6, callers before payloads). +NoDropWhileHeld == ~early +\* The cap: never more than Workers started or starting. +WithinCap == Cardinality(Started) + (IF starting THEN 1 ELSE 0) <= Workers +\* The service does not wait for user code. +TimelyExpiry == \A r \in Req : slot[r] = "open" => now <= dl[r] +\* A wake-up that does not itself block runs within the tick of its expiry, +\* whatever else blocks (another item of the same visit included). +TimelyWake == + \A r \in Req : (slot[r] = "timeout" /\ waiting[r] /\ WakeItem(r) \notin Blocking + /\ now > expAt[r]) => woken[r] +\* At the cap: a late wake-up only while every worker is stuck in a +\* blocking item (MCDisposalCapped). +TimelyWakeBelowCap == + \A r \in Req : (slot[r] = "timeout" /\ waiting[r] /\ WakeItem(r) \notin Blocking + /\ now > expAt[r] /\ ~woken[r]) + => \A w \in W : Stuck(w) + +Safety == + /\ TypeOK /\ CreditWithTransition /\ WakeAfterTransition /\ NoDropWhileHeld + /\ WithinCap /\ TimelyExpiry /\ TimelyWake + +SafetyCapped == + /\ TypeOK /\ CreditWithTransition /\ WakeAfterTransition /\ NoDropWhileHeld + /\ WithinCap /\ TimelyExpiry /\ TimelyWakeBelowCap + +\* Every handed-off wake-up that does not block eventually runs. +EveryWakeRuns == + \A r \in Req : (slot[r] = "timeout" /\ waiting[r] /\ WakeItem(r) \notin Blocking) + ~> woken[r] +\* Every deadline is eventually serviced. +EveryDueExpires == \A r \in Req : (now >= dl[r]) ~> (slot[r] = "timeout") +============================================================================= diff --git a/verification/tla/MCDisposal.cfg b/verification/tla/MCDisposal.cfg new file mode 100644 index 000000000..13632ee8c --- /dev/null +++ b/verification/tla/MCDisposal.cfg @@ -0,0 +1,15 @@ +\* Fast instance (CI, every PR): two requests and one payload of one owner, +\* three workers, clock to 2; the first caller's waker blocks forever. +SPECIFICATION FairSpec +INVARIANT Safety +PROPERTY EveryWakeRuns +PROPERTY EveryDueExpires +CONSTANTS + N = 2 + P = 1 + O = 0 + MaxTime = 2 + Workers = 3 + InitIdle = {} + Blocking <- MCBlocking + Fault = "None" diff --git a/verification/tla/MCDisposal.tla b/verification/tla/MCDisposal.tla new file mode 100644 index 000000000..64ccdfae3 --- /dev/null +++ b/verification/tla/MCDisposal.tla @@ -0,0 +1,5 @@ +----------------------------- MODULE MCDisposal ----------------------------- +EXTENDS Disposal +\* The first caller's waker never returns. +MCBlocking == {WakeItem(1)} +============================================================================= diff --git a/verification/tla/MCDisposalCapped.cfg b/verification/tla/MCDisposalCapped.cfg new file mode 100644 index 000000000..b87a54b38 --- /dev/null +++ b/verification/tla/MCDisposalCapped.cfg @@ -0,0 +1,14 @@ +\* At the cap (CI, every PR): one worker, the first caller's waker blocks. +\* A later wake-up may wait only while every worker is stuck. +SPECIFICATION FairSpec +INVARIANT SafetyCapped +PROPERTY EveryDueExpires +CONSTANTS + N = 2 + P = 0 + O = 0 + MaxTime = 2 + Workers = 1 + InitIdle = {} + Blocking <- MCBlocking + Fault = "None" diff --git a/verification/tla/MCDisposalCapped.tla b/verification/tla/MCDisposalCapped.tla new file mode 100644 index 000000000..c66b1e9c0 --- /dev/null +++ b/verification/tla/MCDisposalCapped.tla @@ -0,0 +1,5 @@ +-------------------------- MODULE MCDisposalCapped -------------------------- +EXTENDS Disposal +\* The first caller's waker never returns, and there is one worker. +MCBlocking == {WakeItem(1)} +============================================================================= diff --git a/verification/tla/MCDisposalDrop.cfg b/verification/tla/MCDisposalDrop.cfg new file mode 100644 index 000000000..bd1767008 --- /dev/null +++ b/verification/tla/MCDisposalDrop.cfg @@ -0,0 +1,15 @@ +\* Fast instance (CI, every PR): a blocking payload destructor; one request, +\* two payloads, one other item, three workers, clock to 2. +SPECIFICATION FairSpec +INVARIANT Safety +PROPERTY EveryWakeRuns +PROPERTY EveryDueExpires +CONSTANTS + N = 1 + P = 2 + O = 1 + MaxTime = 2 + Workers = 3 + InitIdle = {} + Blocking <- MCBlocking + Fault = "None" diff --git a/verification/tla/MCDisposalDrop.tla b/verification/tla/MCDisposalDrop.tla new file mode 100644 index 000000000..a529d2232 --- /dev/null +++ b/verification/tla/MCDisposalDrop.tla @@ -0,0 +1,5 @@ +--------------------------- MODULE MCDisposalDrop --------------------------- +EXTENDS Disposal +\* The first payload's destructor never returns. +MCBlocking == {DropItem(1)} +============================================================================= diff --git a/verification/tla/README.md b/verification/tla/README.md index 68df02249..6b0afb50d 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -17,6 +17,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `TraceCoordinator.tla` | Trace validation of `Coordinator.tla`: replays the shutdown coordinator events recorded from real `ActorSystem::shutdown` runs (`coordinator_trace`). | | `NestedSupervision.tla`, `MCNested.tla/.cfg` | Dynamic subtree ownership and shutdown delivery; parent scope completion waits for descendant completion. Bounded three-node/two-generation model with fair cleanup. | | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with exit-pending drops (unwound or dropped while exiting, answered at death with the final reason), a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Fourteen named negative controls. | +| `Disposal.tla`, `MCDisposal{,Drop,Capped}.tla/.cfg`, `TraceDisposal.tla` | The deadline service's hand-off of user code to disposal workers (companion of `DeferredReplies.tla`, which takes the service's expiry as one step): per-entry `Expire` (transition and credit return on the service thread), payload `Release` and `Other` items, `Submit` one queue entry per item, and the pool as `disposal.rs` runs it: worker states (active, idle, holding an entry, running an item), the idle count, the start rule after `Submit` and every `Take` (one start in flight, up to the cap), `Wait`/`Woke`/`Retire`; wake-ups run by workers; payload drops in two steps (`Settle` decides under the hold lock: held until `Death`, or dropped; then the destructor until `Done`); items that never return (`Blocking`). Invariants: credit with the transition, no wake-up before the transition, no payload drop decided while held, the cap, timely expiry and timely wake-up whatever blocks (`MCDisposalCapped`, one worker: late only while every worker is stuck); liveness: every non-blocking wake-up runs, every deadline is serviced. Five named negative controls; trace-validated, including a trace recorded from code with the shared-batch bug. See [Disposal](#disposal-trace-validation). | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits; `MCDeferredJobsHeld` the one with delayed emits (`emit_at`) and held results (completion, held notification, scheduled emits and credit ownership as separate steps). Twelve named negative controls; `MCDeferredJobsEmitRace` checks retained emit admission at held-result release (306 states). | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Nineteen negative-control faults (19 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point, a parent exit reaching `exit_received`, an exit signal past the sender's death point (`NoSignalFence`) and publication before the queued requests were answered (`PublishBeforeDischarge`). Death is two steps, as in `actor_cleanup.rs`: the death point (`Die`/`RuntimeDrop`: signal fence, seal, table closed) and `Discharge` (queued and exit-pending requests answered), with other actors' steps in between; `Publish` waits for `Discharge`. Per PR: `MCLifecycle` (10.0M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance); manual pre-release: `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | | `Start.tla`, `MCStart{,Names,Cap,Timeout,Continue,Scope,Live,Deep}.tla/.cfg` | Unified start API, 0.9 design (approved after three review rounds) ([design](../../docs/design/unified-start.md), not implemented; TLC-only in `model-map.toml` until trace validation lands with the implementation): `spawn`, `start`, `start_link`, `start_monitor`; placement inheritance; an atomic stage-1 commit with the owner gate; an immutable startup commitment and a waiter; death as D, release, then publication, for the child and the parent; the final reason fixed at D; fallible init with held-signal precedence; the continuation; S8 scope ownership with the polite Shutdown and the scope deadline. 40 negative controls (37 safety, 3 liveness); see [Unified start](#unified-start-09-design). Per PR: `MCStart`, `MCStartNames`, `MCStartCap`, `MCStartTimeout`, `MCStartContinue`, `MCStartScope`, `MCStartLive` (CI leg `start`); weekly: `MCStartDeep`. | @@ -174,12 +175,12 @@ coverage asks the converse: which parts of the model's reachable behavior do the recorded runs never exercise? `trace_coverage.py` reports it per model with trace validation (`Supervision`, `Cohorts`, `NestedSupervision`, and by action keys only `Lifecycle`, `DeferredJobs`, `DeferredReplies`, -`Coordinator`). +`Coordinator`, `Disposal`). **Action-only models.** For `Lifecycle.tla`, `DeferredJobs.tla`, -`DeferredReplies.tla` and `Coordinator.tla` the keys are the log-event arms of -their trace spec (`TraceLifecycle.tla`, `TraceJobs.tla`, `TraceReplies.tla`, -`TraceCoordinator.tla`; each arm is one +`DeferredReplies.tla`, `Coordinator.tla` and `Disposal.tla` the keys are the +log-event arms of their trace spec (`TraceLifecycle.tla`, `TraceJobs.tla`, +`TraceReplies.tla`, `TraceCoordinator.tla`, `TraceDisposal.tla`; each arm is one model action or a named group of them, `A|Send`, `A|Fire`, ...): the model side is the set of arms, a trace covers the arms its accepted replay takes, and no state space is explored for them. `coverage-baseline.json` and the @@ -188,7 +189,7 @@ Model actions without an arm are listed in the map's `[trace.unlogged]` with the reason. ```bash -cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace +cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test disposal_trace --test lifecycle_trace nix develop -c python3 verification/tla/trace_coverage.py --traces dedicated=target/lifecycle-traces \ --model-keys verification/tla/coverage-model.json.gz --baseline verification/tla/coverage-baseline.json # every test binary: each finished supervisor dumps its traces @@ -709,6 +710,12 @@ exactly the property it targets: | `DeferredReplies` | `EarlyStopped` (rank 11 before 5e: a token dropped while exiting or unwound answers `Stopped` at once) | `StoppedAtDeath` | | `DeferredReplies` | `NoReplyFence` (a stored token's reply is admitted after the death point) | `NoReplyAfterDeath` | | `DeferredReplies` | `CallbackBeforeDischarge` (an answer's wake-up runs before the last answer of the death batch) | `WokenAfterDischarge` | +| `DeferredReplies` | `InlineDisposal` (0.8.0: the shared deadline thread runs an expiry's waker or destructor inline, and one never returns) | `TimelyExpiry` | +| `Disposal` (`MCDisposal`) | `InlineDisposal` (0.8.0: the service runs the batch itself; a blocking item stops every later expiry) | `TimelyExpiry` | +| `Disposal` (`MCDisposal`) | `WakeBeforeTransition` (the wake-up handed off before the transition commits) | `WakeAfterTransition` | +| `Disposal` (`MCDisposal`) | `DropWhileHeld` (the worker drops a payload without re-checking the owner's hold) | `NoDropWhileHeld` | +| `Disposal` (`MCDisposal`) | `BatchSerial` (a visit's batch is one queue entry run serially; the shared-batch bug of PR #282's first revision) | `TimelyWake` | +| `Disposal` (`MCDisposal`) | `NoStartOnTake` (the start rule checked only at `Submit`: a queued item stranded behind blocked workers) | `TimelyWake` | | `DeferredJobs` | `NoInstanceCheck` | `NoLateResult` | | `DeferredJobs` | `NoCancelFence` | `NoStaleAfterCancel` | | `DeferredJobs` | `DoubleTerminal` | `AtMostOneTerminal` | @@ -795,6 +802,41 @@ below the bound, a reply after death, a second discharge, a drop before the turn, and a `Stopped` discharge of the panicking turn's token before its `Exit`, the rank 11 behaviour before 5e). +### Disposal trace validation + +While `lifecycle_trace::record_disposal()` is on, the code records every +`Disposal.tla` action at its linearization point. The first event, `Pool`, +is the pool's state when recording starts (the checker requires it +quiescent and makes its idle workers the model's `InitIdle`). On the +service thread: `Expire` (after `Slot::expire_collect`'s transition and +credit return; `x` says whether a wake-up was handed off), `Defer` (every +other hand-off: a payload, `x: drop`, in one section with `DropFailures`' +hold check; anything else, `x: other`: a last reference, emitter wakers, a +slot with an unread reply) and `Submit` (the items queued, `x: spawn` if +the start rule started a worker). In the pool, under its lock: `Begin`, +`Take` (with the start decision), `Wait`, `Woke` and `Retire`; outside it: +`Run`, `Settle` (a payload's hold check, `x: held` or `dropped`) and `Done`. +The owner: `Exit` (`DropFailures::hold`) and `Death` (`DropFailures::release`). +Workers and items carry ids; the checker numbers them. Only the clock is +silent; `Commit` (the `WakeBeforeTransition` fault device) has no event. +Because every worker step names its worker and item, a worker running an +item it never took (serial execution of a batch) is rejected, and because +the start decisions are recorded and the model's start rule must agree, a +start that the rule requires but the code skips (stranding) is rejected. + +Recording is process-wide, so `concurrency/tests/disposal_trace.rs` runs its +scenarios one at a time: expired callers woken by workers (two waiting, one +never polled), payloads released by a job deadline and a discarded scheduled +emit and dropped by workers, a payload handed off before the owner's exit +whose worker is delayed (`disposal.item` failpoint) until `stopped()` runs +(held, and dropped at the death point), and idle workers retiring +(`disposal.keep_alive`) and new ones starting. A control is recorded from +the code with the shared-batch bug switched back on (`disposal.batch_serial`) +into `controls/`; `negative_checks.py` requires it to be rejected, along with +five edited traces (a wake-up before its expiry, a payload run before the +hand-off, a handed-off payload dropped while held, a required worker start +not taken, and a worker running an item it never took). + ## Unified start (0.9 design) `Start.tla` backs the [unified start design](../../docs/design/unified-start.md) diff --git a/verification/tla/TraceDisposal.tla b/verification/tla/TraceDisposal.tla new file mode 100644 index 000000000..0ddf8798e --- /dev/null +++ b/verification/tla/TraceDisposal.tla @@ -0,0 +1,73 @@ +----------------------------- MODULE TraceDisposal ----------------------------- +(***************************************************************************) +(* Trace validation for Disposal.tla: the deadline service's hand-off of *) +(* wake-ups and payloads to the disposal workers, the pool's bookkeeping, *) +(* and the owner's hold. *) +(* *) +(* With the `lifecycle-trace` feature, while `record_disposal` is on, the *) +(* code records (concurrency/src/lifecycle_trace.rs, DisposalEvent): *) +(* on the service thread *) +(* Expire a slot's OPEN -> TIMED_OUT and credit return, in a visit *) +(* (registry.rs); x: "wake" if its wake-up was handed off *) +(* (Expire, with the caller's `waiting` matching x) *) +(* Defer a payload (x: "drop", DropFailures::discard, in one section *) +(* with the hold check) or another item (x: "other": the last *) +(* reference to a job or registry, emitter wakers, a slot with *) +(* an unread reply) handed off (Release, Other) *) +(* Submit the visit's items queued and the start rule (x: "spawn" if *) +(* a worker start was decided) (Submit) *) +(* in the pool (disposal.rs `run`, each under the pool lock except Run, *) +(* Settle and Done) *) +(* Begin, Take (x as for Submit), Wait, Woke, Retire (same names) *) +(* Run the worker starts its item (a wake-up wakes its caller) (Run)*) +(* Settle a payload's hold check, x: "held" or "dropped" (Settle) *) +(* Done the item returned (Done) *) +(* and the owner: Exit (DropFailures::hold) and Death (release). *) +(* *) +(* trace_check.py turns the leading `Pool` event (the pool when recording *) +(* starts: its idle workers) into InitIdle, numbers requests, payloads, *) +(* other items and workers, and sets Workers to the code's cap. The clock *) +(* is silent. No item blocks in a recorded run (Blocking = {}). *) +(***************************************************************************) +EXTENDS Disposal, TLC + +CONSTANT Trace + +VARIABLE i + +traceVars == <> + +TraceInit == Init /\ i = 0 + +Item(e) == [k |-> e.k, id |-> e.n] + +Step(e) == + CASE e.a = "Expire" -> e.n \in Req /\ waiting[e.n] = (e.x = "wake") /\ Expire(e.n) + [] e.a = "Defer" -> \/ e.k = "drop" /\ e.n \in Pay /\ Release(e.n) + \/ e.k = "other" /\ e.n \in Oth /\ Other(e.n) + [] e.a = "Submit" -> SubmitDecide(e.x = "spawn") + [] e.a = "Begin" -> e.w \in W /\ Begin(e.w) + [] e.a = "Take" -> e.w \in W /\ queue # <<>> /\ Head(queue) = <> + /\ TakeDecide(e.w, e.x = "spawn") + [] e.a = "Wait" -> e.w \in W /\ Wait(e.w) + [] e.a = "Woke" -> e.w \in W /\ Woke(e.w) + [] e.a = "Retire" -> e.w \in W /\ Retire(e.w) + [] e.a = "Run" -> e.w \in W /\ cur[e.w] # <<>> /\ Head(cur[e.w]) = Item(e) /\ Run(e.w) + [] e.a = "Settle" -> e.w \in W /\ cur[e.w] # <<>> /\ Head(cur[e.w]) = Item(e) + /\ SettleTo(e.w, e.x) + [] e.a = "Done" -> e.w \in W /\ cur[e.w] # <<>> /\ Head(cur[e.w]) = Item(e) /\ Done(e.w) + [] e.a = "Exit" -> Exit + [] e.a = "Death" -> Death + [] OTHER -> FALSE + +TraceNext == + /\ i < Len(Trace) + /\ \/ Step(Trace[i + 1]) /\ i' = i + 1 + \/ Tick /\ UNCHANGED i + +TraceSpec == TraceInit /\ [][TraceNext]_traceVars + +\* Violated exactly when the whole log has been matched: TLC reporting this +\* invariant as violated means the trace is ACCEPTED. +TraceIncomplete == i < Len(Trace) +============================================================================= diff --git a/verification/tla/coverage-baseline.json b/verification/tla/coverage-baseline.json index fea56ae28..8409ffb20 100644 --- a/verification/tla/coverage-baseline.json +++ b/verification/tla/coverage-baseline.json @@ -149,5 +149,20 @@ "A|Join", "A|Ordered", "A|StopNext" + ], + "Disposal": [ + "A|Begin", + "A|Death", + "A|Defer", + "A|Done", + "A|Exit", + "A|Expire", + "A|Retire", + "A|Run", + "A|Settle", + "A|Submit", + "A|Take", + "A|Wait", + "A|Woke" ] } diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 8004d64d2..f4e6ee91b 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -282,6 +282,12 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDeferredReplies", "EarlyStopped", "StoppedAtDeath"), ("MCDeferredReplies", "NoReplyFence", "NoReplyAfterDeath"), ("MCDeferredReplies", "CallbackBeforeDischarge", "WokenAfterDischarge"), + ("MCDeferredReplies", "InlineDisposal", "TimelyExpiry"), + ("MCDisposal", "InlineDisposal", "TimelyExpiry"), + ("MCDisposal", "WakeBeforeTransition", "WakeAfterTransition"), + ("MCDisposal", "DropWhileHeld", "NoDropWhileHeld"), + ("MCDisposal", "BatchSerial", "TimelyWake"), + ("MCDisposal", "NoStartOnTake", "TimelyWake"), ("MCDeferredJobs", "NoInstanceCheck", "NoLateResult"), ("MCDeferredJobs", "NoCancelFence", "NoStaleAfterCancel"), ("MCDeferredJobs", "DoubleTerminal", "AtMostOneTerminal"), @@ -376,6 +382,52 @@ def start_controls(tlc): ] +def disposal_trace_controls(traces, tlc, directory): + """Disposal traces Disposal.tla must reject: edits of recorded traces, + and traces recorded from deliberately broken code (`controls/`).""" + by_name = {t["name"]: t for t in traces if t.get("schema") == "disposal-v1"} + + def wake_before_expire(events): + expire = next(i for i, e in enumerate(events) if e["a"] == "Expire" and e["x"] == "wake") + run = next(i for i, e in enumerate(events) + if e["a"] == "Run" and e["id"] == events[expire]["id"]) + events.insert(expire, events.pop(run)) + require_rejected(by_name["disposal_expired_callers"], wake_before_expire, + "a wake-up run before its slot's transition", tlc) + + def run_before_submit(events): + run = next(i for i, e in enumerate(events) if e["a"] == "Run") + submit = next(i for i, e in enumerate(events) if e["a"] == "Submit") + events.insert(submit, events.pop(run)) + require_rejected(by_name["disposal_released_payloads"], run_before_submit, + "a payload run by a worker before the hand-off", tlc) + + def drop_while_held(events): + next(e for e in events if e["a"] == "Settle" and e["x"] == "held")["x"] = "dropped" + require_rejected(by_name["disposal_held_until_death"], drop_while_held, + "a handed-off payload dropped while the owner's hold is on", tlc) + + def missing_start(events): + next(e for e in events if e["a"] in {"Submit", "Take"} and e["x"] == "spawn")["x"] = "" + spawning = next(t for t in by_name.values() + if any(e["a"] in {"Submit", "Take"} and e["x"] == "spawn" for e in t["events"])) + require_rejected(spawning, missing_start, + "a worker start the start rule requires is not decided", tlc) + + def serial_in_one_entry(events): + # Two items of one visit run by one worker after one Take. + takes = [i for i, e in enumerate(events) if e["a"] == "Take"] + events.pop(takes[1]) + require_rejected(by_name["disposal_expired_callers"], serial_in_one_entry, + "a worker runs an item it never took", tlc) + + # Recorded from code with `disposal.batch_serial` (the shared-batch bug). + for path in sorted((directory / "controls").glob("disposal_*.json")): + control = json.loads(path.read_text()) + require_rejected(control, lambda events: None, + f"recorded from broken code: {control['name']}", tlc) + + def reply_trace_controls(traces, tlc): """Edits of recorded reply-slot traces that DeferredReplies.tla must reject.""" @@ -799,6 +851,7 @@ def decide_at_once(events): job_trace_controls(traces, args.tlc) coordinator_trace_controls(traces, args.tlc) reply_trace_controls(traces, args.tlc) + disposal_trace_controls(traces, args.tlc, args.traces) lifecycle_trace_controls(traces, args.tlc) for module, fault, invariant in LIFECYCLE_CONTROLS: named_fault(module, fault, invariant, args.tlc) diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index 998de32a2..3d14e4f42 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Check recorded lifecycle traces against Supervision.tla (and, by schema, Cohorts.tla, NestedSupervision.tla, DeferredJobs.tla, DeferredReplies.tla, -Lifecycle.tla and Coordinator.tla). +Disposal.tla, Lifecycle.tla and Coordinator.tla). Each JSON trace (written by concurrency/tests/supervision_trace.rs with the `lifecycle-trace` feature) looks like: @@ -50,7 +50,7 @@ def tla_event(event, cohort=False): def generate(trace, workdir): schema = trace.get("schema", "one-for-one-v1") if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1", "jobs-v1", "replies-v1", - "lifecycle-v1", "coordinator-v1"}: + "lifecycle-v1", "coordinator-v1", "disposal-v1"}: raise ValueError(f"unsupported trace schema: {schema}") if schema == "tree-v1": return generate_tree(trace, workdir) @@ -62,6 +62,8 @@ def generate(trace, workdir): return generate_lifecycle(trace, workdir) if schema == "coordinator-v1": return generate_coordinator(trace, workdir) + if schema == "disposal-v1": + return generate_disposal(trace, workdir) cohort = schema == "cohort-v1" modules = ("Cohorts.tla", "TraceCohorts.tla") if cohort else MODELS for model in modules: @@ -149,6 +151,83 @@ def generate_replies(trace, workdir): "INVARIANT Safety\nINVARIANT TraceIncomplete\n") +def generate_disposal(trace, workdir): + """The deadline service's hand-off and the disposal pool against + Disposal.tla. The leading `Pool` event gives the idle workers when + recording started (InitIdle = 1..K; a worker that appears without + `Begin` is one of them). Requests (Expire), payloads (Defer drop), other + items (Defer other) and started workers (Begin) are numbered in order of + appearance, over the whole log, so an event out of order keeps its item + and the model rejects it; an item never handed off gets kind "none".""" + for model in ("Disposal.tla", "TraceDisposal.tla"): + shutil.copy(os.path.join(HERE, model), workdir) + events = list(trace["events"]) + if not events or events[0]["a"] != "Pool": + raise ValueError("a disposal trace starts with the pool's state (Pool)") + pool = events.pop(0) + idle, started = int(pool["w"]), int(pool["r"]) + if idle != started or pool["x"]: + raise ValueError(f"recording must start with the pool quiescent: {pool}") + reqs, items, counts = {}, {}, {"drop": 0, "other": 0} + for e in events: + if e["a"] == "Expire": + r = int(e["r"]) + if r in reqs: + raise ValueError(f"a slot expired twice: {e}") + reqs[r] = len(reqs) + 1 + if int(e["id"]): + items[int(e["id"])] = ("wake", reqs[r]) + elif e["a"] == "Defer": + kind = e["x"] + counts[kind] += 1 + items[int(e["id"])] = (kind, counts[kind]) + begun = {int(e["w"]) for e in events if e["a"] == "Begin"} + workers, preexisting = {}, 0 + for e in events: + w = int(e.get("w", 0)) + if not w or w in workers: + continue + if w in begun: + workers[w] = None # numbered after the pre-existing ones + else: + preexisting += 1 + if preexisting > idle: + raise ValueError(f"more pre-existing workers than were idle: {e}") + workers[w] = preexisting + following = idle + for w in workers: + if workers[w] is None: + following += 1 + workers[w] = following + records = [] + for e in events: + a = e["a"] + if a == "Expire": + k, n = ("wake" if e["x"] == "wake" else "none"), reqs[int(e["r"])] + elif a in {"Defer", "Take", "Run", "Settle", "Done"}: + k, n = items.get(int(e["id"]), ("none", 0)) + else: + k, n = "none", 0 + records.append("[a |-> {}, w |-> {}, k |-> {}, n |-> {}, x |-> {}]".format( + tla_string(a), workers.get(int(e.get("w", 0)), 0), tla_string(k), n, + tla_string(e["x"]))) + if not records: + raise ValueError("a disposal trace needs at least one event") + with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: + out.write("---- MODULE TraceRun ----\nEXTENDS TraceDisposal\n" + f"TraceBlocking == {{}}\nTraceInitIdle == 1..{idle}\nTraceLog == <<\n " + + ",\n ".join(records) + "\n>>\n====\n") + with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: + # Deadlines are chosen by TLC; one tick makes every slot due. The + # worker cap is the code's (MAX_WORKERS). + out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" + f" N = {len(reqs)}\n P = {counts['drop']}\n O = {counts['other']}\n" + " MaxTime = 1\n Workers = 64\n" + " Blocking <- TraceBlocking\n InitIdle <- TraceInitIdle\n" + " Fault = \"None\"\n Trace <- TraceLog\n" + "INVARIANT Safety\nINVARIANT TraceIncomplete\n") + + OWN_ACTIONS = {"Trap", "Stop", "Link", "Unlink", "Monitor", "Demonitor"} diff --git a/verification/tla/trace_coverage.py b/verification/tla/trace_coverage.py index 8422ed079..bec611705 100644 --- a/verification/tla/trace_coverage.py +++ b/verification/tla/trace_coverage.py @@ -99,6 +99,7 @@ "DeferredJobs": ("TraceJobs.tla", {"jobs-v1"}), "DeferredReplies": ("TraceReplies.tla", {"replies-v1"}), "Coordinator": ("TraceCoordinator.tla", {"coordinator-v1"}), + "Disposal": ("TraceDisposal.tla", {"disposal-v1"}), } ALL = list(MODELS) + list(STATIC)