From 4f355069f7a56302ca6d27f6d2f02b23aef8b41b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 11:56:36 -0300 Subject: [PATCH 01/13] fix: hand deadline-expiry disposal off the shared deadline thread The process-wide deadline thread now only decides expiry (terminal CAS, credit return, job notification enqueue). Wakers, payload destructors and last references it releases go to an elastic disposal pool, so a blocking user destructor or waker in one actor no longer delays other actors' deadlines. --- concurrency/src/mailbox.rs | 10 +- concurrency/src/reply/deadline.rs | 24 ++- concurrency/src/reply/disposal.rs | 243 +++++++++++++++++++++ concurrency/src/reply/jobs.rs | 19 +- concurrency/src/reply/mod.rs | 1 + concurrency/src/reply/registry.rs | 23 +- concurrency/src/reply/slot.rs | 26 ++- concurrency/src/tasks/jobs.rs | 5 + concurrency/src/threads/jobs.rs | 5 + concurrency/tests/deadline_service.rs | 296 ++++++++++++++++++++++++++ 10 files changed, 630 insertions(+), 22 deletions(-) create mode 100644 concurrency/src/reply/disposal.rs diff --git a/concurrency/src/mailbox.rs b/concurrency/src/mailbox.rs index 8c153872f..af311bf5a 100644 --- a/concurrency/src/mailbox.rs +++ b/concurrency/src/mailbox.rs @@ -44,7 +44,10 @@ impl DropFailures { self.drop_contained(value); } } - pub(crate) fn discard(&self, value: T) { + /// On the deadline service thread the drop goes to a disposal worker, + /// which discards it again there: the hold is re-checked, so a drop + /// deferred before the owner's exit still waits for its death point. + pub(crate) fn discard(self: &Arc, value: T) { let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); if let Some(held) = held.as_mut() { // Moved, not dropped: no user code under the lock. @@ -52,6 +55,11 @@ impl DropFailures { return; } drop(held); + if crate::reply::disposal::collecting() { + let failures = self.clone(); + crate::reply::disposal::defer(move || failures.discard(value)); + return; + } self.drop_contained(value); } /// Drop at once (contained): the mailbox's own items, destroyed after diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 7fd354d88..4253c565f 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -12,7 +12,9 @@ //! due entries even when no caller polls, so credits return on time: //! //! - real time: one process-wide service thread with its own timer, so -//! per-test Tokio runtimes cannot break it; +//! per-test Tokio runtimes cannot break it. It runs no user code: what an +//! expiry releases (wakers, payload destructors) goes to a disposal worker +//! (`super::disposal`); //! - under `spawned_sim`: a task of the simulated runtime per actor heap, so //! clock domains never mix and virtual time drives it. @@ -304,15 +306,21 @@ mod thread_service { state.actors.remove(&key); drop(state); // Entries are contained one by one; this also covers the - // visit itself and dropping the upgraded registry, so the - // one process-wide service never stops. - super::super::registry::contain_entry(|| { - if let Some(registry) = weak.upgrade() { - if let Some(next) = registry.service_deadlines(now) { - schedule(®istry, next); + // visit itself, so the one process-wide service never + // stops. Whatever the visit releases that can run user + // code (wakers, payloads, the upgraded registry) goes to + // a disposal worker: this thread runs none of it. + let released = super::super::disposal::collect(|| { + super::super::registry::contain_entry(|| { + if let Some(registry) = weak.upgrade() { + if let Some(next) = registry.service_deadlines(now) { + schedule(®istry, next); + } + super::super::disposal::defer_drop(registry); } - } + }); }); + super::super::disposal::submit(released); state = lock(service); } Some(Reverse((at, _))) => { diff --git a/concurrency/src/reply/disposal.rs b/concurrency/src/reply/disposal.rs new file mode 100644 index 000000000..d52133330 --- /dev/null +++ b/concurrency/src/reply/disposal.rs @@ -0,0 +1,243 @@ +//! Disposal off the deadline service thread +//! (docs/design/deferred-replies.md §2, "Callback progress"). +//! +//! The process-wide deadline thread decides expiry: it pops due entries, +//! makes their terminal transitions and returns their credits (atomics and +//! framework locks only). What an expiry releases that can run user code, +//! a caller's waker, a payload destructor routed to the owner's failure +//! sink, the last reference to an entry, is collected while the thread +//! services a visit ([`collect`]) and handed to a disposal worker +//! ([`submit`]). A worker that blocks in user code holds only itself: a +//! batch that finds no idle worker starts another one, so one actor's +//! blocking destructor or waker never delays another actor's expiry or its +//! wake-ups. +//! +//! Outside a [`collect`] scope ([`defer`], [`defer_drop`]) run inline, so +//! every other thread (owner turns, jobs, the simulated service task) keeps +//! its behaviour. The check is one thread-local read, made only on paths +//! that already have something to wake or drop. + +use std::cell::RefCell; +use std::collections::VecDeque; +use std::sync::{Arc, Condvar, Mutex, OnceLock}; +use std::time::Duration; + +type Work = Box; + +thread_local! { + /// `Some` while the deadline thread services a visit. + static BATCH: RefCell>> = const { RefCell::new(None) }; +} + +/// Run `work` now, or, on the deadline thread inside [`collect`], defer it +/// to a disposal worker. +#[inline] +pub(crate) fn defer(work: impl FnOnce() + Send + 'static) { + let work = BATCH.with(|batch| match batch.try_borrow_mut() { + Ok(mut batch) => match batch.as_mut() { + Some(batch) => { + batch.push(Box::new(work)); + None + } + None => Some(work), + }, + // Re-entered while the batch is borrowed: never on the service + // path, which only pushes. Run it. + Err(_) => Some(work), + }); + if let Some(work) = work { + work(); + } +} + +/// Drop `value` now, or defer its drop (see [`defer`]). +#[inline] +pub(crate) fn defer_drop(value: T) { + defer(move || drop(value)); +} + +/// Whether this thread is collecting (the deadline thread in a visit). +#[inline] +pub(crate) fn collecting() -> bool { + BATCH.with(|batch| batch.try_borrow().is_ok_and(|batch| batch.is_some())) +} + +/// Run `step` with deferral on, and return what it deferred. The scope +/// ends even if `step` unwinds (the deferred work is then dropped here, +/// which is why the service contains each step inside it). +pub(crate) fn collect(step: impl FnOnce()) -> Vec { + struct Scope; + impl Drop for Scope { + fn drop(&mut self) { + let _ = BATCH.with(|batch| batch.borrow_mut().take()); + } + } + BATCH.with(|batch| *batch.borrow_mut() = Some(Vec::new())); + let scope = Scope; + step(); + let work = BATCH.with(|batch| batch.borrow_mut().take().unwrap_or_default()); + drop(scope); + work +} + +// --------------------------------------------------------------------------- +// Workers +// --------------------------------------------------------------------------- + +/// Workers beyond this many wait for a busy one: a bound on threads, reached +/// only when that many callbacks are blocked at once. +const MAX_WORKERS: usize = 64; +/// An idle worker exits after this long without work. +const KEEP_ALIVE: Duration = Duration::from_secs(10); + +#[derive(Default)] +struct State { + queue: VecDeque>, + /// Workers waiting for work. + idle: usize, + workers: usize, +} + +#[derive(Default)] +struct Pool { + state: Mutex, + ready: Condvar, +} + +fn pool() -> &'static Arc { + static POOL: OnceLock> = OnceLock::new(); + POOL.get_or_init(Default::default) +} + +fn lock(pool: &Pool) -> std::sync::MutexGuard<'_, State> { + pool.state.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// Hand a visit's deferred work to a disposal worker. A batch never waits +/// behind a busy worker: when queued batches outnumber idle workers, a new +/// worker starts (up to [`MAX_WORKERS`]). +pub(crate) fn submit(work: Vec) { + if work.is_empty() { + return; + } + let pool = pool(); + let mut state = lock(pool); + state.queue.push_back(work); + let start = state.queue.len() > state.idle && state.workers < MAX_WORKERS; + if start { + state.workers += 1; + } + drop(state); + if start { + let worker = pool.clone(); + let spawned = std::thread::Builder::new() + .name("spawned-disposal".into()) + .spawn(move || run(&worker)); + if spawned.is_err() { + // No new thread: an existing worker takes the batch. With none + // at all, run it here rather than lose a wake-up. + let orphaned = { + let mut state = lock(pool); + state.workers -= 1; + if state.workers == 0 { + std::mem::take(&mut state.queue) + } else { + VecDeque::new() + } + }; + orphaned.into_iter().for_each(run_batch); + } + } + pool.ready.notify_one(); +} + +fn run(pool: &Pool) { + let mut state = lock(pool); + loop { + if let Some(work) = state.queue.pop_front() { + drop(state); + run_batch(work); + state = lock(pool); + continue; + } + state.idle += 1; + let (next, timeout) = pool + .ready + .wait_timeout(state, KEEP_ALIVE) + .unwrap_or_else(|p| p.into_inner()); + state = next; + state.idle -= 1; + if timeout.timed_out() && state.queue.is_empty() { + state.workers -= 1; + return; + } + } +} + +/// Each item contained, so one failing callback skips none of the others. +fn run_batch(work: Vec) { + for item in work { + super::registry::contain_entry(item); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::mpsc; + + #[test] + fn defer_runs_inline_outside_a_scope() { + let (tx, rx) = mpsc::channel(); + defer(move || tx.send(()).unwrap()); + rx.try_recv().expect("ran inline"); + assert!(!collecting()); + } + + #[test] + fn collect_defers_and_ends_its_scope() { + let (tx, rx) = mpsc::channel(); + let work = collect(|| { + assert!(collecting()); + let sender = tx.clone(); + defer(move || sender.send(1).unwrap()); + defer_drop(tx); + }); + assert!(!collecting()); + assert!(rx.try_recv().is_err(), "deferred, not run"); + assert_eq!(work.len(), 2); + submit(work); + assert_eq!(rx.recv_timeout(Duration::from_secs(5)), Ok(1)); + } + + #[test] + fn a_blocked_batch_does_not_hold_back_the_next() { + let (release_tx, release_rx) = mpsc::channel::<()>(); + let (entered_tx, entered_rx) = mpsc::channel(); + let blocked: Vec = vec![Box::new(move || { + entered_tx.send(()).unwrap(); + let _ = release_rx.recv_timeout(Duration::from_secs(10)); + })]; + submit(blocked); + entered_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the first batch started"); + let (tx, rx) = mpsc::channel(); + submit(vec![Box::new(move || tx.send(()).unwrap())]); + assert!( + rx.recv_timeout(Duration::from_secs(5)).is_ok(), + "a blocked worker held back another batch" + ); + release_tx.send(()).unwrap(); + } + + #[test] + fn a_panicking_item_is_contained() { + let (tx, rx) = mpsc::channel(); + submit(vec![ + Box::new(|| panic!("injected disposal panic")), + Box::new(move || tx.send(()).unwrap()), + ]); + assert!(rx.recv_timeout(Duration::from_secs(5)).is_ok()); + } +} diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 79ee101f8..ad58791e7 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -415,9 +415,15 @@ impl JobState { #[cfg(feature = "async")] fn wake_emitters(&self) { let wakers = std::mem::take(&mut *lock(&self.emit_wakers)); - for waker in wakers { - waker.wake(); + if wakers.is_empty() { + return; } + // Off the deadline service thread when a deadline or emit fires. + super::disposal::defer(move || { + for waker in wakers { + waker.wake(); + } + }); } /// Take the terminal notification and run `enqueue` under the job's @@ -456,8 +462,11 @@ impl JobState { (taken, unsent, held) }); // A held result the deadline displaced (user payload): dropped - // outside the trace section, after JobDeadline is recorded. - drop(held); + // outside the trace section, after JobDeadline is recorded, and off + // the deadline service thread. + if let Some(held) = held { + super::disposal::defer_drop(held); + } #[cfg(feature = "async")] if taken { self.wake_emitters(); @@ -1107,6 +1116,8 @@ impl Expirable for ScheduledEmit { fail_point!("job.emit_due"); if let Some(job) = self.job.upgrade() { self.settle(&job, true); + // Possibly the last reference (its other scheduled emits). + super::disposal::defer_drop(job); } } } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index db81bbb0f..31b263140 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -7,6 +7,7 @@ //! gets exactly one terminal discharge. pub(crate) mod deadline; +pub(crate) mod disposal; pub(crate) mod job_core; pub mod jobs; mod pending; diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index cf4503b1f..f6def99fa 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -59,8 +59,14 @@ impl Expirable for Slot { let deadline = self.deadline(); (!self.is_terminal() && deadline != NEVER).then_some(deadline) } + /// The transition and credit return happen here; a parked blocking + /// receiver is unparked here too (no user code); the wakers are handed + /// to a disposal worker when this runs on the deadline thread. fn expire(&self, now: u64) { - self.check_deadline(&|| now); + if let Some(mut wake) = self.expire_collect(now) { + wake.notify_parked(); + super::disposal::defer(move || wake.run()); + } } } @@ -292,10 +298,13 @@ impl Registry { /// The service visits: expire every due entry outside the lock, and /// return the next earliest deadline. /// - /// Entries run framework code that can reach user code (a waker, a - /// payload destructor the entry did not route to its failure sink, the - /// last reference to a job's `done`). Each runs contained, so one - /// failing entry neither skips the others nor ends the shared service. + /// Entries make their terminal transitions and return credits here. + /// What they release that can run user code (a waker, a payload + /// destructor, the last reference to an entry or a job's `done`) is + /// deferred: on the deadline thread it goes to a disposal worker + /// ([`disposal`](super::disposal)); elsewhere it runs inline. Each entry + /// runs contained, so one failing entry neither skips the others nor + /// ends the shared service. pub(crate) fn service_deadlines(&self, now: u64) -> Option { let due = lock(&self.lists).deadlines.due(now); for item in due { @@ -303,8 +312,8 @@ impl Registry { if item.pending_deadline().is_some_and(|d| d <= now) { item.expire(now); } - // Possibly the last reference: dropped inside the guard. - drop(item); + // Possibly the last reference. + super::disposal::defer_drop(item); }); } lock(&self.lists).deadlines.earliest() diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 6c5b63132..87ab6f20f 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -336,7 +336,7 @@ impl Wake { }; /// Wake a parked blocking receiver (no user code). Done first, so a /// parked `recv()` never waits behind an arbitrary waker. - fn notify_parked(&mut self) { + pub(crate) fn notify_parked(&mut self) { if std::mem::take(&mut self.parked) { if let Some(waker) = self.waker.take() { waker.wake(); @@ -344,7 +344,7 @@ impl Wake { } } #[inline] - fn run(self) { + pub(crate) fn run(self) { if let Some(waker) = self.waker { waker.wake(); } @@ -613,6 +613,28 @@ impl Slot { } } + /// The deadline service's `[deadline]` check: the same `OPEN -> TIMED_OUT` + /// transition and credit return as [`check_deadline`](Self::check_deadline), + /// but the wake-up (a caller's waker, the token's watchers: executor or + /// user code) is returned for the caller to run elsewhere. `None` when + /// the deadline has not passed or another transition won. + pub(crate) fn expire_collect(&self, now: u64) -> Option { + let deadline = self.deadline(); + if deadline == NEVER || now < deadline { + return None; + } + let (old, new) = self + .cas::<()>(None, |w| { + if state(w) == OPEN { + Ok(with_state(w, TIMED_OUT)) + } else { + Err(()) + } + }) + .ok()?; + Some(self.after(old, new)) + } + /// `is_closed()`: not `OPEN`, or past its deadline (then expired). pub(crate) fn is_closed(&self, now: &impl Fn() -> u64) -> bool { state(self.word()) != OPEN || self.check_deadline(now) diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index ef691fd7e..928478f1c 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -196,6 +196,9 @@ where if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { run.notify(outcome); } + // Possibly the last reference (`done` and its captures): + // off the deadline service thread. + crate::reply::disposal::defer_drop(run); }) }, ); @@ -717,6 +720,8 @@ impl Context { notify: Mutex::new(Some(Box::new(move || { if let Some(run) = weak.upgrade() { run.notify(Err(JobError::DeadlineExceeded)); + // Possibly the last reference: off the deadline thread. + crate::reply::disposal::defer_drop(run); } }))), }); diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 45cc2972f..2e73ef280 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -178,6 +178,9 @@ where if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { run.notify(outcome); } + // Possibly the last reference (`done` and its captures): + // off the deadline service thread. + crate::reply::disposal::defer_drop(run); }) }, ); @@ -488,6 +491,8 @@ impl Context { notify: Mutex::new(Some(Box::new(move || { if let Some(run) = weak.upgrade() { run.notify(Err(JobError::DeadlineExceeded)); + // Possibly the last reference: off the deadline thread. + crate::reply::disposal::defer_drop(run); } }))), }); diff --git a/concurrency/tests/deadline_service.rs b/concurrency/tests/deadline_service.rs index 4e31c9908..628ae302f 100644 --- a/concurrency/tests/deadline_service.rs +++ b/concurrency/tests/deadline_service.rs @@ -133,3 +133,299 @@ fn deadline_service_child() { }); println!("deadline service survived"); } + +// --------------------------------------------------------------------------- +// Disposal isolation: user code that expiry releases (a payload destructor, a +// caller's waker) never runs on the shared deadline thread, so a blocking one +// in actor A does not delay actor B's deadlines. +// --------------------------------------------------------------------------- + +/// A one-shot latch with bounded waits. +#[derive(Default)] +struct Latch(std::sync::Mutex, std::sync::Condvar); +impl Latch { + fn set(&self) { + *self.0.lock().unwrap() = true; + self.1.notify_all(); + } + fn wait(&self, limit: Duration) -> bool { + let guard = self.0.lock().unwrap(); + let (guard, _) = self + .1 + .wait_timeout_while(guard, limit, |set| !*set) + .unwrap(); + *guard + } +} + +/// Blocking user code: signals `entered`, then waits (bounded) for `release`. +#[derive(Default)] +struct Blocker { + entered: Latch, + release: Latch, +} +impl Blocker { + fn block(&self) { + self.entered.set(); + self.release.wait(Duration::from_secs(10)); + } +} + +/// A message whose destructor blocks while armed. +struct Stall(Option>); +impl Message for Stall { + type Result = (); +} +impl Drop for Stall { + fn drop(&mut self) { + if let Some(blocker) = self.0.take() { + blocker.block(); + } + } +} +impl Handler for Owner { + async fn handle(&mut self, mut msg: Stall, _: PendingReply<'_, ()>, _: &Context) { + msg.0 = None; // delivered: not the scenario under test + } +} + +/// A waker whose `wake` blocks. +struct BlockingWake(std::sync::Arc); +impl std::task::Wake for BlockingWake { + fn wake(self: std::sync::Arc) { + self.0.block(); + } +} + +#[test] +fn deadline_service_isolates_a_blocking_payload_destructor() { + subprocess::run( + "deadline_service_blocking_destructor_child", + "blocking destructor isolated", + ); +} + +#[test] +fn deadline_service_isolates_a_blocking_waker() { + subprocess::run( + "deadline_service_blocking_waker_child", + "blocking waker isolated", + ); +} + +#[test] +fn deadline_service_isolates_a_blocking_destructor_threads() { + subprocess::run( + "deadline_service_blocking_destructor_threads_child", + "threads blocking destructor isolated", + ); +} + +/// Another actor's raw request and job deadlines, serviced by the heap. +async fn other_actor_deadlines_fire(blocker: &Blocker) -> Result<(), String> { + let other = Owner::default().start(); + let parked = other + .send_request(Park, Deadline::after(Duration::from_millis(50))) + .unwrap(); + let result = match tokio::time::timeout(Duration::from_secs(3), parked).await { + Ok(Err(ActorError::RequestTimeout)) => Ok(()), + Ok(other) => Err(format!("unexpected outcome {other:?}")), + Err(_) => Err("a blocked disposal delayed another actor's request deadline".into()), + }; + let fired = spawned_concurrency::__private::in_turn(&other, |_, ctx| { + let (tx, rx) = std::sync::mpsc::channel(); + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + |_| std::thread::sleep(Duration::from_millis(300)), + move |outcome: JobOutcome<()>| { + let _ = tx.send(outcome.is_err()); + Done + }, + ) + .unwrap(); + rx + }) + .unwrap() + .await + .unwrap(); + let job = tokio::task::spawn_blocking(move || fired.recv_timeout(Duration::from_secs(3))) + .await + .unwrap(); + // Release the blocked disposal before reporting, so the child exits. + blocker.release.set(); + let result = result.and(match job { + Ok(true) => Ok(()), + other => Err(format!( + "a blocked disposal delayed another actor's job deadline: {other:?}" + )), + }); + other.context().stop(); + other.join().await; + result +} + +fn tokio_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap() +} + +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_blocking_destructor_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let blocker = std::sync::Arc::new(Blocker::default()); + tokio_runtime().block_on(async { + let owner = Owner::default().start(); + let armed = blocker.clone(); + spawned_concurrency::__private::in_turn(&owner, move |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + move |job| { + // Discarded at 40 ms (the deadline took the notification + // at 20 ms): its destructor blocks. + let at = Instant::now() + Duration::from_millis(40); + job.emit_at_blocking(at, Stall(Some(armed))).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + }) + .unwrap() + .await + .unwrap(); + assert!( + blocker.entered.wait(Duration::from_secs(5)), + "the scheduled emit was never discarded" + ); + let result = other_actor_deadlines_fire(&blocker).await; + owner.context().stop(); + owner.join().await; + result.unwrap(); + }); + println!("blocking destructor isolated"); +} + +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_blocking_waker_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let blocker = std::sync::Arc::new(Blocker::default()); + tokio_runtime().block_on(async { + let owner = Owner::default().start(); + // A raw receiver polled once with a waker that blocks: its deadline + // goes to the owner's heap, whose expiry wakes it. + let receiver = owner + .send_request(Park, Deadline::after(Duration::from_millis(30))) + .unwrap(); + let mut receiver = Box::pin(std::future::IntoFuture::into_future(receiver)); + let waker = std::task::Waker::from(std::sync::Arc::new(BlockingWake(blocker.clone()))); + let mut cx = std::task::Context::from_waker(&waker); + assert!(std::future::Future::poll(receiver.as_mut(), &mut cx).is_pending()); + assert!( + blocker.entered.wait(Duration::from_secs(5)), + "the receiver's waker never ran" + ); + let result = other_actor_deadlines_fire(&blocker).await; + drop(receiver); + owner.context().stop(); + owner.join().await; + result.unwrap(); + }); + println!("blocking waker isolated"); +} + +mod threads_backend { + use super::*; + use spawned_concurrency::threads::{ + Actor, ActorStart, Context, Handler, JobOptions, JobOutcome, + }; + + pub struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + fn handle(&mut self, mut msg: Stall, _: PendingReply<'_, ()>, _: &Context) { + msg.0 = None; + } + } + impl Handler for Owner { + fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + + pub fn run(blocker: &std::sync::Arc) -> Result<(), String> { + let owner = Owner.start(); + let armed = blocker.clone(); + spawned_concurrency::__private::in_turn_threads(&owner, move |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + move |job| { + let at = Instant::now() + Duration::from_millis(40); + job.emit_at_blocking(at, Stall(Some(armed))).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + }) + .unwrap() + .recv() + .unwrap(); + assert!( + blocker.entered.wait(Duration::from_secs(5)), + "the scheduled emit was never discarded" + ); + // Another actor's job deadline, serviced by the same deadline service. + let other = Owner.start(); + let fired = spawned_concurrency::__private::in_turn_threads(&other, |_, ctx| { + let (tx, rx) = std::sync::mpsc::channel(); + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + |_| std::thread::sleep(Duration::from_millis(300)), + move |outcome: JobOutcome<()>| { + let _ = tx.send(outcome.is_err()); + Done + }, + ) + .unwrap(); + rx + }) + .unwrap() + .recv() + .unwrap(); + let job = fired.recv_timeout(Duration::from_secs(3)); + blocker.release.set(); + let result = match job { + Ok(true) => Ok(()), + other => Err(format!( + "a blocked disposal delayed another actor's job deadline: {other:?}" + )), + }; + other.context().stop(); + other.join(); + owner.context().stop(); + owner.join(); + result + } +} + +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_blocking_destructor_threads_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let blocker = std::sync::Arc::new(Blocker::default()); + threads_backend::run(&blocker).unwrap(); + println!("threads blocking destructor isolated"); +} From e93cf699c09419d82d0a41f1103e004678ffb21f Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 12:05:57 -0300 Subject: [PATCH 02/13] docs, models: document disposal off the deadline thread; InlineDisposal control and Loom case --- CHANGELOG.md | 10 ++++++ ROADMAP.md | 14 ++++---- concurrency/tests/reply_slot_loom.rs | 48 ++++++++++++++++++++++++++++ docs/GUIDE.md | 15 +++++---- docs/design/deferred-replies.md | 40 ++++++++++++++++++++--- docs/design/semantics.md | 23 ++++++------- verification/tla/DeferredReplies.tla | 23 +++++++++++-- verification/tla/README.md | 1 + verification/tla/negative_checks.py | 1 + 9 files changed, 143 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb8fa7fe4..9faa0722f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,16 @@ (the selection trace missed it in 24 of 30 runs). New `lifecycle_trace::recorded_with` test helper. +- Keep user code off the shared deadline-service thread (the known 0.8 + limitation). The thread still decides expiry and returns credits, but the + wakers and payload destructors an expiry releases (a caller's waker, a + discarded scheduled emit, a displaced job result, a `done` closure and its + captures) now run on an elastic pool of disposal workers, so a blocking + destructor or waker in one actor no longer delays other actors' deadlines. + A payload handed off still waits for its owner's death point (R6). No cost + on requests that do not expire. New regression tests (tasks and threads), + a Loom case for the handed-off wake-up, and an `InlineDisposal` negative + control in `DeferredReplies.tla`. - Regenerate the 11 planted-bug patches (04, 10-17, 19, 20) that no longer applied, each planting the same fault where the code now lives, and check in the required `model-sync` job that every planted patch still applies. diff --git a/ROADMAP.md b/ROADMAP.md index a07acdace..054bb0760 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -39,13 +39,13 @@ handlers and the existing default behavior. New policies are explicit and opt-in performance campaign against 0.7.3, paired four-hour soaks and the full Lifecycle models (evidence attached to the GitHub release). Ethrex and Ethlambda pin 0.8.0 in their draft PRs and run it on 24-hour canaries. -- **Deadline-service follow-up (known 0.8 limit):** separate user payload - disposal from shared deadline bookkeeping, with no default-path cost. - Acceptance: block a job result destructor in actor A while actor B's - unpolled token/raw deadline expires and returns its credit. Keep - foreign-waker progress assumptions explicit; never make exit wait for a - pre-exit destructor that can wait for the owner's callers. The current - scheduling limit is documented in the guide. +- **Deadline-service follow-up (done, unreleased):** the shared deadline + thread only decides expiry; wakers and payload destructors go to an elastic + disposal pool, with no default-path cost. A blocking destructor or waker in + actor A no longer delays actor B's token, raw or job deadlines + (`concurrency/tests/deadline_service.rs`, tasks and threads). The remaining + assumption (the owner's own executor wake runs on the service thread) is in + `docs/design/deferred-replies.md` §2. - **Post-0.8 cleanups (in progress):** experiment features moved to `benchmarks/`; the large actor and supervision files split; a lint and a `model-sync` mapping that keep the Loom-checked cores tied to their Loom tests. diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 2681172de..00226a2c2 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -352,6 +352,54 @@ fn drop_vs_close_vs_expiry() { }); } +/// The deadline service's expiry (`expire_collect`) versus a reply, with the +/// expiry's wake-up handed off to another thread (a disposal worker) and the +/// caller polling throughout: exactly one wins, the credit returns with the +/// transition (before the wake-up runs), and the polling caller always gets +/// the outcome (the handed-off wake-up is never lost). +#[test] +fn service_expiry_with_handed_off_wake_vs_reply() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 10); + let service = { + let (slot, pool) = (slot.clone(), pool.clone()); + loom::thread::spawn(move || { + let wake = slot.expire_collect(10); + if wake.is_some() { + assert_eq!(pool.used(), 0, "credit returned before the wake-up"); + } + wake + }) + }; + let disposal = loom::thread::spawn(move || { + if let Some(mut wake) = service.join().unwrap() { + wake.notify_parked(); + wake.run(); + true + } else { + false + } + }); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(1).0) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + let expired = disposal.join().unwrap(); + let delivery = replier.join().unwrap(); + if expired { + assert_eq!(got, Err(Terminal::TimedOut)); + assert_eq!(delivery, Delivery::Expired); + } else { + assert_eq!(got, Ok(1)); + assert_eq!(delivery, Delivery::Delivered); + } + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + /// `defer()` versus the caller abandoning: a slot that went terminal never /// acquires a credit, and a deferred one releases it on abandonment. #[test] diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 6f53c07b2..63a1548ae 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -665,12 +665,15 @@ Rules (the [design](design/deferred-replies.md) has the full contract): added later can be answered). **Deadline-service progress.** Managed and unmanaged actors share a service -thread for token/job deadlines and scheduled job emits. Expiry can run user -wakers and payload destructors there; a blocking callback delays other -actors' heap-driven expiry too. Keep these callbacks short and nonblocking. -Ordinary `request()`/`request_with_timeout()`/`Response` timers and the -`send_after`/`send_interval` drivers are separate. This is a scheduling -limitation, not a hard real-time deadline guarantee. +thread for token/job deadlines and scheduled job emits. It only decides +expiry and returns credits; the wakers and payload destructors an expiry +releases (a caller's waker, a discarded emit, a displaced job result, a +`done` closure's captures) run on separate disposal workers, so a blocking +destructor or waker in one actor does not delay another actor's deadlines. +They still occupy a worker (at most 64), so keep them short. Ordinary +`request()`/`request_with_timeout()`/`Response` timers and the +`send_after`/`send_interval` drivers are separate. Deadlines are not a hard +real-time guarantee. **Answering with a reason.** The framework's errors (`ReplyDropped`, `ActorStopped`, `Overloaded`, `RequestTimeout`) say what happened to the diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 6f1e8bdeb..25c829085 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -288,10 +288,39 @@ handles a call even after the caller timed out, and the late reply is dropped. Shortening or a new earliest entry re-registers and wakes it. A dead owner is forgotten at registry close. It is independent of individual Tokio runtime lifetimes. -- **Callback progress.** Expiry may run user wakers and destroy job payloads - or completion captures on that thread. A blocking callback delays other - actors' heap-driven deadlines and credit return. Callbacks must return - promptly; the service provides no isolation from non-returning callbacks. +- **Callback progress (disposal off the service thread).** The service + thread only decides expiry: it pops due entries, makes the terminal + transition (`OPEN -> TIMED_OUT`, a job's `JobDeadline` notification or a + scheduled emit's fire) and returns the credit, all under framework locks. + It also unparks a parked blocking receiver and enqueues a job's message on + its owner's mailbox (the runtime wake of the owner's own task). Anything an + expiry releases that can run user code is collected during the visit and + handed to a disposal worker (`reply/disposal.rs`): the caller's waker and + the token's watchers, payloads routed to the owner's failure sink (a held + or refused job result, a discarded emit, `done` and its captures), and the + last references to entries, jobs and the registry. Disposal workers form an + elastic pool: a batch that finds no idle worker starts one (up to 64, + idle ones exit after 10 s), so a destructor or waker that blocks holds + only its own worker, and no other actor's expiry, credit return or + wake-up waits for it. + - *Why a pool and not the owner.* Deferring disposal to the owning actor + would tie a caller's wake-up to the owner's next turn (a wedged owner + would hang its callers past their deadline), and deferring it to the + caller has no thread to run on for an unpolled token. A single disposal + thread would only move the stall: a caller's waker queued behind + another actor's blocking destructor still waits. + - *Order kept.* The death point's rules hold: the transition precedes the + wake-up as before (a caller that polls in between sees the terminal + state); a payload handed off is discarded again on the worker, where the + owner's hold is re-checked, so a drop deferred before the owner's exit + still waits for its death point (R6, callers before payloads). + - *Cost.* None on a request or reply that does not expire: the hand-off is + in the service's own path, and the shared helpers test a thread-local + only when they already have a waker or a payload to dispose of. The + perf-gate driver measures within noise. + - *Remaining assumption.* The owner's own executor wake (a runtime waker or + condvar) still runs on the service thread; it is framework code. With + more than 64 callbacks blocked at once, later batches queue behind them. Ordinary request/Response runtime timers and send_after/send_interval drivers are separate (see semantics.md, section 10). - **Simulation.** Under `spawned_sim` the service is a task of the simulated @@ -1123,7 +1152,8 @@ semantics above. service thread (not a task on the `ActorSystem` control runtime); a dead owner is forgotten at its registry close. Under `spawned_sim` each owner's heap is served by a task of the simulated runtime, so clock domains never - mix. + mix. The thread runs no user code: wakers and payload destructors an expiry + releases go to an elastic disposal pool (§2, "Callback progress"). - **Cleanup deadline.** Armed when exit begins: the supervisor's `shutdown_timeout` for a supervised child, the remaining system grace during `ActorSystem::shutdown`, otherwise `DEFAULT_STOP_DEADLINE` (5 s). diff --git a/docs/design/semantics.md b/docs/design/semantics.md index da6925d0b..183d56c08 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -160,7 +160,11 @@ the expiry runs is delivered (`Delivery::Delivered`, and the caller gets the value), as an OTP reply already in the caller's mailbox when its `after` clause would fire is received. Timer resolution and when the runtime or deadline service can process expiry determine that window; this is not a -hard real-time bound. +hard real-time bound. The deadline service runs no user code: the wakers and +payload destructors an expiry releases run on disposal workers, so another +actor's blocking destructor or waker does not delay an expiry, its credit +return or its caller's wake-up +([`deadline_service`][deadline_service]). Which clock expires a waiting caller is part of the contract: `request`, `request_with_timeout` and `Response::from(rx)` arm the runtime's timer, so they follow paused Tokio time; a `ReplyReceiver` awaited directly @@ -763,16 +767,12 @@ stopped after `Shutdown`), - **Preemption.** A running handler, a blocking call or a non-returning hook is never interrupted; stop, shutdown deadlines and cleanup timeouts only signal and report. Rust cannot kill a thread. -- **Deadline-service callback isolation.** Outside simulation, managed and - unmanaged actors share one deadline-service thread. Expiry can invoke user - wakers and destroy job payloads or completion captures there. A callback - that blocks delays heap-driven expiry, credit return and scheduled job - emits for other actors too. These callbacks must return promptly. Ordinary - `request`/`request_with_timeout`/`Response` waits use their runtime timer; - `send_after` and `send_interval` use separate tasks or threads. No callback - is preempted. Separating disposal from deadline bookkeeping is follow-up - work; expiry authority, terminal outcomes and R6 caller-before-payload - ordering are unchanged. +- **Deadline-service isolation past 64 blocked callbacks.** The shared + deadline-service thread runs no user code (section 3): wakers and + payload destructors an expiry releases go to a pool of at most 64 disposal + workers. With that many callbacks blocked at once, later disposals (and so + their callers' wake-ups) queue behind them; expiry and credit return do + not. No callback is preempted. - **Bounded memory.** Mailboxes are unbounded (D5); there are no per-actor memory limits (rank 18a). - **Deadlock detection.** A request cycle, or a handler requesting its own @@ -855,6 +855,7 @@ remaining differences of the exit-signal design are listed in See the [ranked backlog](../../ROADMAP.md#ranked-backlog). [deferred_replies]: ../../concurrency/tests/deferred_replies.rs +[deadline_service]: ../../concurrency/tests/deadline_service.rs [deferred_jobs]: ../../concurrency/tests/deferred_jobs.rs [actor_scenarios]: ../../concurrency/tests/actor_scenarios.rs [cancellation_safety]: ../../concurrency/tests/cancellation_safety.rs diff --git a/verification/tla/DeferredReplies.tla b/verification/tla/DeferredReplies.tla index 9572989db..85ea83dd6 100644 --- a/verification/tla/DeferredReplies.tla +++ b/verification/tla/DeferredReplies.tla @@ -75,6 +75,9 @@ (* NoReplyFence (NoReplyAfterDeath) a reply is admitted after D *) (* CallbackBeforeDischarge (WokenAfterDischarge) an answer's wake-up *) (* runs before the last answer *) +(* InlineDisposal (TimelyExpiry) the shared deadline thread runs *) +(* an expiry's user callback inline *) +(* (0.8.0), which never returns *) (***************************************************************************) EXTENDS Naturals, FiniteSets CONSTANTS N, BoundA, BoundB, MaxTime, MaxWait, Fault @@ -82,7 +85,8 @@ ASSUME Fault \in {"None", "NoAutoReply", "NoAtomicExpiry", "LateSuccess", "LoopDeadline", "Lengthen", "Unbounded", "NoDestAdmission", "ExitStateOnly", "UnwindDropped", "ShareRunning", "AttachClosed", "LeakDownstream", "EarlyClose", - "EarlyStopped", "NoReplyFence", "CallbackBeforeDischarge"} + "EarlyStopped", "NoReplyFence", "CallbackBeforeDischarge", + "InlineDisposal"} Req == 1..N \* qA queued; turnA PendingReply in a turn; localA ReplyTo in a turn; @@ -394,8 +398,21 @@ BReply(r) == ------------------------------------------------------------------------------ \* Deadlines -Blocked(r) == Fault = "LoopDeadline" /\ turnA # 0 - /\ holder[r] \in {"qA", "turnA", "localA", "dropA", "stateA", "doneA"} +\* One process-wide thread services every owner's deadline heap (the +\* tokens A holds). It only decides expiry; what an expiry releases that can +\* run user code (the caller's waker, a payload or completion destructor) is +\* handed to a disposal worker, so Expire(r) never waits for another +\* request's callback. InlineDisposal is the 0.8.0 behaviour: the thread ran +\* those callbacks itself, and one that never returns (here: the one an +\* earlier expiry released) stops it from expiring any other token. +ServiceStuck(r) == + /\ Fault = "InlineDisposal" + /\ holder[r] \in {"localA", "dropA", "stateA", "doneA"} + /\ \E q \in Req \ {r} : slot[q] = "timeout" + +Blocked(r) == \/ /\ Fault = "LoopDeadline" /\ turnA # 0 + /\ holder[r] \in {"qA", "turnA", "localA", "dropA", "stateA", "doneA"} + \/ ServiceStuck(r) Shorten(r) == /\ Open(r) /\ holder[r] \in {"localA", "stateA", "doneA"} diff --git a/verification/tla/README.md b/verification/tla/README.md index 68df02249..1796f9fbd 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -709,6 +709,7 @@ exactly the property it targets: | `DeferredReplies` | `EarlyStopped` (rank 11 before 5e: a token dropped while exiting or unwound answers `Stopped` at once) | `StoppedAtDeath` | | `DeferredReplies` | `NoReplyFence` (a stored token's reply is admitted after the death point) | `NoReplyAfterDeath` | | `DeferredReplies` | `CallbackBeforeDischarge` (an answer's wake-up runs before the last answer of the death batch) | `WokenAfterDischarge` | +| `DeferredReplies` | `InlineDisposal` (0.8.0: the shared deadline thread runs an expiry's waker or destructor inline, and one never returns) | `TimelyExpiry` | | `DeferredJobs` | `NoInstanceCheck` | `NoLateResult` | | `DeferredJobs` | `NoCancelFence` | `NoStaleAfterCancel` | | `DeferredJobs` | `DoubleTerminal` | `AtMostOneTerminal` | diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 8004d64d2..6898292f0 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -282,6 +282,7 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDeferredReplies", "EarlyStopped", "StoppedAtDeath"), ("MCDeferredReplies", "NoReplyFence", "NoReplyAfterDeath"), ("MCDeferredReplies", "CallbackBeforeDischarge", "WokenAfterDischarge"), + ("MCDeferredReplies", "InlineDisposal", "TimelyExpiry"), ("MCDeferredJobs", "NoInstanceCheck", "NoLateResult"), ("MCDeferredJobs", "NoCancelFence", "NoStaleAfterCancel"), ("MCDeferredJobs", "DoubleTerminal", "AtMostOneTerminal"), From 4f23f7fb1b790ecf545ce271bf3ea5fb392bc2ab Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 12:09:43 -0300 Subject: [PATCH 03/13] refactor: share the collecting transition between close and service expiry --- concurrency/src/reply/slot.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 87ab6f20f..066a6b530 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -545,10 +545,15 @@ impl Slot { /// [`run_wake`](Self::run_wake) once a whole batch is closed, so no /// waker (executor or user code) runs between two closes. pub(crate) fn close_collect(&self) -> Option { + self.terminate_collect(STOPPED) + } + + /// `OPEN -> target` with its credit return; the wake-up is returned. + fn terminate_collect(&self, target: u64) -> Option { let (old, new) = self .cas::<()>(None, |w| { if state(w) == OPEN { - Ok(with_state(w, STOPPED)) + Ok(with_state(w, target)) } else { Err(()) } @@ -623,16 +628,7 @@ impl Slot { if deadline == NEVER || now < deadline { return None; } - let (old, new) = self - .cas::<()>(None, |w| { - if state(w) == OPEN { - Ok(with_state(w, TIMED_OUT)) - } else { - Err(()) - } - }) - .ok()?; - Some(self.after(old, new)) + self.terminate_collect(TIMED_OUT) } /// `is_closed()`: not `OPEN`, or past its deadline (then expired). From 4f81e3f2a3b3f147850d80d5e7be6f0c7d4843f1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 12:23:42 -0300 Subject: [PATCH 04/13] test: give the disposal pool tests sanitizer-safe waits --- concurrency/src/reply/disposal.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/concurrency/src/reply/disposal.rs b/concurrency/src/reply/disposal.rs index d52133330..df3b5ecf2 100644 --- a/concurrency/src/reply/disposal.rs +++ b/concurrency/src/reply/disposal.rs @@ -186,6 +186,10 @@ mod tests { use super::*; use std::sync::mpsc; + /// Generous: sanitizer builds are slow, and a panic's first report there + /// can take seconds. + const WAIT: Duration = Duration::from_secs(60); + #[test] fn defer_runs_inline_outside_a_scope() { let (tx, rx) = mpsc::channel(); @@ -207,7 +211,7 @@ mod tests { assert!(rx.try_recv().is_err(), "deferred, not run"); assert_eq!(work.len(), 2); submit(work); - assert_eq!(rx.recv_timeout(Duration::from_secs(5)), Ok(1)); + assert_eq!(rx.recv_timeout(WAIT), Ok(1)); } #[test] @@ -216,16 +220,16 @@ mod tests { let (entered_tx, entered_rx) = mpsc::channel(); let blocked: Vec = vec![Box::new(move || { entered_tx.send(()).unwrap(); - let _ = release_rx.recv_timeout(Duration::from_secs(10)); + let _ = release_rx.recv_timeout(WAIT); })]; submit(blocked); entered_rx - .recv_timeout(Duration::from_secs(5)) + .recv_timeout(WAIT) .expect("the first batch started"); let (tx, rx) = mpsc::channel(); submit(vec![Box::new(move || tx.send(()).unwrap())]); assert!( - rx.recv_timeout(Duration::from_secs(5)).is_ok(), + rx.recv_timeout(WAIT).is_ok(), "a blocked worker held back another batch" ); release_tx.send(()).unwrap(); @@ -238,6 +242,6 @@ mod tests { Box::new(|| panic!("injected disposal panic")), Box::new(move || tx.send(()).unwrap()), ]); - assert!(rx.recv_timeout(Duration::from_secs(5)).is_ok()); + assert!(rx.recv_timeout(WAIT).is_ok()); } } From 7ae807b4d79c3dda9e60a1600873c513def152fd Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 12:55:29 -0300 Subject: [PATCH 05/13] fix: queue disposal per item; safe TLS teardown; submit on unwind; last-reference-only hand-off Review follow-ups for the disposal pool: one blocking waker or destructor no longer holds back other items of the same visit (two callers of one actor); deferral after thread-local teardown runs inline instead of aborting; an unwinding visit submits what it collected; registry, job and run references are handed off only when they are the last; the 64-worker cap is logged; docs state where emit credits return. --- concurrency/src/reply/deadline.rs | 2 +- concurrency/src/reply/disposal.rs | 345 +++++++++++++++++++++----- concurrency/src/reply/jobs.rs | 2 +- concurrency/src/reply/registry.rs | 4 +- concurrency/src/reply/slot.rs | 4 + concurrency/src/tasks/jobs.rs | 4 +- concurrency/src/threads/jobs.rs | 4 +- concurrency/tests/deadline_service.rs | 52 ++++ docs/design/deferred-replies.md | 43 ++-- docs/design/semantics.md | 11 +- 10 files changed, 376 insertions(+), 95 deletions(-) diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 4253c565f..19d129a3f 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -316,7 +316,7 @@ mod thread_service { if let Some(next) = registry.service_deadlines(now) { schedule(®istry, next); } - super::super::disposal::defer_drop(registry); + super::super::disposal::defer_drop_arc(registry); } }); }); diff --git a/concurrency/src/reply/disposal.rs b/concurrency/src/reply/disposal.rs index df3b5ecf2..92f7d247f 100644 --- a/concurrency/src/reply/disposal.rs +++ b/concurrency/src/reply/disposal.rs @@ -2,17 +2,17 @@ //! (docs/design/deferred-replies.md §2, "Callback progress"). //! //! The process-wide deadline thread decides expiry: it pops due entries, -//! makes their terminal transitions and returns their credits (atomics and -//! framework locks only). What an expiry releases that can run user code, -//! a caller's waker, a payload destructor routed to the owner's failure -//! sink, the last reference to an entry, is collected while the thread -//! services a visit ([`collect`]) and handed to a disposal worker -//! ([`submit`]). A worker that blocks in user code holds only itself: a -//! batch that finds no idle worker starts another one, so one actor's -//! blocking destructor or waker never delays another actor's expiry or its -//! wake-ups. +//! makes their terminal transitions and returns their deferral credits +//! (atomics and framework locks only). What an expiry releases that can run +//! user code (a caller's waker, a payload destructor routed to the owner's +//! failure sink, the last reference to an entry) is collected while the +//! thread services a visit ([`collect`]) and handed to the disposal workers +//! ([`submit`]) one item at a time. A worker that blocks in user code holds +//! only the item it runs: a queued item that finds no idle worker starts +//! another one, so a blocking destructor or waker delays no other expiry and +//! no other wake-up, of another actor or of the same one. //! -//! Outside a [`collect`] scope ([`defer`], [`defer_drop`]) run inline, so +//! Outside a [`collect`] scope, [`defer`] and [`defer_drop`] run inline, so //! every other thread (owner turns, jobs, the simulated service task) keeps //! its behaviour. The check is one thread-local read, made only on paths //! that already have something to wake or drop. @@ -22,7 +22,7 @@ use std::collections::VecDeque; use std::sync::{Arc, Condvar, Mutex, OnceLock}; use std::time::Duration; -type Work = Box; +pub(crate) type Work = Box; thread_local! { /// `Some` while the deadline thread services a visit. @@ -30,20 +30,17 @@ thread_local! { } /// Run `work` now, or, on the deadline thread inside [`collect`], defer it -/// to a disposal worker. +/// to a disposal worker. Runs inline when this thread's batch is gone (a +/// thread-local destructor dropping a payload at thread exit) or borrowed. #[inline] pub(crate) fn defer(work: impl FnOnce() + Send + 'static) { - let work = BATCH.with(|batch| match batch.try_borrow_mut() { - Ok(mut batch) => match batch.as_mut() { - Some(batch) => { - batch.push(Box::new(work)); - None + let mut work = Some(work); + let _ = BATCH.try_with(|batch| { + if let Ok(mut batch) = batch.try_borrow_mut() { + if let Some(batch) = batch.as_mut() { + batch.push(Box::new(work.take().expect("not yet taken"))); } - None => Some(work), - }, - // Re-entered while the batch is borrowed: never on the service - // path, which only pushes. Run it. - Err(_) => Some(work), + } }); if let Some(work) = work { work(); @@ -56,26 +53,45 @@ pub(crate) fn defer_drop(value: T) { defer(move || drop(value)); } +/// Drop `arc` now, or, when it is the last reference, defer dropping its +/// value (see [`defer`]). A reference that is not the last runs no user +/// code, so it costs no hand-off. +#[inline] +pub(crate) fn defer_drop_arc(arc: Arc) { + if let Some(value) = Arc::into_inner(arc) { + defer_drop(value); + } +} + /// Whether this thread is collecting (the deadline thread in a visit). #[inline] pub(crate) fn collecting() -> bool { - BATCH.with(|batch| batch.try_borrow().is_ok_and(|batch| batch.is_some())) + BATCH + .try_with(|batch| batch.try_borrow().is_ok_and(|batch| batch.is_some())) + .unwrap_or(false) } -/// Run `step` with deferral on, and return what it deferred. The scope -/// ends even if `step` unwinds (the deferred work is then dropped here, -/// which is why the service contains each step inside it). +/// Run `step` with deferral on, and return what it deferred. If `step` +/// unwinds, what it deferred so far is submitted, not dropped here. pub(crate) fn collect(step: impl FnOnce()) -> Vec { struct Scope; + impl Scope { + fn take() -> Vec { + BATCH + .try_with(|batch| batch.borrow_mut().take().unwrap_or_default()) + .unwrap_or_default() + } + } impl Drop for Scope { fn drop(&mut self) { - let _ = BATCH.with(|batch| batch.borrow_mut().take()); + // Only on unwind: the normal path took the batch already. + submit(Self::take()); } } BATCH.with(|batch| *batch.borrow_mut() = Some(Vec::new())); let scope = Scope; step(); - let work = BATCH.with(|batch| batch.borrow_mut().take().unwrap_or_default()); + let work = Scope::take(); drop(scope); work } @@ -88,14 +104,49 @@ pub(crate) fn collect(step: impl FnOnce()) -> Vec { /// only when that many callbacks are blocked at once. const MAX_WORKERS: usize = 64; /// An idle worker exits after this long without work. +#[cfg(not(test))] const KEEP_ALIVE: Duration = Duration::from_secs(10); +/// Short in tests, so submissions race idle exits. +#[cfg(test)] +const KEEP_ALIVE: Duration = Duration::from_millis(2); #[derive(Default)] struct State { - queue: VecDeque>, - /// Workers waiting for work. + queue: VecDeque, + /// Workers waiting for work (or woken and not yet back under the lock). idle: usize, workers: usize, + /// A worker is starting: it will take an item when it runs, so no other + /// starts meanwhile (a burst of quick items does not start one each). + starting: bool, + /// The cap was reported; reset once below it again. + capped: bool, +} + +impl State { + /// Queued work needs another worker: more items than idle workers that + /// could take them, and none starting. Returns whether to start one. + fn needs_worker(&mut self) -> Start { + if self.queue.len() <= self.idle || self.starting { + return Start::No; + } + if self.workers >= MAX_WORKERS { + if std::mem::replace(&mut self.capped, true) { + return Start::No; + } + return Start::Capped; + } + self.workers += 1; + self.starting = true; + Start::Yes + } +} + +enum Start { + No, + Yes, + /// At the cap for the first time since below it: report it. + Capped, } #[derive(Default)] @@ -113,50 +164,74 @@ fn lock(pool: &Pool) -> std::sync::MutexGuard<'_, State> { pool.state.lock().unwrap_or_else(|p| p.into_inner()) } -/// Hand a visit's deferred work to a disposal worker. A batch never waits -/// behind a busy worker: when queued batches outnumber idle workers, a new -/// worker starts (up to [`MAX_WORKERS`]). +/// Hand deferred work to the disposal workers, one queue entry per item, so +/// a blocking item delays none of the others. When queued items outnumber +/// idle workers a new worker starts (one at a time, up to [`MAX_WORKERS`]); +/// each worker re-checks when it takes an item. pub(crate) fn submit(work: Vec) { if work.is_empty() { return; } let pool = pool(); let mut state = lock(pool); - state.queue.push_back(work); - let start = state.queue.len() > state.idle && state.workers < MAX_WORKERS; - if start { - state.workers += 1; - } + let wake = work.len().min(state.idle); + state.queue.extend(work); + let start = state.needs_worker(); drop(state); - if start { - let worker = pool.clone(); - let spawned = std::thread::Builder::new() - .name("spawned-disposal".into()) - .spawn(move || run(&worker)); - if spawned.is_err() { - // No new thread: an existing worker takes the batch. With none - // at all, run it here rather than lose a wake-up. - let orphaned = { - let mut state = lock(pool); - state.workers -= 1; - if state.workers == 0 { - std::mem::take(&mut state.queue) - } else { - VecDeque::new() - } - }; - orphaned.into_iter().for_each(run_batch); + for _ in 0..wake { + pool.ready.notify_one(); + } + start_worker(pool, start); +} + +fn start_worker(pool: &Arc, start: Start) { + match start { + Start::No => {} + Start::Capped => crate::actor_cleanup::log_safely(|| { + tracing::warn!( + event = "disposal_workers_capped", + max = MAX_WORKERS, + "every deadline disposal worker is busy: later wake-ups and \ + payload drops queue until one returns" + ); + }), + Start::Yes => { + let worker = pool.clone(); + let spawned = std::thread::Builder::new() + .name("spawned-disposal".into()) + .spawn(move || run(&worker)); + if spawned.is_err() { + // No new thread: an existing worker takes the work. With + // none at all, run it here rather than lose a wake-up. + let orphaned = { + let mut state = lock(pool); + state.workers -= 1; + state.starting = false; + if state.workers == 0 { + std::mem::take(&mut state.queue) + } else { + VecDeque::new() + } + }; + orphaned.into_iter().for_each(run_item); + } } } - pool.ready.notify_one(); } -fn run(pool: &Pool) { +fn run(pool: &Arc) { let mut state = lock(pool); + state.starting = false; loop { - if let Some(work) = state.queue.pop_front() { + if let Some(item) = state.queue.pop_front() { + // Whatever is still queued must not wait for this item. + if !state.queue.is_empty() && state.idle > 0 { + pool.ready.notify_one(); + } + let start = state.needs_worker(); drop(state); - run_batch(work); + start_worker(pool, start); + run_item(item); state = lock(pool); continue; } @@ -169,16 +244,15 @@ fn run(pool: &Pool) { state.idle -= 1; if timeout.timed_out() && state.queue.is_empty() { state.workers -= 1; + state.capped = false; return; } } } -/// Each item contained, so one failing callback skips none of the others. -fn run_batch(work: Vec) { - for item in work { - super::registry::contain_entry(item); - } +/// Contained, so a failing callback ends neither its worker nor the others. +fn run_item(item: Work) { + super::registry::contain_entry(item); } #[cfg(test)] @@ -214,15 +288,62 @@ mod tests { assert_eq!(rx.recv_timeout(WAIT), Ok(1)); } + #[test] + fn an_unwinding_scope_submits_what_it_collected() { + let (tx, rx) = mpsc::channel(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = collect(|| { + defer(move || tx.send(()).unwrap()); + panic!("injected visit panic"); + }); + })); + crate::mailbox::dispose_panic(result.unwrap_err()); + assert!(!collecting()); + assert!(rx.recv_timeout(WAIT).is_ok(), "collected work was lost"); + } + + #[test] + fn last_reference_drops_are_deferred_others_are_not() { + let shared = Arc::new(()); + let work = collect(|| { + defer_drop_arc(shared.clone()); // not the last: nothing deferred + defer_drop_arc(Arc::new(())); // the last: deferred + }); + assert_eq!(work.len(), 1); + assert_eq!(Arc::strong_count(&shared), 1); + } + + #[test] + fn a_blocked_item_does_not_hold_back_the_next_of_its_batch() { + let (release_tx, release_rx) = mpsc::channel::<()>(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (tx, rx) = mpsc::channel(); + // One visit's work: a blocking callback, then another caller's wake. + submit(vec![ + Box::new(move || { + entered_tx.send(()).unwrap(); + let _ = release_rx.recv_timeout(WAIT); + }), + Box::new(move || tx.send(()).unwrap()), + ]); + entered_rx + .recv_timeout(WAIT) + .expect("the first item started"); + assert!( + rx.recv_timeout(WAIT).is_ok(), + "a blocked item held back the next one of its batch" + ); + release_tx.send(()).unwrap(); + } + #[test] fn a_blocked_batch_does_not_hold_back_the_next() { let (release_tx, release_rx) = mpsc::channel::<()>(); let (entered_tx, entered_rx) = mpsc::channel(); - let blocked: Vec = vec![Box::new(move || { + submit(vec![Box::new(move || { entered_tx.send(()).unwrap(); let _ = release_rx.recv_timeout(WAIT); - })]; - submit(blocked); + })]); entered_rx .recv_timeout(WAIT) .expect("the first batch started"); @@ -244,4 +365,92 @@ mod tests { ]); assert!(rx.recv_timeout(WAIT).is_ok()); } + + /// Submissions race workers going idle and exiting (`KEEP_ALIVE` is 2 ms + /// in tests), with some items blocked meanwhile: every item runs. + #[test] + fn submissions_racing_idle_exits_lose_no_item() { + let (release_tx, release_rx) = mpsc::channel::<()>(); + let release_rx = Arc::new(Mutex::new(release_rx)); + let (tx, rx) = mpsc::channel::(); + let mut expected = 0; + let submitters: Vec<_> = (0..4) + .map(|t| { + let tx = tx.clone(); + let release_rx = release_rx.clone(); + std::thread::spawn(move || { + let mut sent = 0; + for round in 0..150usize { + let mut work: Vec = Vec::new(); + for _ in 0..(round % 3 + 1) { + let tx = tx.clone(); + work.push(Box::new(move || tx.send(1).unwrap())); + sent += 1; + } + if round % 50 == t { + // A blocked callback, released at the end. + let release_rx = release_rx.clone(); + work.insert( + 0, + Box::new(move || { + let _ = release_rx.lock().unwrap().recv_timeout(WAIT); + }), + ); + } + submit(work); + // Around the keep-alive: idle workers time out + // while later submissions arrive. + std::thread::sleep(Duration::from_micros(500 * (round as u64 % 6))); + } + sent + }) + }) + .collect(); + for submitter in submitters { + expected += submitter.join().unwrap(); + } + let mut ran = 0; + while ran < expected { + match rx.recv_timeout(WAIT) { + Ok(n) => ran += n, + Err(_) => break, + } + } + for _ in 0..16 { + let _ = release_tx.send(()); + } + assert_eq!( + ran, expected, + "an item was lost or held behind a blocked one" + ); + } + + /// A thread-local destructor that drops a payload after this thread's + /// batch is destroyed runs it inline instead of aborting. + #[test] + fn deferral_after_thread_local_teardown_runs_inline() { + struct DropsLater(Option>); + impl Drop for DropsLater { + fn drop(&mut self) { + let tx = self.0.take().unwrap(); + let collecting = collecting(); + let sender = tx.clone(); + defer(move || sender.send(true).unwrap()); + let _ = tx.send(collecting); + } + } + thread_local! { + static LATER: RefCell> = const { RefCell::new(None) }; + } + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + // Registered first, so destroyed after the batch (reverse order). + LATER.with(|later| *later.borrow_mut() = Some(DropsLater(Some(tx)))); + let _ = collect(|| {}); + }) + .join() + .expect("a deferral during thread-local teardown must not abort"); + let got: Vec = rx.try_iter().collect(); + assert_eq!(got, [true, false], "ran inline, not collecting"); + } } diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index ad58791e7..aa382cb23 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -1117,7 +1117,7 @@ impl Expirable for ScheduledEmit { if let Some(job) = self.job.upgrade() { self.settle(&job, true); // Possibly the last reference (its other scheduled emits). - super::disposal::defer_drop(job); + super::disposal::defer_drop_arc(job); } } } diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index f6def99fa..64c8cfac7 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -65,7 +65,9 @@ impl Expirable for Slot { fn expire(&self, now: u64) { if let Some(mut wake) = self.expire_collect(now) { wake.notify_parked(); - super::disposal::defer(move || wake.run()); + if !wake.is_empty() { + super::disposal::defer(move || wake.run()); + } } } } diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 066a6b530..f1ce7be0b 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -343,6 +343,10 @@ impl Wake { } } } + /// Nothing left to run: no waker and no watchers. + pub(crate) fn is_empty(&self) -> bool { + self.waker.is_none() && self.watchers.is_empty() + } #[inline] pub(crate) fn run(self) { if let Some(waker) = self.waker { diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 928478f1c..a670d5f14 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -198,7 +198,7 @@ where } // Possibly the last reference (`done` and its captures): // off the deadline service thread. - crate::reply::disposal::defer_drop(run); + crate::reply::disposal::defer_drop_arc(run); }) }, ); @@ -721,7 +721,7 @@ impl Context { if let Some(run) = weak.upgrade() { run.notify(Err(JobError::DeadlineExceeded)); // Possibly the last reference: off the deadline thread. - crate::reply::disposal::defer_drop(run); + crate::reply::disposal::defer_drop_arc(run); } }))), }); diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 2e73ef280..4115d74ef 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -180,7 +180,7 @@ where } // Possibly the last reference (`done` and its captures): // off the deadline service thread. - crate::reply::disposal::defer_drop(run); + crate::reply::disposal::defer_drop_arc(run); }) }, ); @@ -492,7 +492,7 @@ impl Context { if let Some(run) = weak.upgrade() { run.notify(Err(JobError::DeadlineExceeded)); // Possibly the last reference: off the deadline thread. - crate::reply::disposal::defer_drop(run); + crate::reply::disposal::defer_drop_arc(run); } }))), }); diff --git a/concurrency/tests/deadline_service.rs b/concurrency/tests/deadline_service.rs index 628ae302f..63d0c6f70 100644 --- a/concurrency/tests/deadline_service.rs +++ b/concurrency/tests/deadline_service.rs @@ -313,6 +313,58 @@ fn deadline_service_blocking_destructor_child() { println!("blocking destructor isolated"); } +#[test] +fn deadline_service_isolates_callers_of_one_actor() { + subprocess::run( + "deadline_service_same_actor_child", + "same-actor callers isolated", + ); +} + +/// Two callers of one actor with the same deadline, expired in the same +/// service visit: the first caller's waker blocks, the second caller must +/// still be woken with its timeout. +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_same_actor_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let blocker = std::sync::Arc::new(Blocker::default()); + tokio_runtime().block_on(async { + let owner = Owner::default().start(); + let deadline = Deadline::after(Duration::from_millis(100)); + let first = owner.send_request(Park, deadline).unwrap(); + let second = owner.send_request(Park, deadline).unwrap(); + // The first is polled (and put in the heap) first, so its wake-up is + // collected first in the visit. + let mut first = Box::pin(std::future::IntoFuture::into_future(first)); + let waker = std::task::Waker::from(std::sync::Arc::new(BlockingWake(blocker.clone()))); + let mut cx = std::task::Context::from_waker(&waker); + assert!(std::future::Future::poll(first.as_mut(), &mut cx).is_pending()); + // The wake-up must come with the expiry: a final poll at the outer + // timeout would find the slot expired anyway, so time it. + let started = std::time::Instant::now(); + let outcome = tokio::time::timeout(Duration::from_secs(3), second).await; + let woken_after = started.elapsed(); + let entered = blocker.entered.wait(Duration::from_secs(5)); + blocker.release.set(); + drop(first); + owner.context().stop(); + owner.join().await; + assert!(entered, "the first caller's waker never ran"); + assert!( + matches!(outcome, Ok(Err(ActorError::RequestTimeout))), + "unexpected outcome {outcome:?}" + ); + assert!( + woken_after < Duration::from_millis(1500), + "a blocking waker delayed another caller of the same actor ({woken_after:?})" + ); + }); + println!("same-actor callers isolated"); +} + #[test] #[ignore = "subprocess helper exercised by the enabled parent"] fn deadline_service_blocking_waker_child() { diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 25c829085..cfd1695db 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -291,18 +291,23 @@ handles a call even after the caller timed out, and the late reply is dropped. - **Callback progress (disposal off the service thread).** The service thread only decides expiry: it pops due entries, makes the terminal transition (`OPEN -> TIMED_OUT`, a job's `JobDeadline` notification or a - scheduled emit's fire) and returns the credit, all under framework locks. - It also unparks a parked blocking receiver and enqueues a job's message on - its owner's mailbox (the runtime wake of the owner's own task). Anything an - expiry releases that can run user code is collected during the visit and - handed to a disposal worker (`reply/disposal.rs`): the caller's waker and - the token's watchers, payloads routed to the owner's failure sink (a held - or refused job result, a discarded emit, `done` and its captures), and the - last references to entries, jobs and the registry. Disposal workers form an - elastic pool: a batch that finds no idle worker starts one (up to 64, - idle ones exit after 10 s), so a destructor or waker that blocks holds - only its own worker, and no other actor's expiry, credit return or - wake-up waits for it. + scheduled emit's fire) and returns the deferral credit, all under + framework locks. It also unparks a parked blocking receiver and enqueues a + job's message on its owner's mailbox (the runtime wake of the owner's own + task). Anything an expiry releases that can run user code is collected + during the visit and handed to the disposal workers (`reply/disposal.rs`): + the caller's waker and the token's watchers, payloads routed to the + owner's failure sink (a held or refused job result, a discarded emit, + `done` and its captures), the due entries, and the last references to + jobs, runs and the registry (a reference that is not the last is dropped + in place: it runs no user code). Each item is queued on its own, so a + destructor or waker that blocks holds only the worker running it: no + other expiry, credit return or wake-up waits for it, of another actor or + of the same one (two callers of one actor expiring in one visit are woken + independently). Disposal workers form an elastic pool: when queued items + outnumber idle workers one more starts (one start at a time, up to 64; + idle ones exit after 10 s), and each worker re-checks when it takes an + item. Reaching the cap logs `disposal_workers_capped` once. - *Why a pool and not the owner.* Deferring disposal to the owning actor would tie a caller's wake-up to the owner's next turn (a wedged owner would hang its callers past their deadline), and deferring it to the @@ -313,14 +318,20 @@ handles a call even after the caller timed out, and the late reply is dropped. wake-up as before (a caller that polls in between sees the terminal state); a payload handed off is discarded again on the worker, where the owner's hold is re-checked, so a drop deferred before the owner's exit - still waits for its death point (R6, callers before payloads). + still waits for its death point (R6, callers before payloads). If a + visit unwinds, what it collected is still submitted, so no wake-up is + lost. A deferral from a thread-local destructor after this thread's + batch is gone runs inline instead of aborting. + - *Credits.* Deferral credits (`MAX_OUTSTANDING`) return on the service + thread with the transition. A scheduled emit's per-job emit credit + travels inside its message; when the emit is discarded or refused at its + instant, that credit returns when the message is dropped on a worker. - *Cost.* None on a request or reply that does not expire: the hand-off is in the service's own path, and the shared helpers test a thread-local - only when they already have a waker or a payload to dispose of. The - perf-gate driver measures within noise. + only when they already have a waker or a payload to dispose of. - *Remaining assumption.* The owner's own executor wake (a runtime waker or condvar) still runs on the service thread; it is framework code. With - more than 64 callbacks blocked at once, later batches queue behind them. + 64 callbacks blocked at once, later items queue behind them. Ordinary request/Response runtime timers and send_after/send_interval drivers are separate (see semantics.md, section 10). - **Simulation.** Under `spawned_sim` the service is a task of the simulated diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 183d56c08..446f296d3 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -161,8 +161,9 @@ the value), as an OTP reply already in the caller's mailbox when its `after` clause would fire is received. Timer resolution and when the runtime or deadline service can process expiry determine that window; this is not a hard real-time bound. The deadline service runs no user code: the wakers and -payload destructors an expiry releases run on disposal workers, so another -actor's blocking destructor or waker does not delay an expiry, its credit +payload destructors an expiry releases run on disposal workers, one item at a +time, so a blocking destructor or waker (of another actor, or of another +caller of the same actor) does not delay an expiry, its deferral-credit return or its caller's wake-up ([`deadline_service`][deadline_service]). Which clock expires a waiting caller is part of the contract: `request`, @@ -771,8 +772,10 @@ stopped after `Shutdown`), deadline-service thread runs no user code (section 3): wakers and payload destructors an expiry releases go to a pool of at most 64 disposal workers. With that many callbacks blocked at once, later disposals (and so - their callers' wake-ups) queue behind them; expiry and credit return do - not. No callback is preempted. + their callers' wake-ups) queue behind them; expiry and deferral-credit + return do not. A discarded scheduled emit's per-job emit credit returns + with its message's disposal, so it can queue too. No callback is + preempted. - **Bounded memory.** Mailboxes are unbounded (D5); there are no per-actor memory limits (rank 18a). - **Deadlock detection.** A request cycle, or a handler requesting its own From 32ecaab89561fd3835bc6175b4b8d012c35630d8 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 12:58:37 -0300 Subject: [PATCH 06/13] docs: changelog for per-item disposal --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9faa0722f..ed9f3286b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,9 +83,10 @@ limitation). The thread still decides expiry and returns credits, but the wakers and payload destructors an expiry releases (a caller's waker, a discarded scheduled emit, a displaced job result, a `done` closure and its - captures) now run on an elastic pool of disposal workers, so a blocking - destructor or waker in one actor no longer delays other actors' deadlines. - A payload handed off still waits for its owner's death point (R6). No cost + captures) now run on an elastic pool of disposal workers, one item at a + time, so a blocking destructor or waker no longer delays other actors' + deadlines or other callers of the same actor. A payload handed off still + waits for its owner's death point (R6). No cost on requests that do not expire. New regression tests (tasks and threads), a Loom case for the handed-off wake-up, and an `InlineDisposal` negative control in `DeferredReplies.tla`. From 9c049c19533a1c8ca64b24b25b25ee19ed99d09a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 13:14:32 -0300 Subject: [PATCH 07/13] verification: model and trace-validate the deadline service's disposal hand-off (Disposal.tla) --- .github/workflows/verification.yml | 2 +- concurrency/src/failpoints.rs | 8 + concurrency/src/lifecycle_trace.rs | 102 ++++++++- concurrency/src/mailbox.rs | 75 +++++-- concurrency/src/reply/deadline.rs | 4 + concurrency/src/reply/disposal.rs | 1 + concurrency/src/reply/registry.rs | 15 +- concurrency/tests/disposal_trace.rs | 241 +++++++++++++++++++++ verification/model-map.toml | 28 +++ verification/tla/Disposal.tla | 277 ++++++++++++++++++++++++ verification/tla/MCDisposal.cfg | 13 ++ verification/tla/MCDisposal.tla | 5 + verification/tla/MCDisposalDrop.cfg | 13 ++ verification/tla/MCDisposalDrop.tla | 5 + verification/tla/TraceDisposal.tla | 65 ++++++ verification/tla/coverage-baseline.json | 9 + verification/tla/negative_checks.py | 30 +++ verification/tla/trace_check.py | 50 ++++- verification/tla/trace_coverage.py | 1 + 19 files changed, 920 insertions(+), 24 deletions(-) create mode 100644 concurrency/tests/disposal_trace.rs create mode 100644 verification/tla/Disposal.tla create mode 100644 verification/tla/MCDisposal.cfg create mode 100644 verification/tla/MCDisposal.tla create mode 100644 verification/tla/MCDisposalDrop.cfg create mode 100644 verification/tla/MCDisposalDrop.tla create mode 100644 verification/tla/TraceDisposal.tla diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index c29b9984f..c1e628da6 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -81,7 +81,7 @@ jobs: - name: Record supervisor lifecycle traces env: LIFECYCLE_TRACE_DIR: ${{ github.workspace }}/lifecycle-traces - run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace --test coordinator_trace + run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test disposal_trace --test lifecycle_trace --test coordinator_trace - name: Validate traces and negative controls run: nix run .#check-tla -- --traces-only lifecycle-traces --negative-controls # Which model actions, projected states and transitions the recorded diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index ba32a6dd7..814d5fd69 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -79,6 +79,14 @@ pub const SITES: &[Site] = &[ refused, and its payload is destroyed only after every caller is \ answered (tasks and threads).", }, + Site { + name: "disposal.item", + has_error: false, + description: "A disposal worker is about to run an item the deadline service \ + handed off (a wake-up or a payload drop). Delay lets the owner's \ + exit begin first: a payload handed off before it is held until the \ + death point, not dropped by the worker.", + }, Site { name: "job.native_worker_unused", has_error: false, diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index b4eeaa555..44aae9063 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -180,6 +180,32 @@ impl ReplyEvent { } } +/// One action of `verification/tla/Disposal.tla`: the deadline service's +/// expiry and hand-off, the disposal workers' steps, and the owner's payload +/// hold. `Expire` (a slot's transition on the service thread; `x`: `wake` if +/// it handed off a wake-up, else `none`), `Release` (a payload deferred from +/// the service thread), `Submit` (a visit's work handed to the pool), `Wake` +/// (a worker runs a handed-off wake-up), `Drop` (a worker settles a payload; +/// `x`: `held` or `dropped`), `Exit` (the owner's hold begins) and `Death` +/// (its hold is released at the death point). `id` numbers the slot or +/// payload (fresh per Expire/Release; normalized by the checker). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DisposalEvent { + pub action: &'static str, + pub id: u64, + pub x: &'static str, +} + +impl DisposalEvent { + /// JSON object understood by `verification/tla/trace_check.py`. + pub fn to_json(&self) -> String { + format!( + r#"{{"a":"{}","id":{},"x":"{}"}}"#, + self.action, self.id, self.x + ) + } +} + /// One base-lifecycle action of `verification/tla/Lifecycle.tla`, recorded /// for the actors spawned while `record_lifecycle` is on. `t` is the actor /// the action belongs to (the target of a signal or message, the caller of @@ -486,6 +512,66 @@ mod imp { }) } + // Deadline-service disposal (`Disposal.tla`): one process-wide recording + // at a time. + static DISPOSAL_ON: AtomicBool = AtomicBool::new(false); + static DISPOSAL: Mutex> = Mutex::new(Vec::new()); + + /// Start recording disposal events until [`take_disposal`]. Tests that + /// record must not run concurrently with each other or with other + /// deadline expiries or actor exits in the process. + pub fn record_disposal() { + atomic(|| { + lock(&DISPOSAL).clear(); + DISPOSAL_ON.store(true, Ordering::Release); + }); + } + + /// Stop recording and return the events in their recorded order. + pub fn take_disposal() -> Vec { + atomic(|| { + DISPOSAL_ON.store(false, Ordering::Release); + std::mem::take(&mut *lock(&DISPOSAL)) + }) + } + + fn disposal_on() -> bool { + DISPOSAL_ON.load(Ordering::Acquire) + } + + /// Run `f` as one atomic step of the recorded linearization while + /// disposal events are recorded (and as is otherwise). + pub(crate) fn disposal_section(f: impl FnOnce() -> R) -> R { + if disposal_on() { + atomic(f) + } else { + f() + } + } + + /// A fresh slot or payload number for a disposal event (0 when off). + pub(crate) fn disposal_id() -> u64 { + static NEXT: AtomicU64 = AtomicU64::new(1); + if disposal_on() { + NEXT.fetch_add(1, Ordering::Relaxed) + } else { + 0 + } + } + + /// Record a disposal action (inside the section of the step it names). + /// An `id` of 0 (taken while recording was off) is not recorded. + pub(crate) fn disposal(action: &'static str, id: u64, x: &'static str) { + if !disposal_on() { + return; + } + atomic(|| { + if id != 0 || matches!(action, "Submit" | "Exit" | "Death") { + lock(&DISPOSAL).push(super::DisposalEvent { action, id, x }); + } + }); + } + /// Whether actor `id` is recorded. pub(crate) fn life_traced(id: u64) -> bool { LIFE_ON.load(Ordering::Acquire) && lock(&MEMBERS).contains(&id) @@ -1082,10 +1168,22 @@ mod imp { pub(crate) fn in_actor(_: u64, turn: T) -> T { turn } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn disposal(_: &'static str, _: u64, _: &'static str) {} + #[inline(always)] + pub(crate) fn disposal_section(f: impl FnOnce() -> R) -> R { + f() + } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn disposal_id() -> u64 { + 0 + } } #[cfg(feature = "lifecycle-trace")] pub use imp::{ - record_lifecycle, recorded, recorded_with, take, take_jobs, take_lifecycle, take_replies, - take_tree, watch_deferred, + record_disposal, record_lifecycle, recorded, recorded_with, take, take_disposal, take_jobs, + take_lifecycle, take_replies, take_tree, watch_deferred, }; diff --git a/concurrency/src/mailbox.rs b/concurrency/src/mailbox.rs index af311bf5a..d8c560619 100644 --- a/concurrency/src/mailbox.rs +++ b/concurrency/src/mailbox.rs @@ -31,35 +31,74 @@ impl DropFailures { } /// The owner's exit began: hold every later drop until `release`. pub(crate) fn hold(&self) { - self.held - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get_or_insert_with(Vec::new); + // Disposal.tla `Exit`, linearized with the hold checks below. + crate::lifecycle_trace::disposal_section(|| { + let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); + if held.is_none() { + *held = Some(Vec::new()); + crate::lifecycle_trace::disposal("Exit", 0, ""); + } + }); } /// Every caller was answered: drop what was held (contained), and drop /// at once from now on. Idempotent. pub(crate) fn release(&self) { - let held = self.held.lock().unwrap_or_else(|p| p.into_inner()).take(); + // Disposal.tla `Death`: the hold ends. + let held = crate::lifecycle_trace::disposal_section(|| { + let held = self.held.lock().unwrap_or_else(|p| p.into_inner()).take(); + if held.is_some() { + crate::lifecycle_trace::disposal("Death", 0, ""); + } + held + }); for value in held.into_iter().flatten() { self.drop_contained(value); } } /// On the deadline service thread the drop goes to a disposal worker, - /// which discards it again there: the hold is re-checked, so a drop - /// deferred before the owner's exit still waits for its death point. + /// which settles it there ([`settle`](Self::settle)): the hold is + /// re-checked, so a drop deferred before the owner's exit still waits for + /// its death point. pub(crate) fn discard(self: &Arc, value: T) { - let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); - if let Some(held) = held.as_mut() { - // Moved, not dropped: no user code under the lock. - held.push(Box::new(value)); - return; - } - drop(held); - if crate::reply::disposal::collecting() { - let failures = self.clone(); - crate::reply::disposal::defer(move || failures.discard(value)); - return; + let mut value = Some(value); + // One step of Disposal.tla with `Exit` and `Death`: the hold check + // and, on the service thread, `Release` (the hand-off). + let inline = crate::lifecycle_trace::disposal_section(|| { + let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(held) = held.as_mut() { + // Moved, not dropped: no user code under the lock. + held.push(Box::new(value.take())); + return false; + } + drop(held); + if crate::reply::disposal::collecting() { + let id = crate::lifecycle_trace::disposal_id(); + crate::lifecycle_trace::disposal("Release", id, ""); + let failures = self.clone(); + let value = value.take(); + crate::reply::disposal::defer(move || failures.settle(value, id)); + return false; + } + true + }); + if inline { + self.drop_contained(value); } + } + /// A disposal worker's step for a payload the service handed off: held + /// if the owner's exit began meanwhile (dropped at its death point), + /// else dropped now (Disposal.tla `Drop`). + fn settle(&self, value: T, _id: u64) { + let mut value = Some(value); + crate::lifecycle_trace::disposal_section(|| { + let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(held) = held.as_mut() { + held.push(Box::new(value.take())); + crate::lifecycle_trace::disposal("Drop", _id, "held"); + } else { + crate::lifecycle_trace::disposal("Drop", _id, "dropped"); + } + }); self.drop_contained(value); } /// Drop at once (contained): the mailbox's own items, destroyed after diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 19d129a3f..681c0a226 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -320,6 +320,10 @@ mod thread_service { } }); }); + // Disposal.tla `Submit`: the visit's work goes to the pool. + if !released.is_empty() { + crate::lifecycle_trace::disposal("Submit", 0, ""); + } super::super::disposal::submit(released); state = lock(service); } diff --git a/concurrency/src/reply/disposal.rs b/concurrency/src/reply/disposal.rs index 92f7d247f..ca9fbce33 100644 --- a/concurrency/src/reply/disposal.rs +++ b/concurrency/src/reply/disposal.rs @@ -252,6 +252,7 @@ fn run(pool: &Arc) { /// Contained, so a failing callback ends neither its worker nor the others. fn run_item(item: Work) { + fail_point!("disposal.item"); super::registry::contain_entry(item); } diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 64c8cfac7..9f811a8e8 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -65,8 +65,21 @@ impl Expirable for Slot { fn expire(&self, now: u64) { if let Some(mut wake) = self.expire_collect(now) { wake.notify_parked(); + // Disposal.tla `Expire` (after the transition and its credit + // return) and, on a worker, `Wake`. Compiled out without + // `lifecycle-trace`; recorded only on the collecting service. + let id = if cfg!(feature = "lifecycle-trace") && super::disposal::collecting() { + crate::lifecycle_trace::disposal_id() + } else { + 0 + }; + let kind = if wake.is_empty() { "none" } else { "wake" }; + crate::lifecycle_trace::disposal("Expire", id, kind); if !wake.is_empty() { - super::disposal::defer(move || wake.run()); + super::disposal::defer(move || { + crate::lifecycle_trace::disposal("Wake", id, ""); + wake.run() + }); } } } diff --git a/concurrency/tests/disposal_trace.rs b/concurrency/tests/disposal_trace.rs new file mode 100644 index 000000000..ca5834730 --- /dev/null +++ b/concurrency/tests/disposal_trace.rs @@ -0,0 +1,241 @@ +//! Records the deadline service's hand-off to the disposal workers from real +//! actors, for TLA+ trace validation against `verification/tla/Disposal.tla` +//! (verification/tla/README.md). Each scenario writes `disposal_.json` +//! to `$LIFECYCLE_TRACE_DIR` (default `target/lifecycle-traces`); +//! `verification/tla/trace_check.py` checks them (schema `disposal-v1`). +//! +//! The recording is process-wide (the service is), so scenarios run one at a +//! time and each waits for its workers before taking the trace. +#![cfg(all(feature = "lifecycle-trace", feature = "async"))] + +use spawned_concurrency::{ + lifecycle_trace, + message::Message, + tasks::{Actor, ActorRef, ActorStart, Context, Handler, Instant, JobOptions, JobOutcome}, + ActorError, Deadline, PendingReply, ReplyTo, +}; +use std::{ + path::PathBuf, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +/// One recording at a time (held across awaits). +static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn trace_dir() -> PathBuf { + std::env::var_os("LIFECYCLE_TRACE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/lifecycle-traces") + }) +} + +/// Stop recording and write the trace; returns the events as `Action:x`. +fn write(name: &str) -> Vec { + let events = lifecycle_trace::take_disposal(); + let json = format!( + "{{\"name\":\"disposal_{name}\",\"schema\":\"disposal-v1\",\"events\":[\n{}\n]}}\n", + events + .iter() + .map(|e| e.to_json()) + .collect::>() + .join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("disposal_{name}.json")), json).unwrap(); + events + .iter() + .map(|e| { + if e.x.is_empty() { + e.action.to_string() + } else { + format!("{}:{}", e.action, e.x) + } + }) + .collect() +} + +fn count(events: &[String], event: &str) -> usize { + events.iter().filter(|e| *e == event).count() +} + +/// A payload whose destructor is counted. +struct Payload(Arc); +impl Message for Payload { + type Result = (); +} +impl Drop for Payload { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +struct Done; +impl Message for Done { + type Result = (); +} + +/// Parks requests (deferred, never answered); `stopped()` waits for `gate`. +#[derive(Default)] +struct Owner { + parked: Vec>, + gate: Option<(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>)>, +} +impl Actor for Owner { + async fn stopped(&mut self, _: &Context) { + if let Some((gate, stopping)) = self.gate.take() { + let _ = stopping.send(()); + let _ = gate.recv_timeout(Duration::from_secs(10)); + } + } +} +struct Park; +impl Message for Park { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Park, reply: PendingReply<'_, ()>, _: &Context) { + self.parked.push(reply.defer().unwrap()); + } +} +impl Handler for Owner { + async fn handle(&mut self, _: Payload, _: PendingReply<'_, ()>, _: &Context) {} +} +impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} +} + +/// A job whose deadline (20 ms) takes its notification while the work runs; +/// its emit scheduled at 40 ms is then discarded on the deadline service. +async fn discard_an_emit(owner: &ActorRef, drops: &Arc) { + let payload = Payload(drops.clone()); + spawned_concurrency::__private::in_turn(owner, move |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + move |job| { + let at = Instant::now() + Duration::from_millis(40); + job.emit_at_blocking(at, payload).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + }) + .unwrap() + .await + .unwrap(); +} + +/// Expired requests: two waiting callers are woken by workers, a third +/// (never polled) has nothing to wake. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn expired_callers_are_woken_by_workers() { + let _serial = SERIAL.lock().await; + let owner = Owner::default().start(); + lifecycle_trace::record_disposal(); + let deadline = Deadline::after(Duration::from_millis(50)); + let first = owner.send_request(Park, deadline).unwrap(); + let second = owner.send_request(Park, deadline).unwrap(); + let _unpolled = owner.send_request(Park, deadline).unwrap(); + let (a, b) = tokio::join!(first, second); + assert!(matches!(a, Err(ActorError::RequestTimeout))); + assert!(matches!(b, Err(ActorError::RequestTimeout))); + tokio::time::sleep(Duration::from_millis(100)).await; + owner.context().stop(); + owner.join().await; + tokio::time::sleep(Duration::from_millis(50)).await; + let events = write("expired_callers"); + assert_eq!(count(&events, "Expire:wake"), 2, "{events:?}"); + assert_eq!(count(&events, "Expire:none"), 1, "{events:?}"); + assert_eq!(count(&events, "Wake"), 2, "{events:?}"); + assert!(count(&events, "Submit") >= 1, "{events:?}"); +} + +/// Payloads released on the deadline service are dropped by workers while +/// the owner runs. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn released_payloads_are_dropped_by_workers() { + let _serial = SERIAL.lock().await; + let owner = Owner::default().start(); + let drops = Arc::new(AtomicUsize::new(0)); + lifecycle_trace::record_disposal(); + discard_an_emit(&owner, &drops).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + drops.load(Ordering::SeqCst), + 1, + "the discarded emit was dropped" + ); + owner.context().stop(); + owner.join().await; + tokio::time::sleep(Duration::from_millis(50)).await; + let events = write("released_payloads"); + assert!(count(&events, "Release") >= 1, "{events:?}"); + assert_eq!( + count(&events, "Drop:dropped"), + count(&events, "Release"), + "{events:?}" + ); + assert_eq!(count(&events, "Drop:held"), 0, "{events:?}"); +} + +/// A payload handed off before the owner's exit, reaching its worker after +/// the exit began (the worker is delayed at `disposal.item`), is held and +/// dropped only at the death point. +#[cfg(feature = "failpoints")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_payload_handed_off_before_exit_waits_for_the_death_point() { + use spawned_concurrency::failpoints::{self, Action, FailPoint, FailScenario}; + let _serial = SERIAL.lock().await; + let _scenario = FailScenario::setup(); + let (release, gate) = std::sync::mpsc::channel(); + let (stopping_tx, stopping) = std::sync::mpsc::channel(); + let owner = Owner { + gate: Some((gate, stopping_tx)), + ..Owner::default() + } + .start(); + let drops = Arc::new(AtomicUsize::new(0)); + failpoints::configure( + "disposal.item", + FailPoint::new(Action::Delay(Duration::from_millis(400))), + ) + .unwrap(); + lifecycle_trace::record_disposal(); + discard_an_emit(&owner, &drops).await; + // The emit is discarded at 40 ms; its worker waits until ~440 ms. + tokio::time::sleep(Duration::from_millis(100)).await; + owner.context().stop(); + tokio::task::spawn_blocking(move || stopping.recv_timeout(Duration::from_secs(5))) + .await + .unwrap() + .expect("stopped() began"); + // The worker settles the payload while stopped() still runs. + tokio::time::sleep(Duration::from_millis(600)).await; + let before_death = drops.load(Ordering::SeqCst); + release.send(()).unwrap(); + owner.join().await; + failpoints::remove("disposal.item"); + tokio::time::sleep(Duration::from_millis(50)).await; + let events = write("held_until_death"); + assert_eq!( + before_death, 0, + "a payload handed off before the exit was dropped before the death point" + ); + assert_eq!( + drops.load(Ordering::SeqCst), + 1, + "dropped at the death point" + ); + assert!(count(&events, "Drop:held") >= 1, "{events:?}"); + let held = events.iter().position(|e| e == "Drop:held").unwrap(); + let exit = events.iter().position(|e| e == "Exit").unwrap(); + let death = events.iter().position(|e| e == "Death").unwrap(); + assert!(exit < held && held < death, "{events:?}"); +} diff --git a/verification/model-map.toml b/verification/model-map.toml index c9f8d5b90..134bf9012 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -137,6 +137,25 @@ code = [ ] coverage_floor = 14 # of 14 action keys (TraceReplies arms) +[models.Disposal] +# The deadline service's hand-off of user code to the disposal workers: +# expiry and credit return on the service thread, per-item queueing, the +# worker steps (wake after the transition, payload drops re-checking the +# owner's hold), the elastic pool, and the owner's hold until death. +files = [ + "verification/tla/Disposal.tla", + "verification/tla/MCDisposal*", + "verification/tla/TraceDisposal.tla", +] +code = [ + "concurrency/src/reply/disposal.rs", + "concurrency/src/reply/deadline.rs", + "concurrency/src/reply/registry.rs", + "concurrency/src/mailbox.rs", + "concurrency/src/lifecycle_trace.rs", +] +coverage_floor = 7 # of 7 action keys (TraceDisposal arms) + [models.ReplyTimer] # Receiver-side refinement: the shared runtime deadline and shortening mark. files = ["verification/tla/ReplyTimer.tla", "verification/tla/MCReplyTimer*"] @@ -323,6 +342,15 @@ specs = ["verification/tla/TraceJobs.tla"] [trace.unlogged] Restart = "a new owner instance reading the old one's slot-addressed mailbox: the code has no such mailbox (a job's messages go to its owner instance's own mailbox, discarded at that instance's death), so the model's cross-instance delivery is a conservative over-approximation (NoLateResult holds a fortiori); a trace is one instance" +[[trace]] +struct = "DisposalEvent" +emitters = [ + { call = '\blifecycle_trace::disposal\(', arg = 0 }, +] +specs = ["verification/tla/TraceDisposal.tla"] +[trace.unlogged] +Commit = "fault device only (Fault = WakeBeforeTransition: the transition committed after the hand-off); the code commits in Expire" + [[trace]] struct = "ReplyEvent" emitters = [ diff --git a/verification/tla/Disposal.tla b/verification/tla/Disposal.tla new file mode 100644 index 000000000..94dda49b6 --- /dev/null +++ b/verification/tla/Disposal.tla @@ -0,0 +1,277 @@ +------------------------------ MODULE Disposal ------------------------------ +(***************************************************************************) +(* The deadline service's hand-off of user code to disposal workers *) +(* (concurrency/src/reply/disposal.rs, deadline.rs, registry.rs, and *) +(* DropFailures in mailbox.rs; docs/design/deferred-replies.md §2, *) +(* "Callback progress"). A companion of DeferredReplies.tla, which models *) +(* the reply slots themselves and takes the service's expiry as one step. *) +(* *) +(* One owner's deadline heap holds N requests (deferred tokens or awaited *) +(* raw receivers), each with a deadline fixed at Init and possibly a *) +(* waiting caller (a waker). The one process-wide service thread visits *) +(* the heap: *) +(* Expire(r) the slot's OPEN -> TIMED_OUT transition and its credit *) +(* return, in one step; a waiting caller's wake-up is put *) +(* in the visit's batch (not run) *) +(* Release(p) a payload the visit releases (a discarded scheduled emit, *) +(* a displaced job result, `done` and its captures) is put *) +(* in the batch; one released while the owner's hold is on *) +(* is held at once and is no step (DropFailures::discard) *) +(* Submit the visit's batch goes to the pool, one queue entry per *) +(* item *) +(* Workers (at most Workers) are started while queued entries outnumber *) +(* idle workers, take entries, run them, and retire when idle with nothing *) +(* queued. Running a wake-up wakes the caller; running a payload re-checks *) +(* the owner's hold: held if its exit began (dropped at its death point), *) +(* dropped otherwise. Items in Blocking never return (a blocking waker or *) +(* destructor): their worker is lost, nothing else waits for it. *) +(* *) +(* The owner: Exit (DropFailures::hold, its exit begins) and Death (the *) +(* death point; DropFailures::release drops what was held). *) +(* *) +(* The clock: Tick only when no framework step is pending (an expiry due, *) +(* a batch not submitted, a runnable or startable worker), so a property *) +(* "within the tick" says the step does not wait for a blocked item. *) +(* *) +(* Fault selects a negative control; negative_checks.py requires each to *) +(* violate the named invariant: *) +(* InlineDisposal (TimelyExpiry) the service runs the batch *) +(* itself (0.8.0): a blocking item *) +(* stops every later expiry *) +(* WakeBeforeTransition (WakeAfterTransition) the wake-up is handed off *) +(* before the transition commits *) +(* DropWhileHeld (NoDropWhileHeld) the worker drops a payload *) +(* without re-checking the hold *) +(* BatchSerial (TimelyWake) a visit's batch is one queue *) +(* entry run serially: a blocking *) +(* item holds back the other *) +(* callers' wake-ups (review of *) +(* PR #282) *) +(***************************************************************************) +EXTENDS Naturals, Sequences, FiniteSets + +CONSTANTS N, P, MaxTime, Workers, Blocking, Fault +ASSUME Fault \in {"None", "InlineDisposal", "WakeBeforeTransition", + "DropWhileHeld", "BatchSerial"} + +Req == 1..N +Pay == 1..P +W == 1..Workers +WakeItem(r) == [k |-> "wake", id |-> r] +DropItem(p) == [k |-> "drop", id |-> p] +Items == {WakeItem(r) : r \in Req} \cup {DropItem(p) : p \in Pay} +ASSUME Blocking \subseteq Items + +VARIABLES + now, dl, waiting, \* the clock; each request's deadline and caller + slot, credit, expAt, \* "open"/"timeout"; its credit; when it expired + woken, \* the caller's wake-up ran + cas, \* WakeBeforeTransition: handed off, not committed + payload, \* "none" (owned), "out" (handed off), "held", "dropped" + batch, \* the current visit's collected items + queue, \* queued entries (each a sequence of items) + cur, \* the entry a worker runs (<<>>: idle or not started) + alive, \* started workers + svc, \* "idle", or "stuck" in a blocking item (InlineDisposal) + phase, \* the owner: "run", "exiting" (hold on), "dead" + early \* a payload was dropped while the hold was on + +vars == <> + +Init == + /\ now = 0 + /\ dl \in [Req -> 1..MaxTime] + /\ waiting \in [Req -> BOOLEAN] + /\ slot = [r \in Req |-> "open"] + /\ credit = [r \in Req |-> TRUE] + /\ expAt = [r \in Req |-> 0] + /\ woken = [r \in Req |-> FALSE] + /\ cas = {} + /\ payload = [p \in Pay |-> "none"] + /\ batch = <<>> + /\ queue = <<>> + /\ cur = [w \in W |-> <<>>] + /\ alive = {} + /\ svc = "idle" + /\ phase = "run" + /\ early = FALSE + +------------------------------------------------------------------------------ +\* The service thread + +\* InlineDisposal: the service runs the item itself. +RunInline(item) == + IF item \in Blocking THEN svc' = "stuck" ELSE UNCHANGED svc + +Due(r) == slot[r] = "open" /\ r \notin cas /\ now >= dl[r] + +Expire(r) == + /\ svc = "idle" /\ Due(r) + /\ IF Fault = "WakeBeforeTransition" + THEN /\ cas' = cas \cup {r} + /\ batch' = IF waiting[r] THEN Append(batch, WakeItem(r)) ELSE batch + /\ UNCHANGED <> + ELSE /\ slot' = [slot EXCEPT ![r] = "timeout"] + /\ credit' = [credit EXCEPT ![r] = FALSE] + /\ expAt' = [expAt EXCEPT ![r] = now] + /\ UNCHANGED cas + /\ IF Fault = "InlineDisposal" + THEN /\ UNCHANGED batch + /\ IF waiting[r] /\ WakeItem(r) \notin Blocking + THEN woken' = [woken EXCEPT ![r] = TRUE] ELSE UNCHANGED woken + /\ IF waiting[r] THEN RunInline(WakeItem(r)) ELSE UNCHANGED svc + ELSE /\ batch' = IF waiting[r] THEN Append(batch, WakeItem(r)) ELSE batch + /\ UNCHANGED <> + /\ UNCHANGED <> + +\* WakeBeforeTransition only: the transition commits after the hand-off. +Commit(r) == + /\ r \in cas + /\ cas' = cas \ {r} + /\ slot' = [slot EXCEPT ![r] = "timeout"] + /\ credit' = [credit EXCEPT ![r] = FALSE] + /\ expAt' = [expAt EXCEPT ![r] = now] + /\ UNCHANGED <> + +Release(p) == + /\ svc = "idle" /\ payload[p] = "none" /\ phase # "exiting" + /\ IF Fault = "InlineDisposal" + THEN /\ payload' = [payload EXCEPT ![p] = IF DropItem(p) \in Blocking THEN "out" ELSE "dropped"] + /\ RunInline(DropItem(p)) + /\ UNCHANGED batch + ELSE /\ payload' = [payload EXCEPT ![p] = "out"] + /\ batch' = Append(batch, DropItem(p)) + /\ UNCHANGED svc + /\ UNCHANGED <> + +\* One queue entry per item; BatchSerial: the whole batch as one entry. +Entries(b) == IF Fault = "BatchSerial" THEN <> ELSE [i \in 1..Len(b) |-> <>] + +Submit == + /\ batch # <<>> + /\ queue' = queue \o Entries(batch) + /\ batch' = <<>> + /\ UNCHANGED <> + +------------------------------------------------------------------------------ +\* Disposal workers + +Idle(w) == w \in alive /\ cur[w] = <<>> +IdleCount == Cardinality({w \in W : Idle(w)}) + +Start(w) == + /\ w \notin alive /\ Len(queue) > IdleCount + /\ alive' = alive \cup {w} + /\ UNCHANGED <> + +Take(w) == + /\ Idle(w) /\ queue # <<>> + /\ cur' = [cur EXCEPT ![w] = Head(queue)] + /\ queue' = Tail(queue) + /\ UNCHANGED <> + +\* Run the worker's next item; a blocking one never returns. +Run(w) == + /\ cur[w] # <<>> + /\ LET item == Head(cur[w]) IN + /\ item \notin Blocking + /\ cur' = [cur EXCEPT ![w] = Tail(cur[w])] + /\ IF item.k = "wake" + THEN /\ woken' = [woken EXCEPT ![item.id] = TRUE] + /\ UNCHANGED <> + ELSE /\ IF phase = "exiting" /\ Fault # "DropWhileHeld" + THEN /\ payload' = [payload EXCEPT ![item.id] = "held"] + /\ UNCHANGED early + ELSE /\ payload' = [payload EXCEPT ![item.id] = "dropped"] + /\ early' = (early \/ phase = "exiting") + /\ UNCHANGED woken + /\ UNCHANGED <> + +Retire(w) == + /\ Idle(w) /\ queue = <<>> + /\ alive' = alive \ {w} + /\ UNCHANGED <> + +------------------------------------------------------------------------------ +\* The owner + +Exit == + /\ phase = "run" /\ phase' = "exiting" + /\ UNCHANGED <> + +\* The death point: what was held is dropped. +Death == + /\ phase = "exiting" /\ phase' = "dead" + /\ payload' = [p \in Pay |-> IF payload[p] = "held" THEN "dropped" ELSE payload[p]] + /\ UNCHANGED <> + +------------------------------------------------------------------------------ +\* The clock + +\* A framework step is pending (it never waits for user code). +Pending == + \/ svc = "idle" /\ \E r \in Req : Due(r) + \/ cas # {} + \/ batch # <<>> + \/ \E w \in W : cur[w] # <<>> /\ Head(cur[w]) \notin Blocking + \/ queue # <<>> /\ \E w \in W : Idle(w) \/ w \notin alive + +Tick == + /\ now < MaxTime /\ ~Pending + /\ now' = now + 1 + /\ UNCHANGED <> + +Next == + \/ Tick \/ Submit \/ Exit \/ Death + \/ \E r \in Req : Expire(r) \/ Commit(r) + \/ \E p \in Pay : Release(p) + \/ \E w \in W : Start(w) \/ Take(w) \/ Run(w) \/ Retire(w) + +Spec == Init /\ [][Next]_vars + +\* Fairness for the clock, the service and the workers (framework code); +\* none for the owner or for a blocking item (Run is not enabled for it). +FairSpec == Spec /\ WF_vars(Tick) /\ WF_vars(Submit) + /\ \A r \in Req : WF_vars(Expire(r)) /\ WF_vars(Commit(r)) + /\ \A w \in W : WF_vars(Start(w)) /\ WF_vars(Take(w)) /\ WF_vars(Run(w)) + +------------------------------------------------------------------------------ +\* Properties + +TypeOK == + /\ now \in 0..MaxTime + /\ slot \in [Req -> {"open", "timeout"}] + /\ credit \in [Req -> BOOLEAN] /\ woken \in [Req -> BOOLEAN] + /\ payload \in [Pay -> {"none", "out", "held", "dropped"}] + /\ cas \subseteq Req /\ alive \subseteq W + /\ phase \in {"run", "exiting", "dead"} /\ svc \in {"idle", "stuck"} + +\* A deferral credit is held exactly while the slot is open: it returns with +\* the transition, on the service thread, never with a disposal. +CreditWithTransition == \A r \in Req : credit[r] = (slot[r] = "open") +\* No wake-up runs before its slot's transition and credit return. +WakeAfterTransition == \A r \in Req : woken[r] => (slot[r] = "timeout" /\ ~credit[r]) +\* No payload handed off is dropped while the owner's hold is on: it waits +\* for the death point (R6, callers before payloads). +NoDropWhileHeld == ~early +\* The service does not wait for user code: no due slot stays open past its +\* deadline (the clock passes it only if the service is stuck). +TimelyExpiry == \A r \in Req : slot[r] = "open" => now <= dl[r] +\* A wake-up that does not itself block runs within the tick of its expiry, +\* whatever else blocks (another item of the same visit included). +TimelyWake == + \A r \in Req : (slot[r] = "timeout" /\ waiting[r] /\ WakeItem(r) \notin Blocking + /\ now > expAt[r]) => woken[r] + +Safety == + /\ TypeOK /\ CreditWithTransition /\ WakeAfterTransition /\ NoDropWhileHeld + /\ TimelyExpiry /\ TimelyWake + +\* Every handed-off wake-up that does not block eventually runs. +EveryWakeRuns == + \A r \in Req : (slot[r] = "timeout" /\ waiting[r] /\ WakeItem(r) \notin Blocking) + ~> woken[r] +\* Every deadline is eventually serviced. +EveryDueExpires == \A r \in Req : (now >= dl[r]) ~> (slot[r] = "timeout") +============================================================================= diff --git a/verification/tla/MCDisposal.cfg b/verification/tla/MCDisposal.cfg new file mode 100644 index 000000000..305d4b107 --- /dev/null +++ b/verification/tla/MCDisposal.cfg @@ -0,0 +1,13 @@ +\* Fast instance (CI, every PR): two requests and one payload of one owner, +\* three workers, clock to 2; the first caller's waker blocks forever. +SPECIFICATION FairSpec +INVARIANT Safety +PROPERTY EveryWakeRuns +PROPERTY EveryDueExpires +CONSTANTS + N = 2 + P = 1 + MaxTime = 2 + Workers = 3 + Blocking <- MCBlocking + Fault = "None" diff --git a/verification/tla/MCDisposal.tla b/verification/tla/MCDisposal.tla new file mode 100644 index 000000000..64ccdfae3 --- /dev/null +++ b/verification/tla/MCDisposal.tla @@ -0,0 +1,5 @@ +----------------------------- MODULE MCDisposal ----------------------------- +EXTENDS Disposal +\* The first caller's waker never returns. +MCBlocking == {WakeItem(1)} +============================================================================= diff --git a/verification/tla/MCDisposalDrop.cfg b/verification/tla/MCDisposalDrop.cfg new file mode 100644 index 000000000..8e190140f --- /dev/null +++ b/verification/tla/MCDisposalDrop.cfg @@ -0,0 +1,13 @@ +\* Fast instance (CI, every PR): a blocking payload destructor; two requests, +\* two payloads, three workers, clock to 2. +SPECIFICATION FairSpec +INVARIANT Safety +PROPERTY EveryWakeRuns +PROPERTY EveryDueExpires +CONSTANTS + N = 2 + P = 2 + MaxTime = 2 + Workers = 3 + Blocking <- MCBlocking + Fault = "None" diff --git a/verification/tla/MCDisposalDrop.tla b/verification/tla/MCDisposalDrop.tla new file mode 100644 index 000000000..a529d2232 --- /dev/null +++ b/verification/tla/MCDisposalDrop.tla @@ -0,0 +1,5 @@ +--------------------------- MODULE MCDisposalDrop --------------------------- +EXTENDS Disposal +\* The first payload's destructor never returns. +MCBlocking == {DropItem(1)} +============================================================================= diff --git a/verification/tla/TraceDisposal.tla b/verification/tla/TraceDisposal.tla new file mode 100644 index 000000000..273e3b9d0 --- /dev/null +++ b/verification/tla/TraceDisposal.tla @@ -0,0 +1,65 @@ +----------------------------- MODULE TraceDisposal ----------------------------- +(***************************************************************************) +(* Trace validation for Disposal.tla: the deadline service's hand-off of *) +(* wake-ups and payloads to the disposal workers, and the owner's hold. *) +(* *) +(* With the `lifecycle-trace` feature, while `record_disposal` is on, the *) +(* code records (concurrency/src/lifecycle_trace.rs, DisposalEvent; `id` *) +(* is fresh per Expire or Release and numbered by trace_check.py): *) +(* Expire a slot's OPEN -> TIMED_OUT and credit return on the service *) +(* thread, in a collecting visit (registry.rs, Expirable for *) +(* Slot); x: "wake" if a wake-up was handed off, else "none" *) +(* (Expire, with the caller's `waiting` matching x) *) +(* Release a payload deferred from the service thread, in one section *) +(* with the hold check (DropFailures::discard) (Release) *) +(* Submit the visit's work handed to the pool (deadline.rs) (Submit; *) +(* a visit whose items are all untracked, such as the last *) +(* reference to an entry, submits an empty batch here) *) +(* Wake a worker runs a handed-off wake-up (Run of a wake item) *) +(* Drop a worker settles a payload in one section with the hold *) +(* check (DropFailures::settle); x: "held" or "dropped" (Run of *) +(* a drop item) *) +(* Exit the owner's hold begins (DropFailures::hold) (Exit) *) +(* Death the hold is released at the death point (Death) *) +(* *) +(* Silent: the clock, and the workers' Start, Take and Retire. No item *) +(* blocks in a recorded run (Blocking = {}). *) +(***************************************************************************) +EXTENDS Disposal, TLC + +CONSTANT Trace + +VARIABLE i + +traceVars == <> + +TraceInit == Init /\ i = 0 + +\* Some worker runs `item` next. +WorkerRuns(item) == \E w \in W : cur[w] # <<>> /\ Head(cur[w]) = item /\ Run(w) + +Step(e) == + CASE e.a = "Expire" -> e.id \in Req /\ waiting[e.id] = (e.x = "wake") /\ Expire(e.id) + [] e.a = "Release" -> e.id \in Pay /\ Release(e.id) + [] e.a = "Submit" -> \/ Submit + \/ batch = <<>> /\ UNCHANGED vars + [] e.a = "Wake" -> e.id \in Req /\ WorkerRuns(WakeItem(e.id)) + [] e.a = "Drop" -> e.id \in Pay /\ WorkerRuns(DropItem(e.id)) /\ payload'[e.id] = e.x + [] e.a = "Exit" -> Exit + [] e.a = "Death" -> Death + [] OTHER -> FALSE + +Silent == \/ Tick + \/ \E w \in W : Start(w) \/ Take(w) \/ Retire(w) + +TraceNext == + /\ i < Len(Trace) + /\ \/ Step(Trace[i + 1]) /\ i' = i + 1 + \/ Silent /\ UNCHANGED i + +TraceSpec == TraceInit /\ [][TraceNext]_traceVars + +\* Violated exactly when the whole log has been matched: TLC reporting this +\* invariant as violated means the trace is ACCEPTED. +TraceIncomplete == i < Len(Trace) +============================================================================= diff --git a/verification/tla/coverage-baseline.json b/verification/tla/coverage-baseline.json index fea56ae28..19c0b89ab 100644 --- a/verification/tla/coverage-baseline.json +++ b/verification/tla/coverage-baseline.json @@ -149,5 +149,14 @@ "A|Join", "A|Ordered", "A|StopNext" + ], + "Disposal": [ + "A|Death", + "A|Drop", + "A|Exit", + "A|Expire", + "A|Release", + "A|Submit", + "A|Wake" ] } diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 6898292f0..b3f30b8cc 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -283,6 +283,10 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDeferredReplies", "NoReplyFence", "NoReplyAfterDeath"), ("MCDeferredReplies", "CallbackBeforeDischarge", "WokenAfterDischarge"), ("MCDeferredReplies", "InlineDisposal", "TimelyExpiry"), + ("MCDisposal", "InlineDisposal", "TimelyExpiry"), + ("MCDisposal", "WakeBeforeTransition", "WakeAfterTransition"), + ("MCDisposal", "DropWhileHeld", "NoDropWhileHeld"), + ("MCDisposal", "BatchSerial", "TimelyWake"), ("MCDeferredJobs", "NoInstanceCheck", "NoLateResult"), ("MCDeferredJobs", "NoCancelFence", "NoStaleAfterCancel"), ("MCDeferredJobs", "DoubleTerminal", "AtMostOneTerminal"), @@ -377,6 +381,31 @@ def start_controls(tlc): ] +def disposal_trace_controls(traces, tlc): + """Edits of recorded disposal traces that Disposal.tla must reject.""" + by_name = {t["name"]: t for t in traces if t.get("schema") == "disposal-v1"} + + def wake_before_expire(events): + wake = next(i for i, e in enumerate(events) if e["a"] == "Wake") + expire = next(i for i, e in enumerate(events) + if e["a"] == "Expire" and e["id"] == events[wake]["id"]) + events.insert(expire, events.pop(wake)) + require_rejected(by_name["disposal_expired_callers"], wake_before_expire, + "a wake-up run before its slot's transition", tlc) + + def drop_before_submit(events): + drop = next(i for i, e in enumerate(events) if e["a"] == "Drop") + submit = next(i for i, e in enumerate(events) if e["a"] == "Submit") + events.insert(submit, events.pop(drop)) + require_rejected(by_name["disposal_released_payloads"], drop_before_submit, + "a payload dropped by a worker before the hand-off", tlc) + + def drop_while_held(events): + next(e for e in events if e["a"] == "Drop" and e["x"] == "held")["x"] = "dropped" + require_rejected(by_name["disposal_held_until_death"], drop_while_held, + "a handed-off payload dropped while the owner's hold is on", tlc) + + def reply_trace_controls(traces, tlc): """Edits of recorded reply-slot traces that DeferredReplies.tla must reject.""" @@ -800,6 +829,7 @@ def decide_at_once(events): job_trace_controls(traces, args.tlc) coordinator_trace_controls(traces, args.tlc) reply_trace_controls(traces, args.tlc) + disposal_trace_controls(traces, args.tlc) lifecycle_trace_controls(traces, args.tlc) for module, fault, invariant in LIFECYCLE_CONTROLS: named_fault(module, fault, invariant, args.tlc) diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index 998de32a2..7169f4ded 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Check recorded lifecycle traces against Supervision.tla (and, by schema, Cohorts.tla, NestedSupervision.tla, DeferredJobs.tla, DeferredReplies.tla, -Lifecycle.tla and Coordinator.tla). +Disposal.tla, Lifecycle.tla and Coordinator.tla). Each JSON trace (written by concurrency/tests/supervision_trace.rs with the `lifecycle-trace` feature) looks like: @@ -50,7 +50,7 @@ def tla_event(event, cohort=False): def generate(trace, workdir): schema = trace.get("schema", "one-for-one-v1") if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1", "jobs-v1", "replies-v1", - "lifecycle-v1", "coordinator-v1"}: + "lifecycle-v1", "coordinator-v1", "disposal-v1"}: raise ValueError(f"unsupported trace schema: {schema}") if schema == "tree-v1": return generate_tree(trace, workdir) @@ -62,6 +62,8 @@ def generate(trace, workdir): return generate_lifecycle(trace, workdir) if schema == "coordinator-v1": return generate_coordinator(trace, workdir) + if schema == "disposal-v1": + return generate_disposal(trace, workdir) cohort = schema == "cohort-v1" modules = ("Cohorts.tla", "TraceCohorts.tla") if cohort else MODELS for model in modules: @@ -149,6 +151,50 @@ def generate_replies(trace, workdir): "INVARIANT Safety\nINVARIANT TraceIncomplete\n") +def generate_disposal(trace, workdir): + """The deadline service's hand-off and the disposal workers against + Disposal.tla. Expire ids number the requests 1..N, Release ids the + payloads 1..P, each in order of first appearance.""" + for model in ("Disposal.tla", "TraceDisposal.tla"): + shutil.copy(os.path.join(HERE, model), workdir) + reqs, pays, records = {}, {}, [] + for e in trace["events"]: + a, raw = e["a"], int(e["id"]) + if a == "Expire": + if raw in reqs: + raise ValueError(f"a slot expired twice: {e}") + reqs[raw] = len(reqs) + 1 + elif a == "Release": + if raw in pays: + raise ValueError(f"a payload released twice: {e}") + pays[raw] = len(pays) + 1 + if a in {"Expire", "Wake"}: + ident = reqs.get(raw, 0) + elif a in {"Release", "Drop"}: + ident = pays.get(raw, 0) + else: + ident = 0 + if a in {"Wake", "Drop"} and not ident: + raise ValueError(f"a worker step for an item never handed off: {e}") + records.append("[a |-> {}, id |-> {}, x |-> {}]".format( + tla_string(a), ident, tla_string(e["x"]))) + if not records: + raise ValueError("a disposal trace needs at least one event") + with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: + out.write("---- MODULE TraceRun ----\nEXTENDS TraceDisposal\n" + "TraceBlocking == {}\nTraceLog == <<\n " + + ",\n ".join(records) + "\n>>\n====\n") + with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: + # Deadlines are chosen by TLC; one tick makes every slot due. One + # worker per item, plus one, so no recorded run waits for a worker. + out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" + f" N = {len(reqs)}\n P = {len(pays)}\n MaxTime = 1\n" + f" Workers = {len(reqs) + len(pays) + 1}\n" + " Blocking <- TraceBlocking\n Fault = \"None\"\n" + " Trace <- TraceLog\n" + "INVARIANT Safety\nINVARIANT TraceIncomplete\n") + + OWN_ACTIONS = {"Trap", "Stop", "Link", "Unlink", "Monitor", "Demonitor"} diff --git a/verification/tla/trace_coverage.py b/verification/tla/trace_coverage.py index 8422ed079..bec611705 100644 --- a/verification/tla/trace_coverage.py +++ b/verification/tla/trace_coverage.py @@ -99,6 +99,7 @@ "DeferredJobs": ("TraceJobs.tla", {"jobs-v1"}), "DeferredReplies": ("TraceReplies.tla", {"replies-v1"}), "Coordinator": ("TraceCoordinator.tla", {"coordinator-v1"}), + "Disposal": ("TraceDisposal.tla", {"disposal-v1"}), } ALL = list(MODELS) + list(STATIC) From 26a8230c11a728d68b00cb1988c2b8656a31a1e0 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 13:31:16 -0300 Subject: [PATCH 08/13] verification: number disposal trace items over the whole log; docs for the Disposal model --- docs/TESTING.md | 4 +++- docs/design/deferred-replies.md | 6 +++++- verification/tla/README.md | 36 +++++++++++++++++++++++++++++---- verification/tla/trace_check.py | 8 +++++--- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/docs/TESTING.md b/docs/TESTING.md index 4adf4741d..689e5879b 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1882,12 +1882,14 @@ and prints the seed when a test fails. Configuration is process-global. | `link.before_install` | no | Peer exiting while a link is installed (tasks and threads) | | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | | `job.emit_due` | no | A scheduled `emit_at` coming due during the owner's exit: its payload is held until every caller is answered (`emit_due_during_exit_*`) | +| `disposal.item` | no | A disposal worker delayed before an item the deadline service handed off, so the owner's exit begins first: the payload is held until the death point (`disposal_trace`: `a_payload_handed_off_before_exit_waits_for_the_death_point`) | | `job.native_worker_unused` | no | Threads mode: the parked thread of a dropped, unused blocking-job permit held at its exit; the native slot is already back (`an_unused_permit_returns_its_slot_without_its_thread`) | | `reply.close_upgraded` | no | The death point's close holding an already-answered slot while its caller drops the receiver: the unread reply is destroyed only after every caller is answered (tasks and threads) | | `reply.recv_parked` | no | Handshake: a blocking `recv()` is parked, so only a notification wakes it (discharge notifies parked receivers before any waker; tasks and threads) | | `monitor.down_before_dispatch` | no | `demonitor` on another thread returning while the monitor's `Down` is in dispatch: it is flushed (tasks and threads) | -`tests/failpoints.rs` drives every site and checks invariants through public +`tests/failpoints.rs` drives every site (`disposal.item` is driven by +`tests/disposal_trace.rs`) and checks invariants through public APIs: restarts follow the budget (generation count equals injected failures plus started actors), a stop during a restart or publication window starts or publishes nothing, shutdown reports at its deadline and later completes, diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index cfd1695db..f3c6efced 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -307,7 +307,11 @@ handles a call even after the caller timed out, and the late reply is dropped. independently). Disposal workers form an elastic pool: when queued items outnumber idle workers one more starts (one start at a time, up to 64; idle ones exit after 10 s), and each worker re-checks when it takes an - item. Reaching the cap logs `disposal_workers_capped` once. + item. Reaching the cap logs `disposal_workers_capped` once. Modelled in + `verification/tla/Disposal.tla` (trace-validated against + `concurrency/tests/disposal_trace.rs`): no wake-up before the transition + and credit return, no payload drop while held, and no expiry or wake-up + waiting for a blocked item; its controls include the shared-batch bug. - *Why a pool and not the owner.* Deferring disposal to the owning actor would tie a caller's wake-up to the owner's next turn (a wedged owner would hang its callers past their deadline), and deferring it to the diff --git a/verification/tla/README.md b/verification/tla/README.md index 1796f9fbd..8844f3ded 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -17,6 +17,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `TraceCoordinator.tla` | Trace validation of `Coordinator.tla`: replays the shutdown coordinator events recorded from real `ActorSystem::shutdown` runs (`coordinator_trace`). | | `NestedSupervision.tla`, `MCNested.tla/.cfg` | Dynamic subtree ownership and shutdown delivery; parent scope completion waits for descendant completion. Bounded three-node/two-generation model with fair cleanup. | | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with exit-pending drops (unwound or dropped while exiting, answered at death with the final reason), a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Fourteen named negative controls. | +| `Disposal.tla`, `MCDisposal{,Drop}.tla/.cfg`, `TraceDisposal.tla` | The deadline service's hand-off of user code to disposal workers (companion of `DeferredReplies.tla`, which takes the service's expiry as one step): per-entry `Expire` (transition and credit return on the service thread), payload `Release`, `Submit` one queue entry per item, an elastic worker pool (start while queued entries outnumber idle workers, take, run, retire), wake-ups run by workers, payload drops that re-check the owner's hold (held until `Death`), and items that never return (`Blocking`). Invariants: credit with the transition, no wake-up before the transition, no payload drop while held, timely expiry and timely wake-up whatever blocks; liveness: every non-blocking wake-up runs, every deadline is serviced. Four named negative controls; trace-validated. See [Disposal](#disposal-trace-validation). | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits; `MCDeferredJobsHeld` the one with delayed emits (`emit_at`) and held results (completion, held notification, scheduled emits and credit ownership as separate steps). Twelve named negative controls; `MCDeferredJobsEmitRace` checks retained emit admission at held-result release (306 states). | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Nineteen negative-control faults (19 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point, a parent exit reaching `exit_received`, an exit signal past the sender's death point (`NoSignalFence`) and publication before the queued requests were answered (`PublishBeforeDischarge`). Death is two steps, as in `actor_cleanup.rs`: the death point (`Die`/`RuntimeDrop`: signal fence, seal, table closed) and `Discharge` (queued and exit-pending requests answered), with other actors' steps in between; `Publish` waits for `Discharge`. Per PR: `MCLifecycle` (10.0M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance); manual pre-release: `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | | `Start.tla`, `MCStart{,Names,Cap,Timeout,Continue,Scope,Live,Deep}.tla/.cfg` | Unified start API, 0.9 design (approved after three review rounds) ([design](../../docs/design/unified-start.md), not implemented; TLC-only in `model-map.toml` until trace validation lands with the implementation): `spawn`, `start`, `start_link`, `start_monitor`; placement inheritance; an atomic stage-1 commit with the owner gate; an immutable startup commitment and a waiter; death as D, release, then publication, for the child and the parent; the final reason fixed at D; fallible init with held-signal precedence; the continuation; S8 scope ownership with the polite Shutdown and the scope deadline. 40 negative controls (37 safety, 3 liveness); see [Unified start](#unified-start-09-design). Per PR: `MCStart`, `MCStartNames`, `MCStartCap`, `MCStartTimeout`, `MCStartContinue`, `MCStartScope`, `MCStartLive` (CI leg `start`); weekly: `MCStartDeep`. | @@ -174,12 +175,12 @@ coverage asks the converse: which parts of the model's reachable behavior do the recorded runs never exercise? `trace_coverage.py` reports it per model with trace validation (`Supervision`, `Cohorts`, `NestedSupervision`, and by action keys only `Lifecycle`, `DeferredJobs`, `DeferredReplies`, -`Coordinator`). +`Coordinator`, `Disposal`). **Action-only models.** For `Lifecycle.tla`, `DeferredJobs.tla`, -`DeferredReplies.tla` and `Coordinator.tla` the keys are the log-event arms of -their trace spec (`TraceLifecycle.tla`, `TraceJobs.tla`, `TraceReplies.tla`, -`TraceCoordinator.tla`; each arm is one +`DeferredReplies.tla`, `Coordinator.tla` and `Disposal.tla` the keys are the +log-event arms of their trace spec (`TraceLifecycle.tla`, `TraceJobs.tla`, +`TraceReplies.tla`, `TraceCoordinator.tla`, `TraceDisposal.tla`; each arm is one model action or a named group of them, `A|Send`, `A|Fire`, ...): the model side is the set of arms, a trace covers the arms its accepted replay takes, and no state space is explored for them. `coverage-baseline.json` and the @@ -710,6 +711,10 @@ exactly the property it targets: | `DeferredReplies` | `NoReplyFence` (a stored token's reply is admitted after the death point) | `NoReplyAfterDeath` | | `DeferredReplies` | `CallbackBeforeDischarge` (an answer's wake-up runs before the last answer of the death batch) | `WokenAfterDischarge` | | `DeferredReplies` | `InlineDisposal` (0.8.0: the shared deadline thread runs an expiry's waker or destructor inline, and one never returns) | `TimelyExpiry` | +| `Disposal` (`MCDisposal`) | `InlineDisposal` (0.8.0: the service runs the batch itself; a blocking item stops every later expiry) | `TimelyExpiry` | +| `Disposal` (`MCDisposal`) | `WakeBeforeTransition` (the wake-up handed off before the transition commits) | `WakeAfterTransition` | +| `Disposal` (`MCDisposal`) | `DropWhileHeld` (the worker drops a payload without re-checking the owner's hold) | `NoDropWhileHeld` | +| `Disposal` (`MCDisposal`) | `BatchSerial` (a visit's batch is one queue entry run serially; the shared-batch bug of PR #282's first revision) | `TimelyWake` | | `DeferredJobs` | `NoInstanceCheck` | `NoLateResult` | | `DeferredJobs` | `NoCancelFence` | `NoStaleAfterCancel` | | `DeferredJobs` | `DoubleTerminal` | `AtMostOneTerminal` | @@ -796,6 +801,29 @@ below the bound, a reply after death, a second discharge, a drop before the turn, and a `Stopped` discharge of the panicking turn's token before its `Exit`, the rank 11 behaviour before 5e). +### Disposal trace validation + +While `lifecycle_trace::record_disposal()` is on, the code records the +`Disposal.tla` actions at their linearization points: `Expire` (on the +service thread, after `Slot::expire_collect`'s transition and credit return; +`x` says whether a wake-up was handed off), `Release` (a payload deferred +from the service thread, in one section with `DropFailures`' hold check), +`Submit` (`deadline.rs`, the visit's work handed to the pool), `Wake` (a +worker runs a handed-off wake-up), `Drop` (a worker settles a payload, in +one section with the hold check; `x`: `held` or `dropped`), `Exit` +(`DropFailures::hold`) and `Death` (`DropFailures::release`). Ids are fresh +per `Expire`/`Release`; the checker numbers requests and payloads. The +workers' start, take and retire, and the clock, are silent; `Commit` (the +`WakeBeforeTransition` fault device) has no event. Recording is +process-wide, so `concurrency/tests/disposal_trace.rs` runs its scenarios +one at a time: expired callers woken by workers (two waiting, one never +polled), payloads released by a job deadline and a discarded scheduled emit +and dropped by workers, and a payload handed off before the owner's exit +whose worker is delayed (`disposal.item` failpoint) until `stopped()` runs: +it is held and dropped at the death point. `negative_checks.py` requires +three edited traces to be rejected (a wake-up before its expiry, a payload +drop before the hand-off, and a handed-off payload dropped while held). + ## Unified start (0.9 design) `Start.tla` backs the [unified start design](../../docs/design/unified-start.md) diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index 7169f4ded..5162cc228 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -154,7 +154,9 @@ def generate_replies(trace, workdir): def generate_disposal(trace, workdir): """The deadline service's hand-off and the disposal workers against Disposal.tla. Expire ids number the requests 1..N, Release ids the - payloads 1..P, each in order of first appearance.""" + payloads 1..P, each in order of appearance. Numbering is over the whole + log, so a worker step logged before its hand-off keeps its request or + payload and the model rejects it; one never handed off gets 0 (no item).""" for model in ("Disposal.tla", "TraceDisposal.tla"): shutil.copy(os.path.join(HERE, model), workdir) reqs, pays, records = {}, {}, [] @@ -168,14 +170,14 @@ def generate_disposal(trace, workdir): if raw in pays: raise ValueError(f"a payload released twice: {e}") pays[raw] = len(pays) + 1 + for e in trace["events"]: + a, raw = e["a"], int(e["id"]) if a in {"Expire", "Wake"}: ident = reqs.get(raw, 0) elif a in {"Release", "Drop"}: ident = pays.get(raw, 0) else: ident = 0 - if a in {"Wake", "Drop"} and not ident: - raise ValueError(f"a worker step for an item never handed off: {e}") records.append("[a |-> {}, id |-> {}, x |-> {}]".format( tla_string(a), ident, tla_string(e["x"]))) if not records: From 65a1e1180748b2b4c3e8ebb54984ff2cb19dd925 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 13:42:14 -0300 Subject: [PATCH 09/13] verification: map job runs' hand-off to Disposal; changelog --- CHANGELOG.md | 8 ++++++-- verification/model-map.toml | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed9f3286b..50fb8ae9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,8 +88,12 @@ deadlines or other callers of the same actor. A payload handed off still waits for its owner's death point (R6). No cost on requests that do not expire. New regression tests (tasks and threads), - a Loom case for the handed-off wake-up, and an `InlineDisposal` negative - control in `DeferredReplies.tla`. + a Loom case for the handed-off wake-up, an `InlineDisposal` negative + control in `DeferredReplies.tla`, and a new trace-validated model, + `Disposal.tla` (expiry and hand-off, per-item queueing, worker wake-ups + after the transition, payload drops held until the death point), with the + `disposal_trace` scenarios, a `disposal.item` failpoint and four negative + controls including the shared-batch bug. - Regenerate the 11 planted-bug patches (04, 10-17, 19, 20) that no longer applied, each planting the same fault where the code now lives, and check in the required `model-sync` job that every planted patch still applies. diff --git a/verification/model-map.toml b/verification/model-map.toml index 134bf9012..ac54b30f9 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -152,6 +152,8 @@ code = [ "concurrency/src/reply/deadline.rs", "concurrency/src/reply/registry.rs", "concurrency/src/mailbox.rs", + "concurrency/src/tasks/jobs.rs", # job runs' last references (`Run`) + "concurrency/src/threads/jobs.rs", "concurrency/src/lifecycle_trace.rs", ] coverage_floor = 7 # of 7 action keys (TraceDisposal arms) From ff3999d3d3c676aac2ed62c673386b20f8f24c68 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 19:24:23 -0300 Subject: [PATCH 10/13] docs(tla): list disposal_trace in the trace recording command --- verification/tla/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verification/tla/README.md b/verification/tla/README.md index 8844f3ded..0704431ca 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -189,7 +189,7 @@ Model actions without an arm are listed in the map's `[trace.unlogged]` with the reason. ```bash -cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace +cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test disposal_trace --test lifecycle_trace nix develop -c python3 verification/tla/trace_coverage.py --traces dedicated=target/lifecycle-traces \ --model-keys verification/tla/coverage-model.json.gz --baseline verification/tla/coverage-baseline.json # every test binary: each finished supervisor dumps its traces From 65e394a18bca171e13fd3503ac02345b7839fb42 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 20:45:22 -0300 Subject: [PATCH 11/13] fix(disposal): skip empty hand-offs; record the pool's steps; model the start rule and two-step drops --- concurrency/src/failpoints.rs | 14 ++ concurrency/src/lifecycle_trace.rs | 74 +++++-- concurrency/src/mailbox.rs | 35 ++- concurrency/src/reply/deadline.rs | 11 +- concurrency/src/reply/disposal.rs | 192 +++++++++++++--- concurrency/src/reply/jobs.rs | 25 ++- concurrency/src/reply/registry.rs | 42 ++-- concurrency/src/tasks/jobs.rs | 8 +- concurrency/src/threads/jobs.rs | 8 +- concurrency/tests/disposal_trace.rs | 220 +++++++++++------- docs/TESTING.md | 4 +- verification/model-map.toml | 2 +- verification/tla/Disposal.tla | 283 ++++++++++++++++-------- verification/tla/MCDisposal.cfg | 2 + verification/tla/MCDisposalCapped.cfg | 14 ++ verification/tla/MCDisposalCapped.tla | 5 + verification/tla/MCDisposalDrop.cfg | 8 +- verification/tla/TraceDisposal.tla | 76 ++++--- verification/tla/coverage-baseline.json | 12 +- verification/tla/negative_checks.py | 48 ++-- verification/tla/trace_check.py | 93 +++++--- 21 files changed, 832 insertions(+), 344 deletions(-) create mode 100644 verification/tla/MCDisposalCapped.cfg create mode 100644 verification/tla/MCDisposalCapped.tla diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index 814d5fd69..16ebc8b0e 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -87,6 +87,20 @@ pub const SITES: &[Site] = &[ exit begin first: a payload handed off before it is held until the \ death point, not dropped by the worker.", }, + Site { + name: "disposal.batch_serial", + has_error: true, + description: "Error: the deadline service's handed-off batch runs as one entry, \ + serially on one worker (the shared-batch bug fixed in PR #282). A \ + negative control: the disposal trace it records must be rejected by \ + Disposal.tla.", + }, + Site { + name: "disposal.keep_alive", + has_error: true, + description: "Error: an idle disposal worker waits 5 ms instead of the keep-alive \ + before it retires, so a test sees workers retire and new ones start.", + }, Site { name: "job.native_worker_unused", has_error: false, diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index 44aae9063..f0fd3d46e 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -181,18 +181,22 @@ impl ReplyEvent { } /// One action of `verification/tla/Disposal.tla`: the deadline service's -/// expiry and hand-off, the disposal workers' steps, and the owner's payload -/// hold. `Expire` (a slot's transition on the service thread; `x`: `wake` if -/// it handed off a wake-up, else `none`), `Release` (a payload deferred from -/// the service thread), `Submit` (a visit's work handed to the pool), `Wake` -/// (a worker runs a handed-off wake-up), `Drop` (a worker settles a payload; -/// `x`: `held` or `dropped`), `Exit` (the owner's hold begins) and `Death` -/// (its hold is released at the death point). `id` numbers the slot or -/// payload (fresh per Expire/Release; normalized by the checker). +/// expiry and hand-off, the disposal pool's steps, and the owner's payload +/// hold. On the service thread: `Expire` (a slot's transition; `r` the +/// request, `id` its wake-up item, `x`: `wake` or `none`), `Defer` (an item +/// handed off; `x`: `drop` or `other`) and `Submit` (the visit's items +/// queued; `x`: `spawn` if a worker start was decided). In the pool (`w` the +/// worker): `Begin`, `Take` (`id`, `x` as for `Submit`), `Run` and `Done` +/// (an item starts and returns), `Settle` (a payload's hold check; `x`: +/// `held` or `dropped`), `Wait`, `Woke` and `Retire`. The owner: `Exit` +/// (its hold begins) and `Death` (the hold is released at the death point). +/// Numbers are fresh and normalized by the checker. #[derive(Debug, Clone, PartialEq, Eq)] pub struct DisposalEvent { pub action: &'static str, + pub w: u64, pub id: u64, + pub r: u64, pub x: &'static str, } @@ -200,8 +204,8 @@ impl DisposalEvent { /// JSON object understood by `verification/tla/trace_check.py`. pub fn to_json(&self) -> String { format!( - r#"{{"a":"{}","id":{},"x":"{}"}}"#, - self.action, self.id, self.x + r#"{{"a":"{}","w":{},"id":{},"r":{},"x":"{}"}}"#, + self.action, self.w, self.id, self.r, self.x ) } } @@ -520,10 +524,24 @@ mod imp { /// Start recording disposal events until [`take_disposal`]. Tests that /// record must not run concurrently with each other or with other /// deadline expiries or actor exits in the process. + /// + /// The first event, `Pool`, is the pool's state when recording starts + /// (`w`: idle workers, `r`: workers, `x`: `starting` if a start is in + /// flight); the checker starts the model from it. pub fn record_disposal() { - atomic(|| { - lock(&DISPOSAL).clear(); - DISPOSAL_ON.store(true, Ordering::Release); + crate::reply::disposal::with_pool_state(|idle, workers, starting| { + atomic(|| { + let mut events = lock(&DISPOSAL); + events.clear(); + events.push(super::DisposalEvent { + action: "Pool", + w: idle as u64, + id: 0, + r: workers as u64, + x: if starting { "starting" } else { "" }, + }); + DISPOSAL_ON.store(true, Ordering::Release); + }); }); } @@ -560,16 +578,30 @@ mod imp { } /// Record a disposal action (inside the section of the step it names). - /// An `id` of 0 (taken while recording was off) is not recorded. - pub(crate) fn disposal(action: &'static str, id: u64, x: &'static str) { + /// An item or request numbered 0 (taken while recording was off), or a + /// worker step off a worker, is not recorded. + pub(crate) fn disposal(action: &'static str, w: u64, id: u64, r: u64, x: &'static str) { if !disposal_on() { return; } - atomic(|| { - if id != 0 || matches!(action, "Submit" | "Exit" | "Death") { - lock(&DISPOSAL).push(super::DisposalEvent { action, id, x }); - } - }); + let recorded = match action { + "Expire" => r != 0, + "Defer" => id != 0, + "Take" | "Run" | "Done" | "Settle" => id != 0 && w != 0, + "Begin" | "Wait" | "Woke" | "Retire" => w != 0, + _ => true, + }; + if recorded { + atomic(|| { + lock(&DISPOSAL).push(super::DisposalEvent { + action, + w, + id, + r, + x, + }); + }); + } } /// Whether actor `id` is recorded. @@ -1170,7 +1202,7 @@ mod imp { } #[allow(dead_code)] #[inline(always)] - pub(crate) fn disposal(_: &'static str, _: u64, _: &'static str) {} + pub(crate) fn disposal(_: &'static str, _: u64, _: u64, _: u64, _: &'static str) {} #[inline(always)] pub(crate) fn disposal_section(f: impl FnOnce() -> R) -> R { f() diff --git a/concurrency/src/mailbox.rs b/concurrency/src/mailbox.rs index d8c560619..6d473e27d 100644 --- a/concurrency/src/mailbox.rs +++ b/concurrency/src/mailbox.rs @@ -36,7 +36,7 @@ impl DropFailures { let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); if held.is_none() { *held = Some(Vec::new()); - crate::lifecycle_trace::disposal("Exit", 0, ""); + crate::lifecycle_trace::disposal("Exit", 0, 0, 0, ""); } }); } @@ -47,7 +47,7 @@ impl DropFailures { let held = crate::lifecycle_trace::disposal_section(|| { let held = self.held.lock().unwrap_or_else(|p| p.into_inner()).take(); if held.is_some() { - crate::lifecycle_trace::disposal("Death", 0, ""); + crate::lifecycle_trace::disposal("Death", 0, 0, 0, ""); } held }); @@ -62,7 +62,8 @@ impl DropFailures { pub(crate) fn discard(self: &Arc, value: T) { let mut value = Some(value); // One step of Disposal.tla with `Exit` and `Death`: the hold check - // and, on the service thread, `Release` (the hand-off). + // and, on the service thread, `Release` (the hand-off, recorded as + // `Defer` with `x: drop`). let inline = crate::lifecycle_trace::disposal_section(|| { let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); if let Some(held) = held.as_mut() { @@ -72,11 +73,9 @@ impl DropFailures { } drop(held); if crate::reply::disposal::collecting() { - let id = crate::lifecycle_trace::disposal_id(); - crate::lifecycle_trace::disposal("Release", id, ""); let failures = self.clone(); let value = value.take(); - crate::reply::disposal::defer(move || failures.settle(value, id)); + crate::reply::disposal::defer_as("drop", move || failures.settle(value)); return false; } true @@ -85,18 +84,30 @@ impl DropFailures { self.drop_contained(value); } } - /// A disposal worker's step for a payload the service handed off: held - /// if the owner's exit began meanwhile (dropped at its death point), - /// else dropped now (Disposal.tla `Drop`). - fn settle(&self, value: T, _id: u64) { + /// [`discard`](Self::discard) of an optional payload: nothing to dispose + /// of (and no hand-off) when it is `None`. + #[inline] + pub(crate) fn discard_some(self: &Arc, value: Option) { + if let Some(value) = value { + self.discard(value); + } + } + /// A disposal worker's step for a payload the service handed off + /// (Disposal.tla `Settle`, then the destructor until `Done`): held if + /// the owner's exit began meanwhile (dropped at its death point), else + /// dropped now. A drop decided before the exit is pre-exit work and may + /// still run while the exit proceeds; the exit never waits for it + /// (deferred-replies.md §5). + fn settle(&self, value: T) { let mut value = Some(value); + let (w, item) = crate::reply::disposal::current(); crate::lifecycle_trace::disposal_section(|| { let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); if let Some(held) = held.as_mut() { held.push(Box::new(value.take())); - crate::lifecycle_trace::disposal("Drop", _id, "held"); + crate::lifecycle_trace::disposal("Settle", w, item, 0, "held"); } else { - crate::lifecycle_trace::disposal("Drop", _id, "dropped"); + crate::lifecycle_trace::disposal("Settle", w, item, 0, "dropped"); } }); self.drop_contained(value); diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 681c0a226..0b83edd6b 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -29,6 +29,12 @@ pub(crate) trait Expirable: Send + Sync { fn pending_deadline(&self) -> Option; /// The deadline passed (`now >= deadline`): expire the item. fn expire(&self, now: u64); + /// Drop the service's reference after the visit. It may be the last: + /// an item whose last drop can run user code hands that drop off + /// ([`super::disposal`]); others drop in place, with no hand-off. + fn dispose(self: Arc) { + drop(self); + } } struct Entry { @@ -320,10 +326,7 @@ mod thread_service { } }); }); - // Disposal.tla `Submit`: the visit's work goes to the pool. - if !released.is_empty() { - crate::lifecycle_trace::disposal("Submit", 0, ""); - } + // Disposal.tla `Submit` (recorded there). super::super::disposal::submit(released); state = lock(service); } diff --git a/concurrency/src/reply/disposal.rs b/concurrency/src/reply/disposal.rs index ca9fbce33..90c6106ff 100644 --- a/concurrency/src/reply/disposal.rs +++ b/concurrency/src/reply/disposal.rs @@ -1,21 +1,33 @@ //! Disposal off the deadline service thread -//! (docs/design/deferred-replies.md §2, "Callback progress"). +//! (docs/design/deferred-replies.md §2, "Callback progress"; modelled in +//! verification/tla/Disposal.tla, whose actions the comments name). //! //! The process-wide deadline thread decides expiry: it pops due entries, //! makes their terminal transitions and returns their deferral credits //! (atomics and framework locks only). What an expiry releases that can run //! user code (a caller's waker, a payload destructor routed to the owner's -//! failure sink, the last reference to an entry) is collected while the -//! thread services a visit ([`collect`]) and handed to the disposal workers -//! ([`submit`]) one item at a time. A worker that blocks in user code holds -//! only the item it runs: a queued item that finds no idle worker starts -//! another one, so a blocking destructor or waker delays no other expiry and -//! no other wake-up, of another actor or of the same one. +//! failure sink, the last reference to a job or registry, a slot holding an +//! unread reply) is collected while the thread services a visit +//! ([`collect`]) and handed to the disposal workers ([`submit`]) one item at +//! a time. A worker that blocks in user code holds only the item it runs: a +//! queued item that finds no idle worker starts another one, so, while fewer +//! than [`MAX_WORKERS`] callbacks are blocked at once, a blocking destructor +//! or waker delays no other expiry and no other wake-up, of another actor or +//! of the same one. Past the cap, later items queue (expiry does not). +//! +//! The service thread still runs, inline: the owner's own executor wake +//! (enqueueing a job's message), dropping (not waking) a job's stored +//! wakers, the `tracing` subscriber for a contained panic or the cap +//! warning, a panic payload's destructor (`dispose_panic`), and, only if no +//! disposal thread exists and none can be spawned, the work itself. //! //! Outside a [`collect`] scope, [`defer`] and [`defer_drop`] run inline, so //! every other thread (owner turns, jobs, the simulated service task) keeps //! its behaviour. The check is one thread-local read, made only on paths //! that already have something to wake or drop. +//! +//! A burst of quick items can start workers that are then idle: at most one +//! start is in flight, and idle workers exit after [`KEEP_ALIVE`]. use std::cell::RefCell; use std::collections::VecDeque; @@ -24,9 +36,55 @@ use std::time::Duration; pub(crate) type Work = Box; +/// One handed-off item. With `lifecycle-trace`, its number in disposal +/// traces (0: not recorded). +pub(crate) struct Item { + work: Work, + #[cfg(feature = "lifecycle-trace")] + id: u64, +} + +impl Item { + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + fn new(work: Work, id: u64) -> Self { + Self { + work, + #[cfg(feature = "lifecycle-trace")] + id, + } + } + fn id(&self) -> u64 { + #[cfg(feature = "lifecycle-trace")] + return self.id; + #[cfg(not(feature = "lifecycle-trace"))] + 0 + } +} + +impl From for Item { + fn from(work: Work) -> Self { + Self::new(work, 0) + } +} + thread_local! { /// `Some` while the deadline thread services a visit. - static BATCH: RefCell>> = const { RefCell::new(None) }; + static BATCH: RefCell>> = const { RefCell::new(None) }; +} + +#[cfg(feature = "lifecycle-trace")] +thread_local! { + /// This disposal worker's number in traces (0: not a worker), and the + /// item it runs. + static WORKER: std::cell::Cell<(u64, u64)> = const { std::cell::Cell::new((0, 0)) }; +} + +/// The current worker and item numbers (0, 0 outside a worker's item). +pub(crate) fn current() -> (u64, u64) { + #[cfg(feature = "lifecycle-trace")] + return WORKER.try_with(|w| w.get()).unwrap_or((0, 0)); + #[cfg(not(feature = "lifecycle-trace"))] + (0, 0) } /// Run `work` now, or, on the deadline thread inside [`collect`], defer it @@ -34,17 +92,33 @@ thread_local! { /// thread-local destructor dropping a payload at thread exit) or borrowed. #[inline] pub(crate) fn defer(work: impl FnOnce() + Send + 'static) { + defer_as("other", work); +} + +/// [`defer`] an item of `kind` (`wake`, `drop` or `other`). Returns its +/// number in disposal traces (0 if it ran inline or nothing is recorded). +/// Records `Defer` (Disposal.tla `Release`/`Other`) except for a wake-up, +/// whose hand-off is part of `Expire`. +#[inline] +pub(crate) fn defer_as(kind: &'static str, work: impl FnOnce() + Send + 'static) -> u64 { let mut work = Some(work); + let mut id = 0; let _ = BATCH.try_with(|batch| { if let Ok(mut batch) = batch.try_borrow_mut() { if let Some(batch) = batch.as_mut() { - batch.push(Box::new(work.take().expect("not yet taken"))); + id = crate::lifecycle_trace::disposal_id(); + if kind != "wake" { + crate::lifecycle_trace::disposal("Defer", 0, id, 0, kind); + } + let work: Work = Box::new(work.take().expect("not yet taken")); + batch.push(Item::new(work, id)); } } }); if let Some(work) = work { work(); } + id } /// Drop `value` now, or defer its drop (see [`defer`]). @@ -73,10 +147,10 @@ pub(crate) fn collecting() -> bool { /// Run `step` with deferral on, and return what it deferred. If `step` /// unwinds, what it deferred so far is submitted, not dropped here. -pub(crate) fn collect(step: impl FnOnce()) -> Vec { +pub(crate) fn collect(step: impl FnOnce()) -> Vec { struct Scope; impl Scope { - fn take() -> Vec { + fn take() -> Vec { BATCH .try_with(|batch| batch.borrow_mut().take().unwrap_or_default()) .unwrap_or_default() @@ -102,7 +176,7 @@ pub(crate) fn collect(step: impl FnOnce()) -> Vec { /// Workers beyond this many wait for a busy one: a bound on threads, reached /// only when that many callbacks are blocked at once. -const MAX_WORKERS: usize = 64; +pub(crate) const MAX_WORKERS: usize = 64; /// An idle worker exits after this long without work. #[cfg(not(test))] const KEEP_ALIVE: Duration = Duration::from_secs(10); @@ -112,7 +186,7 @@ const KEEP_ALIVE: Duration = Duration::from_millis(2); #[derive(Default)] struct State { - queue: VecDeque, + queue: VecDeque, /// Workers waiting for work (or woken and not yet back under the lock). idle: usize, workers: usize, @@ -124,8 +198,8 @@ struct State { } impl State { - /// Queued work needs another worker: more items than idle workers that - /// could take them, and none starting. Returns whether to start one. + /// `NeedsWorker`: queued work needs another worker (more items than idle + /// workers that could take them, none starting, below the cap). fn needs_worker(&mut self) -> Start { if self.queue.len() <= self.idle || self.starting { return Start::No; @@ -149,6 +223,17 @@ enum Start { Capped, } +impl Start { + /// The decision as a disposal trace's `x`. + #[allow(dead_code)] // recorded only with `lifecycle-trace` + fn x(&self) -> &'static str { + match self { + Start::Yes => "spawn", + _ => "", + } + } +} + #[derive(Default)] struct Pool { state: Mutex, @@ -164,19 +249,38 @@ fn lock(pool: &Pool) -> std::sync::MutexGuard<'_, State> { pool.state.lock().unwrap_or_else(|p| p.into_inner()) } -/// Hand deferred work to the disposal workers, one queue entry per item, so -/// a blocking item delays none of the others. When queued items outnumber -/// idle workers a new worker starts (one at a time, up to [`MAX_WORKERS`]); -/// each worker re-checks when it takes an item. -pub(crate) fn submit(work: Vec) { +/// Run `f` with the pool's (idle, workers, starting) under its lock: a +/// recording starts from a consistent pool state (lock order: pool, then +/// the trace section, as everywhere). +#[cfg(feature = "lifecycle-trace")] +pub(crate) fn with_pool_state(f: impl FnOnce(usize, usize, bool) -> R) -> R { + let state = lock(pool()); + f(state.idle, state.workers, state.starting) +} + +/// `Submit`: hand deferred work to the disposal workers, one queue entry per +/// item, so a blocking item delays none of the others. When queued items +/// outnumber idle workers a new worker starts (one at a time, up to +/// [`MAX_WORKERS`]); each worker re-checks when it takes an item. +pub(crate) fn submit(work: Vec) { if work.is_empty() { return; } + #[allow(unused_mut)] + let mut work = work; + // Test-only: run the batch serially on one worker (the shared-batch + // bug), so trace validation can be shown to reject it. + fail_point!("disposal.batch_serial" => { + let items = std::mem::take(&mut work); + let serial: Work = Box::new(move || items.into_iter().for_each(run_item)); + work = vec![Item::from(serial)]; + }); let pool = pool(); let mut state = lock(pool); let wake = work.len().min(state.idle); state.queue.extend(work); let start = state.needs_worker(); + crate::lifecycle_trace::disposal("Submit", 0, 0, 0, start.x()); drop(state); for _ in 0..wake { pool.ready.notify_one(); @@ -219,9 +323,21 @@ fn start_worker(pool: &Arc, start: Start) { } } +/// A worker's loop (Disposal.tla `Begin`, `Take`, `Wait`, `Woke`, `Retire`; +/// each recorded under the pool lock, with the start decision on `Take`). fn run(pool: &Arc) { + #[cfg(feature = "lifecycle-trace")] + let w = { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + let w = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let _ = WORKER.try_with(|c| c.set((w, 0))); + w + }; + #[cfg(not(feature = "lifecycle-trace"))] + let w = 0; let mut state = lock(pool); state.starting = false; + crate::lifecycle_trace::disposal("Begin", w, 0, 0, ""); loop { if let Some(item) = state.queue.pop_front() { // Whatever is still queued must not wait for this item. @@ -229,6 +345,7 @@ fn run(pool: &Arc) { pool.ready.notify_one(); } let start = state.needs_worker(); + crate::lifecycle_trace::disposal("Take", w, item.id(), 0, start.x()); drop(state); start_worker(pool, start); run_item(item); @@ -236,24 +353,44 @@ fn run(pool: &Arc) { continue; } state.idle += 1; + crate::lifecycle_trace::disposal("Wait", w, 0, 0, ""); + let keep_alive = keep_alive(); let (next, timeout) = pool .ready - .wait_timeout(state, KEEP_ALIVE) + .wait_timeout(state, keep_alive) .unwrap_or_else(|p| p.into_inner()); state = next; state.idle -= 1; + crate::lifecycle_trace::disposal("Woke", w, 0, 0, ""); if timeout.timed_out() && state.queue.is_empty() { state.workers -= 1; state.capped = false; + crate::lifecycle_trace::disposal("Retire", w, 0, 0, ""); return; } } } -/// Contained, so a failing callback ends neither its worker nor the others. -fn run_item(item: Work) { +/// [`KEEP_ALIVE`]; the `disposal.keep_alive` failpoint (Error) shortens it so +/// a test sees workers retire. +fn keep_alive() -> Duration { + fail_point!("disposal.keep_alive" => return Duration::from_millis(5)); + KEEP_ALIVE +} + +/// `Run` then `Done`: contained, so a failing callback ends neither its +/// worker nor the others. +fn run_item(item: Item) { fail_point!("disposal.item"); - super::registry::contain_entry(item); + let (w, _) = current(); + let id = item.id(); + #[cfg(feature = "lifecycle-trace")] + let _ = WORKER.try_with(|c| c.set((w, id))); + crate::lifecycle_trace::disposal("Run", w, id, 0, ""); + super::registry::contain_entry(item.work); + crate::lifecycle_trace::disposal("Done", w, id, 0, ""); + #[cfg(feature = "lifecycle-trace")] + let _ = WORKER.try_with(|c| c.set((w, 0))); } #[cfg(test)] @@ -261,6 +398,11 @@ mod tests { use super::*; use std::sync::mpsc; + /// Submit plain work (untraced items). + fn submit(work: Vec) { + super::submit(work.into_iter().map(Item::from).collect()); + } + /// Generous: sanitizer builds are slow, and a panic's first report there /// can take seconds. const WAIT: Duration = Duration::from_secs(60); @@ -285,7 +427,7 @@ mod tests { assert!(!collecting()); assert!(rx.try_recv().is_err(), "deferred, not run"); assert_eq!(work.len(), 2); - submit(work); + super::submit(work); assert_eq!(rx.recv_timeout(WAIT), Ok(1)); } diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index aa382cb23..5a3e67cc8 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -257,6 +257,22 @@ pub(crate) struct JobState { /// Payloads an owner's exit took from a job, destroyed after its death point. type ExitReleased = (Option, Vec>); +impl Drop for JobState { + /// The last reference can go on the deadline service (a job deadline or + /// emit entry): what exit released and nobody disposed of yet is user + /// payload, so it goes to a disposal worker there (inline elsewhere). + fn drop(&mut self) { + let released = self + .exit_released + .get_mut() + .unwrap_or_else(|p| p.into_inner()) + .take(); + if let Some(released) = released { + super::disposal::defer_drop(released); + } + } +} + fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().unwrap_or_else(|p| p.into_inner()) } @@ -1056,7 +1072,7 @@ impl Drop for ScheduledEmit { .get_mut() .unwrap_or_else(|p| p.into_inner()) .take(); - self.failures.discard(send); + self.failures.discard_some(send); } } @@ -1092,8 +1108,11 @@ impl ScheduledEmit { let delivered = result.is_ok(); // Whatever was not enqueued (the refused envelope, the uncalled send // or the discarded one, each with its credit guard) drops here, - // unlocked and contained. - self.failures.discard((result, unused, discarded)); + // unlocked and contained. A delivered emit leaves nothing to drop: + // no hand-off then. + if result.is_err() || unused.is_some() || discarded.is_some() { + self.failures.discard((result, unused, discarded)); + } lock(&job.scheduled).retain(|emit| !std::ptr::eq(Arc::as_ptr(emit), self)); if let Some(held) = held { held(); diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 9f811a8e8..73931faa5 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -54,7 +54,18 @@ pub(crate) trait Closeable: Send + Sync { /// with the wake-up its close collected, if it closed it. type PendingWake = (Arc, Option); -impl Expirable for Slot { +impl Expirable for Slot { + /// Only a `REPLIED` slot can hold a value (an unread reply, user code + /// when destroyed), so only its drop is handed off, and that of a slot + /// not yet terminal (a reply still `WRITING` may store one). Any other + /// terminal state is final and holds no value: dropped in place. + fn dispose(self: Arc) { + if !self.is_terminal() + || Terminal::of(super::slot::state(self.word())) == Some(Terminal::Replied) + { + super::disposal::defer_drop(self); + } + } fn pending_deadline(&self) -> Option { let deadline = self.deadline(); (!self.is_terminal() && deadline != NEVER).then_some(deadline) @@ -65,22 +76,23 @@ impl Expirable for Slot { fn expire(&self, now: u64) { if let Some(mut wake) = self.expire_collect(now) { wake.notify_parked(); - // Disposal.tla `Expire` (after the transition and its credit - // return) and, on a worker, `Wake`. Compiled out without - // `lifecycle-trace`; recorded only on the collecting service. - let id = if cfg!(feature = "lifecycle-trace") && super::disposal::collecting() { + // Disposal.tla `Expire`: after the transition and its credit + // return, the wake-up (if any) is handed off. Compiled out + // without `lifecycle-trace`; recorded only on the service. + let request = if cfg!(feature = "lifecycle-trace") && super::disposal::collecting() { crate::lifecycle_trace::disposal_id() } else { 0 }; - let kind = if wake.is_empty() { "none" } else { "wake" }; - crate::lifecycle_trace::disposal("Expire", id, kind); - if !wake.is_empty() { - super::disposal::defer(move || { - crate::lifecycle_trace::disposal("Wake", id, ""); - wake.run() - }); - } + let (item, kind) = if wake.is_empty() { + (0, "none") + } else { + ( + super::disposal::defer_as("wake", move || wake.run()), + "wake", + ) + }; + crate::lifecycle_trace::disposal("Expire", 0, item, request, kind); } } } @@ -327,8 +339,8 @@ impl Registry { if item.pending_deadline().is_some_and(|d| d <= now) { item.expire(now); } - // Possibly the last reference. - super::disposal::defer_drop(item); + // Possibly the last reference (see `Expirable::dispose`). + item.dispose(); }); } lock(&self.lists).deadlines.earliest() diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index a670d5f14..e7962e1d6 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -151,7 +151,7 @@ where let (_, unsent) = self.state.notify(end, || self.enqueue_terminal(outcome)); // A refused terminal message (`done` and the outcome) may drop on // the deadline service thread: contained. - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); } fn enqueue_terminal( @@ -202,7 +202,7 @@ where }) }, ); - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); } } @@ -215,7 +215,7 @@ impl Drop for Run { .get_mut() .unwrap_or_else(|p| p.into_inner()) .take(); - self.ctx.shared.sender.failures().discard(done); + self.ctx.shared.sender.failures().discard_some(done); } } @@ -440,7 +440,7 @@ impl JobCtx { self.ctx.shared.sender.send(envelope) }) .map_err(|(gone, unsent)| { - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); gone }) } diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 4115d74ef..ca7e5da37 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -130,7 +130,7 @@ where let (_, unsent) = self.state.notify(end, || self.enqueue_terminal(outcome)); // A refused terminal message (`done` and the outcome) may drop on // the deadline service thread: contained. - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); } fn enqueue_terminal(&self, outcome: Contained>) -> Option> { @@ -184,7 +184,7 @@ where }) }, ); - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); } } @@ -197,7 +197,7 @@ impl Drop for Run { .get_mut() .unwrap_or_else(|p| p.into_inner()) .take(); - self.ctx.shared.sender.failures().discard(done); + self.ctx.shared.sender.failures().discard_some(done); } } @@ -323,7 +323,7 @@ impl JobCtx { self.ctx.shared.sender.send(MailboxItem::Message(envelope)) }) .map_err(|(gone, unsent)| { - self.ctx.shared.sender.failures().discard(unsent); + self.ctx.shared.sender.failures().discard_some(unsent); gone }) } diff --git a/concurrency/tests/disposal_trace.rs b/concurrency/tests/disposal_trace.rs index ca5834730..da19314ba 100644 --- a/concurrency/tests/disposal_trace.rs +++ b/concurrency/tests/disposal_trace.rs @@ -2,10 +2,14 @@ //! actors, for TLA+ trace validation against `verification/tla/Disposal.tla` //! (verification/tla/README.md). Each scenario writes `disposal_.json` //! to `$LIFECYCLE_TRACE_DIR` (default `target/lifecycle-traces`); -//! `verification/tla/trace_check.py` checks them (schema `disposal-v1`). +//! `verification/tla/trace_check.py` checks them (schema `disposal-v1`). A +//! negative control recorded from deliberately broken code (the +//! `disposal.batch_serial` failpoint) goes to `controls/` there, and +//! `negative_checks.py` requires it to be rejected. //! -//! The recording is process-wide (the service is), so scenarios run one at a -//! time and each waits for its workers before taking the trace. +//! The recording is process-wide (the service and the pool are), so +//! scenarios run one at a time, start with the pool quiescent and wait for +//! their workers before taking the trace. #![cfg(all(feature = "lifecycle-trace", feature = "async"))] use spawned_concurrency::{ @@ -34,8 +38,16 @@ fn trace_dir() -> PathBuf { }) } -/// Stop recording and write the trace; returns the events as `Action:x`. -fn write(name: &str) -> Vec { +/// Let the pool settle (every worker waiting), then record. +async fn record() { + tokio::time::sleep(Duration::from_millis(100)).await; + lifecycle_trace::record_disposal(); +} + +/// Let the workers finish, stop recording and write the trace to `dir`; +/// returns the events as `Action:x`. +async fn write_to(dir: PathBuf, name: &str) -> Vec { + tokio::time::sleep(Duration::from_millis(100)).await; let events = lifecycle_trace::take_disposal(); let json = format!( "{{\"name\":\"disposal_{name}\",\"schema\":\"disposal-v1\",\"events\":[\n{}\n]}}\n", @@ -45,7 +57,6 @@ fn write(name: &str) -> Vec { .collect::>() .join(",\n") ); - let dir = trace_dir(); std::fs::create_dir_all(&dir).unwrap(); std::fs::write(dir.join(format!("disposal_{name}.json")), json).unwrap(); events @@ -60,6 +71,10 @@ fn write(name: &str) -> Vec { .collect() } +async fn write(name: &str) -> Vec { + write_to(trace_dir(), name).await +} + fn count(events: &[String], event: &str) -> usize { events.iter().filter(|e| *e == event).count() } @@ -132,13 +147,9 @@ async fn discard_an_emit(owner: &ActorRef, drops: &Arc) { .unwrap(); } -/// Expired requests: two waiting callers are woken by workers, a third -/// (never polled) has nothing to wake. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn expired_callers_are_woken_by_workers() { - let _serial = SERIAL.lock().await; - let owner = Owner::default().start(); - lifecycle_trace::record_disposal(); +/// Two callers waiting with the same deadline (expired in one visit) and +/// one that never polls. +async fn expire_callers(owner: &ActorRef) { let deadline = Deadline::after(Duration::from_millis(50)); let first = owner.send_request(Park, deadline).unwrap(); let second = owner.send_request(Park, deadline).unwrap(); @@ -146,15 +157,28 @@ async fn expired_callers_are_woken_by_workers() { let (a, b) = tokio::join!(first, second); assert!(matches!(a, Err(ActorError::RequestTimeout))); assert!(matches!(b, Err(ActorError::RequestTimeout))); - tokio::time::sleep(Duration::from_millis(100)).await; +} + +async fn stop(owner: ActorRef) { owner.context().stop(); owner.join().await; - tokio::time::sleep(Duration::from_millis(50)).await; - let events = write("expired_callers"); +} + +/// Expired requests: two waiting callers are woken by workers, a third +/// (never polled) has nothing to wake. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn expired_callers_are_woken_by_workers() { + let _serial = SERIAL.lock().await; + let owner = Owner::default().start(); + record().await; + expire_callers(&owner).await; + tokio::time::sleep(Duration::from_millis(100)).await; + stop(owner).await; + let events = write("expired_callers").await; assert_eq!(count(&events, "Expire:wake"), 2, "{events:?}"); assert_eq!(count(&events, "Expire:none"), 1, "{events:?}"); - assert_eq!(count(&events, "Wake"), 2, "{events:?}"); - assert!(count(&events, "Submit") >= 1, "{events:?}"); + assert!(count(&events, "Run") >= 2, "{events:?}"); + assert_eq!(count(&events, "Run"), count(&events, "Done"), "{events:?}"); } /// Payloads released on the deadline service are dropped by workers while @@ -164,7 +188,7 @@ async fn released_payloads_are_dropped_by_workers() { let _serial = SERIAL.lock().await; let owner = Owner::default().start(); let drops = Arc::new(AtomicUsize::new(0)); - lifecycle_trace::record_disposal(); + record().await; discard_an_emit(&owner, &drops).await; tokio::time::sleep(Duration::from_millis(300)).await; assert_eq!( @@ -172,70 +196,112 @@ async fn released_payloads_are_dropped_by_workers() { 1, "the discarded emit was dropped" ); - owner.context().stop(); - owner.join().await; - tokio::time::sleep(Duration::from_millis(50)).await; - let events = write("released_payloads"); - assert!(count(&events, "Release") >= 1, "{events:?}"); + stop(owner).await; + let events = write("released_payloads").await; + assert!(count(&events, "Defer:drop") >= 1, "{events:?}"); assert_eq!( - count(&events, "Drop:dropped"), - count(&events, "Release"), + count(&events, "Settle:dropped"), + count(&events, "Defer:drop"), "{events:?}" ); - assert_eq!(count(&events, "Drop:held"), 0, "{events:?}"); + assert_eq!(count(&events, "Settle:held"), 0, "{events:?}"); } -/// A payload handed off before the owner's exit, reaching its worker after -/// the exit began (the worker is delayed at `disposal.item`), is held and -/// dropped only at the death point. #[cfg(feature = "failpoints")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_payload_handed_off_before_exit_waits_for_the_death_point() { +mod with_failpoints { + use super::*; use spawned_concurrency::failpoints::{self, Action, FailPoint, FailScenario}; - let _serial = SERIAL.lock().await; - let _scenario = FailScenario::setup(); - let (release, gate) = std::sync::mpsc::channel(); - let (stopping_tx, stopping) = std::sync::mpsc::channel(); - let owner = Owner { - gate: Some((gate, stopping_tx)), - ..Owner::default() + + /// A payload handed off before the owner's exit, reaching its worker + /// after the exit began (the worker is delayed at `disposal.item`), is + /// held and dropped only at the death point. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_payload_handed_off_before_exit_waits_for_the_death_point() { + let _serial = SERIAL.lock().await; + let _scenario = FailScenario::setup(); + let (release, gate) = std::sync::mpsc::channel(); + let (stopping_tx, stopping) = std::sync::mpsc::channel(); + let owner = Owner { + gate: Some((gate, stopping_tx)), + ..Owner::default() + } + .start(); + let drops = Arc::new(AtomicUsize::new(0)); + record().await; + failpoints::configure( + "disposal.item", + FailPoint::new(Action::Delay(Duration::from_millis(400))), + ) + .unwrap(); + discard_an_emit(&owner, &drops).await; + // The emit is discarded at 40 ms; its worker waits until ~440 ms. + tokio::time::sleep(Duration::from_millis(100)).await; + owner.context().stop(); + tokio::task::spawn_blocking(move || stopping.recv_timeout(Duration::from_secs(5))) + .await + .unwrap() + .expect("stopped() began"); + // The worker settles the payload while stopped() still runs. + tokio::time::sleep(Duration::from_millis(600)).await; + let before_death = drops.load(Ordering::SeqCst); + release.send(()).unwrap(); + owner.join().await; + failpoints::remove("disposal.item"); + let events = write("held_until_death").await; + assert_eq!( + before_death, 0, + "a payload handed off before the exit was dropped before the death point" + ); + assert_eq!( + drops.load(Ordering::SeqCst), + 1, + "dropped at the death point" + ); + let held = events.iter().position(|e| e == "Settle:held"); + let exit = events.iter().position(|e| e == "Exit"); + let death = events.iter().position(|e| e == "Death"); + assert!(exit < held && held < death && held.is_some(), "{events:?}"); + } + + /// Idle workers retire (keep-alive shortened) and later work starts new + /// ones. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn idle_workers_retire_and_new_ones_start() { + let _serial = SERIAL.lock().await; + let _scenario = FailScenario::setup(); + failpoints::configure("disposal.keep_alive", FailPoint::new(Action::Error)).unwrap(); + // Let the workers left by earlier scenarios retire first. + tokio::time::sleep(Duration::from_millis(100)).await; + let owner = Owner::default().start(); + record().await; + expire_callers(&owner).await; + tokio::time::sleep(Duration::from_millis(100)).await; + expire_callers(&owner).await; + tokio::time::sleep(Duration::from_millis(100)).await; + stop(owner).await; + let events = write("workers_retire").await; + failpoints::remove("disposal.keep_alive"); + assert!(count(&events, "Retire") >= 1, "{events:?}"); + assert!(count(&events, "Begin") >= 2, "{events:?}"); + } + + /// Negative control recorded from broken code: with + /// `disposal.batch_serial` the batch runs serially on one worker. The + /// callers are still woken (nothing blocks here), but the trace must be + /// rejected: a worker runs an item it never took. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn batch_serial_control() { + let _serial = SERIAL.lock().await; + let _scenario = FailScenario::setup(); + let owner = Owner::default().start(); + record().await; + failpoints::configure("disposal.batch_serial", FailPoint::new(Action::Error)).unwrap(); + expire_callers(&owner).await; + tokio::time::sleep(Duration::from_millis(100)).await; + failpoints::remove("disposal.batch_serial"); + stop(owner).await; + let events = write_to(trace_dir().join("controls"), "batch_serial").await; + assert_eq!(count(&events, "Expire:wake"), 2, "{events:?}"); + assert!(count(&events, "Run") > count(&events, "Take"), "{events:?}"); } - .start(); - let drops = Arc::new(AtomicUsize::new(0)); - failpoints::configure( - "disposal.item", - FailPoint::new(Action::Delay(Duration::from_millis(400))), - ) - .unwrap(); - lifecycle_trace::record_disposal(); - discard_an_emit(&owner, &drops).await; - // The emit is discarded at 40 ms; its worker waits until ~440 ms. - tokio::time::sleep(Duration::from_millis(100)).await; - owner.context().stop(); - tokio::task::spawn_blocking(move || stopping.recv_timeout(Duration::from_secs(5))) - .await - .unwrap() - .expect("stopped() began"); - // The worker settles the payload while stopped() still runs. - tokio::time::sleep(Duration::from_millis(600)).await; - let before_death = drops.load(Ordering::SeqCst); - release.send(()).unwrap(); - owner.join().await; - failpoints::remove("disposal.item"); - tokio::time::sleep(Duration::from_millis(50)).await; - let events = write("held_until_death"); - assert_eq!( - before_death, 0, - "a payload handed off before the exit was dropped before the death point" - ); - assert_eq!( - drops.load(Ordering::SeqCst), - 1, - "dropped at the death point" - ); - assert!(count(&events, "Drop:held") >= 1, "{events:?}"); - let held = events.iter().position(|e| e == "Drop:held").unwrap(); - let exit = events.iter().position(|e| e == "Exit").unwrap(); - let death = events.iter().position(|e| e == "Death").unwrap(); - assert!(exit < held && held < death, "{events:?}"); } diff --git a/docs/TESTING.md b/docs/TESTING.md index 689e5879b..1cb677fa7 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1883,12 +1883,14 @@ and prints the seed when a test fails. Configuration is process-global. | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | | `job.emit_due` | no | A scheduled `emit_at` coming due during the owner's exit: its payload is held until every caller is answered (`emit_due_during_exit_*`) | | `disposal.item` | no | A disposal worker delayed before an item the deadline service handed off, so the owner's exit begins first: the payload is held until the death point (`disposal_trace`: `a_payload_handed_off_before_exit_waits_for_the_death_point`) | +| `disposal.batch_serial` | yes | Error: the deadline service's batch runs serially on one worker (the shared-batch bug); its recorded trace must be rejected (`disposal_trace`, negative control) | +| `disposal.keep_alive` | yes | Error: idle disposal workers retire after 5 ms, so the trace covers `Retire` and later starts (`disposal_trace`) | | `job.native_worker_unused` | no | Threads mode: the parked thread of a dropped, unused blocking-job permit held at its exit; the native slot is already back (`an_unused_permit_returns_its_slot_without_its_thread`) | | `reply.close_upgraded` | no | The death point's close holding an already-answered slot while its caller drops the receiver: the unread reply is destroyed only after every caller is answered (tasks and threads) | | `reply.recv_parked` | no | Handshake: a blocking `recv()` is parked, so only a notification wakes it (discharge notifies parked receivers before any waker; tasks and threads) | | `monitor.down_before_dispatch` | no | `demonitor` on another thread returning while the monitor's `Down` is in dispatch: it is flushed (tasks and threads) | -`tests/failpoints.rs` drives every site (`disposal.item` is driven by +`tests/failpoints.rs` drives every site (the `disposal.*` sites are driven by `tests/disposal_trace.rs`) and checks invariants through public APIs: restarts follow the budget (generation count equals injected failures plus started actors), a stop during a restart or publication window starts or diff --git a/verification/model-map.toml b/verification/model-map.toml index ac54b30f9..a485265c0 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -156,7 +156,7 @@ code = [ "concurrency/src/threads/jobs.rs", "concurrency/src/lifecycle_trace.rs", ] -coverage_floor = 7 # of 7 action keys (TraceDisposal arms) +coverage_floor = 13 # of 13 action keys (TraceDisposal arms) [models.ReplyTimer] # Receiver-side refinement: the shared runtime deadline and shortening mark. diff --git a/verification/tla/Disposal.tla b/verification/tla/Disposal.tla index 94dda49b6..185b1527c 100644 --- a/verification/tla/Disposal.tla +++ b/verification/tla/Disposal.tla @@ -9,57 +9,71 @@ (* One owner's deadline heap holds N requests (deferred tokens or awaited *) (* raw receivers), each with a deadline fixed at Init and possibly a *) (* waiting caller (a waker). The one process-wide service thread visits *) -(* the heap: *) -(* Expire(r) the slot's OPEN -> TIMED_OUT transition and its credit *) -(* return, in one step; a waiting caller's wake-up is put *) -(* in the visit's batch (not run) *) -(* Release(p) a payload the visit releases (a discarded scheduled emit, *) -(* a displaced job result, `done` and its captures) is put *) -(* in the batch; one released while the owner's hold is on *) -(* is held at once and is no step (DropFailures::discard) *) -(* Submit the visit's batch goes to the pool, one queue entry per *) -(* item *) -(* Workers (at most Workers) are started while queued entries outnumber *) -(* idle workers, take entries, run them, and retire when idle with nothing *) -(* queued. Running a wake-up wakes the caller; running a payload re-checks *) -(* the owner's hold: held if its exit began (dropped at its death point), *) -(* dropped otherwise. Items in Blocking never return (a blocking waker or *) -(* destructor): their worker is lost, nothing else waits for it. *) +(* the heap and collects what it releases in the visit's batch: *) +(* Expire(r) the slot's OPEN -> TIMED_OUT transition and its credit *) +(* return, in one step; a waiting caller's wake-up item *) +(* Release(p) a payload (a discarded scheduled emit, a displaced job *) +(* result, `done` and its captures); one released while the *) +(* owner's hold is on is held at once and is no step *) +(* Other(o) anything else whose drop or call may run user code (the *) +(* last reference to a job or registry, a job's emitter *) +(* wakers, a slot holding an unread reply) *) +(* Submit the batch goes to the pool, one queue entry per item; *) +(* then the start rule *) +(* The pool mirrors run()/submit(): a worker is "none" (not started), *) +(* "active" (in its loop, holding the lock), "idle" (waiting: counted in *) +(* `idle`), "taken" (holds an entry), "in" (runs its next item) or *) +(* "settled" (a payload's hold check is done, its destructor runs). The *) +(* start rule (`needs_worker`): when queued entries outnumber idle *) +(* workers, none is starting and fewer than Workers exist (the cap), one *) +(* starts; it is checked after Submit and after every Take. A payload's *) +(* run is two steps, as `DropFailures::settle`: Settle decides under the *) +(* hold lock (held if the owner's exit began, else dropped) and the *) +(* destructor then runs until Done. Items in Blocking never reach Done (a *) +(* blocking waker or destructor): their worker is lost. *) (* *) (* The owner: Exit (DropFailures::hold, its exit begins) and Death (the *) -(* death point; DropFailures::release drops what was held). *) +(* death point; DropFailures::release drops what was held). A destructor *) +(* whose drop was decided before Exit may still run during the exit: it *) +(* is pre-exit work and Exit never waits for it (deferred-replies.md §5); *) +(* NoDropWhileHeld is about the decision. *) (* *) (* The clock: Tick only when no framework step is pending (an expiry due, *) -(* a batch not submitted, a runnable or startable worker), so a property *) -(* "within the tick" says the step does not wait for a blocked item. *) +(* a batch not submitted, a worker that can start, take or run), so a *) +(* property "within the tick" says the step does not wait for a blocked *) +(* item. *) (* *) (* Fault selects a negative control; negative_checks.py requires each to *) (* violate the named invariant: *) (* InlineDisposal (TimelyExpiry) the service runs the batch *) -(* itself (0.8.0): a blocking item *) -(* stops every later expiry *) +(* itself (0.8.0) *) (* WakeBeforeTransition (WakeAfterTransition) the wake-up is handed off *) (* before the transition commits *) (* DropWhileHeld (NoDropWhileHeld) the worker drops a payload *) (* without re-checking the hold *) -(* BatchSerial (TimelyWake) a visit's batch is one queue *) -(* entry run serially: a blocking *) -(* item holds back the other *) -(* callers' wake-ups (review of *) -(* PR #282) *) +(* BatchSerial (TimelyWake) a visit's batch is one entry *) +(* run serially (the shared-batch *) +(* bug of PR #282's first rev.) *) +(* NoStartOnTake (TimelyWake) the start rule is checked only *) +(* at Submit: a queued item can be *) +(* stranded behind blocked workers *) (***************************************************************************) EXTENDS Naturals, Sequences, FiniteSets -CONSTANTS N, P, MaxTime, Workers, Blocking, Fault +CONSTANTS N, P, O, MaxTime, Workers, Blocking, Fault, InitIdle ASSUME Fault \in {"None", "InlineDisposal", "WakeBeforeTransition", - "DropWhileHeld", "BatchSerial"} + "DropWhileHeld", "BatchSerial", "NoStartOnTake"} Req == 1..N Pay == 1..P +Oth == 1..O W == 1..Workers +ASSUME InitIdle \subseteq W WakeItem(r) == [k |-> "wake", id |-> r] DropItem(p) == [k |-> "drop", id |-> p] +OtherItem(o) == [k |-> "other", id |-> o] Items == {WakeItem(r) : r \in Req} \cup {DropItem(p) : p \in Pay} + \cup {OtherItem(o) : o \in Oth} ASSUME Blocking \subseteq Items VARIABLES @@ -67,17 +81,20 @@ VARIABLES slot, credit, expAt, \* "open"/"timeout"; its credit; when it expired woken, \* the caller's wake-up ran cas, \* WakeBeforeTransition: handed off, not committed - payload, \* "none" (owned), "out" (handed off), "held", "dropped" + payload, \* "none", "out", "held", "dropping", "dropped" + other, \* "none", "out", "done" batch, \* the current visit's collected items queue, \* queued entries (each a sequence of items) - cur, \* the entry a worker runs (<<>>: idle or not started) - alive, \* started workers + ws, \* each worker's state (see above) + cur, \* the entry a worker holds (<<>> if none) + starting, \* a worker start was decided, not yet begun svc, \* "idle", or "stuck" in a blocking item (InlineDisposal) phase, \* the owner: "run", "exiting" (hold on), "dead" - early \* a payload was dropped while the hold was on + early \* a payload's drop was decided while the hold was on -vars == <> +vars == <> +pool == <> Init == /\ now = 0 @@ -89,20 +106,30 @@ Init == /\ woken = [r \in Req |-> FALSE] /\ cas = {} /\ payload = [p \in Pay |-> "none"] + /\ other = [o \in Oth |-> "none"] /\ batch = <<>> /\ queue = <<>> + /\ ws = [w \in W |-> IF w \in InitIdle THEN "idle" ELSE "none"] /\ cur = [w \in W |-> <<>>] - /\ alive = {} + /\ starting = FALSE /\ svc = "idle" /\ phase = "run" /\ early = FALSE +------------------------------------------------------------------------------ +\* The pool's bookkeeping (disposal.rs `State`) + +IdleCount == Cardinality({w \in W : ws[w] = "idle"}) +Started == {w \in W : ws[w] # "none"} +\* `needs_worker` on queue `q` with `idle` idle workers. +NeedsWorker(q, idle) == + /\ Len(q) > idle /\ ~starting + /\ Cardinality(Started) < Workers + ------------------------------------------------------------------------------ \* The service thread -\* InlineDisposal: the service runs the item itself. -RunInline(item) == - IF item \in Blocking THEN svc' = "stuck" ELSE UNCHANGED svc +RunInline(item) == IF item \in Blocking THEN svc' = "stuck" ELSE UNCHANGED svc Due(r) == slot[r] = "open" /\ r \notin cas /\ now >= dl[r] @@ -123,7 +150,7 @@ Expire(r) == /\ IF waiting[r] THEN RunInline(WakeItem(r)) ELSE UNCHANGED svc ELSE /\ batch' = IF waiting[r] THEN Append(batch, WakeItem(r)) ELSE batch /\ UNCHANGED <> - /\ UNCHANGED <> + /\ UNCHANGED <> \* WakeBeforeTransition only: the transition commits after the hand-off. Commit(r) == @@ -132,109 +159,166 @@ Commit(r) == /\ slot' = [slot EXCEPT ![r] = "timeout"] /\ credit' = [credit EXCEPT ![r] = FALSE] /\ expAt' = [expAt EXCEPT ![r] = now] - /\ UNCHANGED <> + /\ UNCHANGED <> Release(p) == /\ svc = "idle" /\ payload[p] = "none" /\ phase # "exiting" /\ IF Fault = "InlineDisposal" - THEN /\ payload' = [payload EXCEPT ![p] = IF DropItem(p) \in Blocking THEN "out" ELSE "dropped"] + THEN /\ payload' = [payload EXCEPT ![p] = IF DropItem(p) \in Blocking THEN "dropping" ELSE "dropped"] /\ RunInline(DropItem(p)) /\ UNCHANGED batch ELSE /\ payload' = [payload EXCEPT ![p] = "out"] /\ batch' = Append(batch, DropItem(p)) /\ UNCHANGED svc - /\ UNCHANGED <> + /\ UNCHANGED <> + +Other(o) == + /\ svc = "idle" /\ other[o] = "none" + /\ IF Fault = "InlineDisposal" + THEN /\ other' = [other EXCEPT ![o] = IF OtherItem(o) \in Blocking THEN "out" ELSE "done"] + /\ RunInline(OtherItem(o)) + /\ UNCHANGED batch + ELSE /\ other' = [other EXCEPT ![o] = "out"] + /\ batch' = Append(batch, OtherItem(o)) + /\ UNCHANGED svc + /\ UNCHANGED <> \* One queue entry per item; BatchSerial: the whole batch as one entry. Entries(b) == IF Fault = "BatchSerial" THEN <> ELSE [i \in 1..Len(b) |-> <>] -Submit == +\* `spawn`: whether the code decided to start a worker (the trace's x). +SubmitDecide(spawn) == /\ batch # <<>> /\ queue' = queue \o Entries(batch) /\ batch' = <<>> - /\ UNCHANGED <> + /\ starting' = (starting \/ NeedsWorker(queue', IdleCount)) + /\ spawn = (starting' /\ ~starting) + /\ UNCHANGED <> +Submit == \E spawn \in BOOLEAN : SubmitDecide(spawn) ------------------------------------------------------------------------------ -\* Disposal workers - -Idle(w) == w \in alive /\ cur[w] = <<>> -IdleCount == Cardinality({w \in W : Idle(w)}) - -Start(w) == - /\ w \notin alive /\ Len(queue) > IdleCount - /\ alive' = alive \cup {w} - /\ UNCHANGED <> - -Take(w) == - /\ Idle(w) /\ queue # <<>> +\* Disposal workers (disposal.rs `run`) + +\* A started thread enters its loop (`starting = false`). +Begin(w) == + /\ ws[w] = "none" /\ starting + /\ ws' = [ws EXCEPT ![w] = "active"] /\ starting' = FALSE + /\ UNCHANGED <> + +\* Pop an entry; then the start rule for what is still queued. +TakeDecide(w, spawn) == + /\ ws[w] = "active" /\ queue # <<>> + /\ ws' = [ws EXCEPT ![w] = "taken"] /\ cur' = [cur EXCEPT ![w] = Head(queue)] /\ queue' = Tail(queue) - /\ UNCHANGED <> + /\ starting' = (starting \/ (Fault # "NoStartOnTake" /\ NeedsWorker(Tail(queue), IdleCount))) + /\ spawn = (starting' /\ ~starting) + /\ UNCHANGED <> +Take(w) == \E spawn \in BOOLEAN : TakeDecide(w, spawn) + +\* Nothing queued: wait (counted idle). +Wait(w) == + /\ ws[w] = "active" /\ queue = <<>> + /\ ws' = [ws EXCEPT ![w] = "idle"] + /\ UNCHANGED <> + +\* Back under the lock after a notification, a time-out or spuriously. +Woke(w) == + /\ ws[w] = "idle" + /\ ws' = [ws EXCEPT ![w] = "active"] + /\ UNCHANGED <> -\* Run the worker's next item; a blocking one never returns. +Retire(w) == + /\ ws[w] = "active" /\ queue = <<>> + /\ ws' = [ws EXCEPT ![w] = "none"] + /\ UNCHANGED <> + +\* Start the entry's next item: a wake-up wakes its caller here. Run(w) == - /\ cur[w] # <<>> + /\ ws[w] = "taken" + /\ ws' = [ws EXCEPT ![w] = "in"] + /\ LET item == Head(cur[w]) IN + IF item.k = "wake" THEN woken' = [woken EXCEPT ![item.id] = TRUE] + ELSE UNCHANGED woken + /\ UNCHANGED <> + +\* A payload's hold check (`DropFailures::settle`, under the hold lock). +SettleTo(w, x) == + /\ ws[w] = "in" /\ Head(cur[w]).k = "drop" + /\ LET p == Head(cur[w]).id + held == phase = "exiting" /\ Fault # "DropWhileHeld" IN + /\ x = IF held THEN "held" ELSE "dropped" + /\ payload' = [payload EXCEPT ![p] = IF held THEN "held" ELSE "dropping"] + /\ early' = (early \/ (~held /\ phase = "exiting")) + /\ ws' = [ws EXCEPT ![w] = "settled"] + /\ UNCHANGED <> +Settle(w) == \E x \in {"held", "dropped"} : SettleTo(w, x) + +\* The item returns (a blocking one never does). +Done(w) == + /\ \/ ws[w] = "in" /\ Head(cur[w]).k # "drop" + \/ ws[w] = "settled" /\ LET item == Head(cur[w]) IN /\ item \notin Blocking - /\ cur' = [cur EXCEPT ![w] = Tail(cur[w])] - /\ IF item.k = "wake" - THEN /\ woken' = [woken EXCEPT ![item.id] = TRUE] - /\ UNCHANGED <> - ELSE /\ IF phase = "exiting" /\ Fault # "DropWhileHeld" - THEN /\ payload' = [payload EXCEPT ![item.id] = "held"] - /\ UNCHANGED early - ELSE /\ payload' = [payload EXCEPT ![item.id] = "dropped"] - /\ early' = (early \/ phase = "exiting") - /\ UNCHANGED woken - /\ UNCHANGED <> - -Retire(w) == - /\ Idle(w) /\ queue = <<>> - /\ alive' = alive \ {w} - /\ UNCHANGED <> + /\ IF item.k = "drop" /\ payload[item.id] = "dropping" + THEN payload' = [payload EXCEPT ![item.id] = "dropped"] ELSE UNCHANGED payload + /\ IF item.k = "other" THEN other' = [other EXCEPT ![item.id] = "done"] ELSE UNCHANGED other + /\ cur' = [cur EXCEPT ![w] = Tail(cur[w])] + /\ ws' = [ws EXCEPT ![w] = IF Tail(cur[w]) = <<>> THEN "active" ELSE "taken"] + /\ UNCHANGED <> ------------------------------------------------------------------------------ \* The owner Exit == /\ phase = "run" /\ phase' = "exiting" - /\ UNCHANGED <> + /\ UNCHANGED <> \* The death point: what was held is dropped. Death == /\ phase = "exiting" /\ phase' = "dead" /\ payload' = [p \in Pay |-> IF payload[p] = "held" THEN "dropped" ELSE payload[p]] - /\ UNCHANGED <> + /\ UNCHANGED <> ------------------------------------------------------------------------------ \* The clock +Stuck(w) == ws[w] \in {"in", "settled"} /\ Head(cur[w]) \in Blocking + \* A framework step is pending (it never waits for user code). Pending == \/ svc = "idle" /\ \E r \in Req : Due(r) \/ cas # {} \/ batch # <<>> - \/ \E w \in W : cur[w] # <<>> /\ Head(cur[w]) \notin Blocking - \/ queue # <<>> /\ \E w \in W : Idle(w) \/ w \notin alive + \/ starting + \/ \E w \in W : ws[w] \in {"active", "taken"} + \/ \E w \in W : ws[w] \in {"in", "settled"} /\ ~Stuck(w) + \/ queue # <<>> /\ \E w \in W : ws[w] = "idle" Tick == /\ now < MaxTime /\ ~Pending /\ now' = now + 1 - /\ UNCHANGED <> + /\ UNCHANGED <> Next == \/ Tick \/ Submit \/ Exit \/ Death \/ \E r \in Req : Expire(r) \/ Commit(r) \/ \E p \in Pay : Release(p) - \/ \E w \in W : Start(w) \/ Take(w) \/ Run(w) \/ Retire(w) + \/ \E o \in Oth : Other(o) + \/ \E w \in W : Begin(w) \/ Take(w) \/ Wait(w) \/ Woke(w) \/ Retire(w) + \/ Run(w) \/ Settle(w) \/ Done(w) Spec == Init /\ [][Next]_vars -\* Fairness for the clock, the service and the workers (framework code); -\* none for the owner or for a blocking item (Run is not enabled for it). -FairSpec == Spec /\ WF_vars(Tick) /\ WF_vars(Submit) - /\ \A r \in Req : WF_vars(Expire(r)) /\ WF_vars(Commit(r)) - /\ \A w \in W : WF_vars(Start(w)) /\ WF_vars(Take(w)) /\ WF_vars(Run(w)) +\* Fairness for the clock, the service and the workers (framework code; an +\* idle worker is notified when work is queued); none for the owner, for +\* Retire, or for a blocking item (Done is not enabled for it). +FairSpec == + /\ Spec /\ WF_vars(Tick) /\ WF_vars(Submit) + /\ \A r \in Req : WF_vars(Expire(r)) /\ WF_vars(Commit(r)) + /\ \A w \in W : /\ WF_vars(Begin(w)) /\ WF_vars(Take(w)) /\ WF_vars(Wait(w)) + /\ WF_vars(Woke(w) /\ queue # <<>>) + /\ WF_vars(Run(w)) /\ WF_vars(Settle(w)) /\ WF_vars(Done(w)) ------------------------------------------------------------------------------ \* Properties @@ -243,8 +327,10 @@ TypeOK == /\ now \in 0..MaxTime /\ slot \in [Req -> {"open", "timeout"}] /\ credit \in [Req -> BOOLEAN] /\ woken \in [Req -> BOOLEAN] - /\ payload \in [Pay -> {"none", "out", "held", "dropped"}] - /\ cas \subseteq Req /\ alive \subseteq W + /\ payload \in [Pay -> {"none", "out", "held", "dropping", "dropped"}] + /\ other \in [Oth -> {"none", "out", "done"}] + /\ ws \in [W -> {"none", "active", "idle", "taken", "in", "settled"}] + /\ cas \subseteq Req /\ starting \in BOOLEAN /\ phase \in {"run", "exiting", "dead"} /\ svc \in {"idle", "stuck"} \* A deferral credit is held exactly while the slot is open: it returns with @@ -252,21 +338,32 @@ TypeOK == CreditWithTransition == \A r \in Req : credit[r] = (slot[r] = "open") \* No wake-up runs before its slot's transition and credit return. WakeAfterTransition == \A r \in Req : woken[r] => (slot[r] = "timeout" /\ ~credit[r]) -\* No payload handed off is dropped while the owner's hold is on: it waits -\* for the death point (R6, callers before payloads). +\* No handed-off payload's drop is decided while the owner's hold is on: it +\* is held and dropped at the death point (R6, callers before payloads). NoDropWhileHeld == ~early -\* The service does not wait for user code: no due slot stays open past its -\* deadline (the clock passes it only if the service is stuck). +\* The cap: never more than Workers started or starting. +WithinCap == Cardinality(Started) + (IF starting THEN 1 ELSE 0) <= Workers +\* The service does not wait for user code. TimelyExpiry == \A r \in Req : slot[r] = "open" => now <= dl[r] \* A wake-up that does not itself block runs within the tick of its expiry, \* whatever else blocks (another item of the same visit included). TimelyWake == \A r \in Req : (slot[r] = "timeout" /\ waiting[r] /\ WakeItem(r) \notin Blocking /\ now > expAt[r]) => woken[r] +\* At the cap: a late wake-up only while every worker is stuck in a +\* blocking item (MCDisposalCapped). +TimelyWakeBelowCap == + \A r \in Req : (slot[r] = "timeout" /\ waiting[r] /\ WakeItem(r) \notin Blocking + /\ now > expAt[r] /\ ~woken[r]) + => \A w \in W : Stuck(w) Safety == /\ TypeOK /\ CreditWithTransition /\ WakeAfterTransition /\ NoDropWhileHeld - /\ TimelyExpiry /\ TimelyWake + /\ WithinCap /\ TimelyExpiry /\ TimelyWake + +SafetyCapped == + /\ TypeOK /\ CreditWithTransition /\ WakeAfterTransition /\ NoDropWhileHeld + /\ WithinCap /\ TimelyExpiry /\ TimelyWakeBelowCap \* Every handed-off wake-up that does not block eventually runs. EveryWakeRuns == diff --git a/verification/tla/MCDisposal.cfg b/verification/tla/MCDisposal.cfg index 305d4b107..13632ee8c 100644 --- a/verification/tla/MCDisposal.cfg +++ b/verification/tla/MCDisposal.cfg @@ -7,7 +7,9 @@ PROPERTY EveryDueExpires CONSTANTS N = 2 P = 1 + O = 0 MaxTime = 2 Workers = 3 + InitIdle = {} Blocking <- MCBlocking Fault = "None" diff --git a/verification/tla/MCDisposalCapped.cfg b/verification/tla/MCDisposalCapped.cfg new file mode 100644 index 000000000..b87a54b38 --- /dev/null +++ b/verification/tla/MCDisposalCapped.cfg @@ -0,0 +1,14 @@ +\* At the cap (CI, every PR): one worker, the first caller's waker blocks. +\* A later wake-up may wait only while every worker is stuck. +SPECIFICATION FairSpec +INVARIANT SafetyCapped +PROPERTY EveryDueExpires +CONSTANTS + N = 2 + P = 0 + O = 0 + MaxTime = 2 + Workers = 1 + InitIdle = {} + Blocking <- MCBlocking + Fault = "None" diff --git a/verification/tla/MCDisposalCapped.tla b/verification/tla/MCDisposalCapped.tla new file mode 100644 index 000000000..c66b1e9c0 --- /dev/null +++ b/verification/tla/MCDisposalCapped.tla @@ -0,0 +1,5 @@ +-------------------------- MODULE MCDisposalCapped -------------------------- +EXTENDS Disposal +\* The first caller's waker never returns, and there is one worker. +MCBlocking == {WakeItem(1)} +============================================================================= diff --git a/verification/tla/MCDisposalDrop.cfg b/verification/tla/MCDisposalDrop.cfg index 8e190140f..bd1767008 100644 --- a/verification/tla/MCDisposalDrop.cfg +++ b/verification/tla/MCDisposalDrop.cfg @@ -1,13 +1,15 @@ -\* Fast instance (CI, every PR): a blocking payload destructor; two requests, -\* two payloads, three workers, clock to 2. +\* Fast instance (CI, every PR): a blocking payload destructor; one request, +\* two payloads, one other item, three workers, clock to 2. SPECIFICATION FairSpec INVARIANT Safety PROPERTY EveryWakeRuns PROPERTY EveryDueExpires CONSTANTS - N = 2 + N = 1 P = 2 + O = 1 MaxTime = 2 Workers = 3 + InitIdle = {} Blocking <- MCBlocking Fault = "None" diff --git a/verification/tla/TraceDisposal.tla b/verification/tla/TraceDisposal.tla index 273e3b9d0..0ddf8798e 100644 --- a/verification/tla/TraceDisposal.tla +++ b/verification/tla/TraceDisposal.tla @@ -1,29 +1,33 @@ ----------------------------- MODULE TraceDisposal ----------------------------- (***************************************************************************) (* Trace validation for Disposal.tla: the deadline service's hand-off of *) -(* wake-ups and payloads to the disposal workers, and the owner's hold. *) +(* wake-ups and payloads to the disposal workers, the pool's bookkeeping, *) +(* and the owner's hold. *) (* *) (* With the `lifecycle-trace` feature, while `record_disposal` is on, the *) -(* code records (concurrency/src/lifecycle_trace.rs, DisposalEvent; `id` *) -(* is fresh per Expire or Release and numbered by trace_check.py): *) -(* Expire a slot's OPEN -> TIMED_OUT and credit return on the service *) -(* thread, in a collecting visit (registry.rs, Expirable for *) -(* Slot); x: "wake" if a wake-up was handed off, else "none" *) +(* code records (concurrency/src/lifecycle_trace.rs, DisposalEvent): *) +(* on the service thread *) +(* Expire a slot's OPEN -> TIMED_OUT and credit return, in a visit *) +(* (registry.rs); x: "wake" if its wake-up was handed off *) (* (Expire, with the caller's `waiting` matching x) *) -(* Release a payload deferred from the service thread, in one section *) -(* with the hold check (DropFailures::discard) (Release) *) -(* Submit the visit's work handed to the pool (deadline.rs) (Submit; *) -(* a visit whose items are all untracked, such as the last *) -(* reference to an entry, submits an empty batch here) *) -(* Wake a worker runs a handed-off wake-up (Run of a wake item) *) -(* Drop a worker settles a payload in one section with the hold *) -(* check (DropFailures::settle); x: "held" or "dropped" (Run of *) -(* a drop item) *) -(* Exit the owner's hold begins (DropFailures::hold) (Exit) *) -(* Death the hold is released at the death point (Death) *) +(* Defer a payload (x: "drop", DropFailures::discard, in one section *) +(* with the hold check) or another item (x: "other": the last *) +(* reference to a job or registry, emitter wakers, a slot with *) +(* an unread reply) handed off (Release, Other) *) +(* Submit the visit's items queued and the start rule (x: "spawn" if *) +(* a worker start was decided) (Submit) *) +(* in the pool (disposal.rs `run`, each under the pool lock except Run, *) +(* Settle and Done) *) +(* Begin, Take (x as for Submit), Wait, Woke, Retire (same names) *) +(* Run the worker starts its item (a wake-up wakes its caller) (Run)*) +(* Settle a payload's hold check, x: "held" or "dropped" (Settle) *) +(* Done the item returned (Done) *) +(* and the owner: Exit (DropFailures::hold) and Death (release). *) (* *) -(* Silent: the clock, and the workers' Start, Take and Retire. No item *) -(* blocks in a recorded run (Blocking = {}). *) +(* trace_check.py turns the leading `Pool` event (the pool when recording *) +(* starts: its idle workers) into InitIdle, numbers requests, payloads, *) +(* other items and workers, and sets Workers to the code's cap. The clock *) +(* is silent. No item blocks in a recorded run (Blocking = {}). *) (***************************************************************************) EXTENDS Disposal, TLC @@ -35,27 +39,31 @@ traceVars == <> TraceInit == Init /\ i = 0 -\* Some worker runs `item` next. -WorkerRuns(item) == \E w \in W : cur[w] # <<>> /\ Head(cur[w]) = item /\ Run(w) +Item(e) == [k |-> e.k, id |-> e.n] Step(e) == - CASE e.a = "Expire" -> e.id \in Req /\ waiting[e.id] = (e.x = "wake") /\ Expire(e.id) - [] e.a = "Release" -> e.id \in Pay /\ Release(e.id) - [] e.a = "Submit" -> \/ Submit - \/ batch = <<>> /\ UNCHANGED vars - [] e.a = "Wake" -> e.id \in Req /\ WorkerRuns(WakeItem(e.id)) - [] e.a = "Drop" -> e.id \in Pay /\ WorkerRuns(DropItem(e.id)) /\ payload'[e.id] = e.x - [] e.a = "Exit" -> Exit - [] e.a = "Death" -> Death - [] OTHER -> FALSE - -Silent == \/ Tick - \/ \E w \in W : Start(w) \/ Take(w) \/ Retire(w) + CASE e.a = "Expire" -> e.n \in Req /\ waiting[e.n] = (e.x = "wake") /\ Expire(e.n) + [] e.a = "Defer" -> \/ e.k = "drop" /\ e.n \in Pay /\ Release(e.n) + \/ e.k = "other" /\ e.n \in Oth /\ Other(e.n) + [] e.a = "Submit" -> SubmitDecide(e.x = "spawn") + [] e.a = "Begin" -> e.w \in W /\ Begin(e.w) + [] e.a = "Take" -> e.w \in W /\ queue # <<>> /\ Head(queue) = <> + /\ TakeDecide(e.w, e.x = "spawn") + [] e.a = "Wait" -> e.w \in W /\ Wait(e.w) + [] e.a = "Woke" -> e.w \in W /\ Woke(e.w) + [] e.a = "Retire" -> e.w \in W /\ Retire(e.w) + [] e.a = "Run" -> e.w \in W /\ cur[e.w] # <<>> /\ Head(cur[e.w]) = Item(e) /\ Run(e.w) + [] e.a = "Settle" -> e.w \in W /\ cur[e.w] # <<>> /\ Head(cur[e.w]) = Item(e) + /\ SettleTo(e.w, e.x) + [] e.a = "Done" -> e.w \in W /\ cur[e.w] # <<>> /\ Head(cur[e.w]) = Item(e) /\ Done(e.w) + [] e.a = "Exit" -> Exit + [] e.a = "Death" -> Death + [] OTHER -> FALSE TraceNext == /\ i < Len(Trace) /\ \/ Step(Trace[i + 1]) /\ i' = i + 1 - \/ Silent /\ UNCHANGED i + \/ Tick /\ UNCHANGED i TraceSpec == TraceInit /\ [][TraceNext]_traceVars diff --git a/verification/tla/coverage-baseline.json b/verification/tla/coverage-baseline.json index 19c0b89ab..8409ffb20 100644 --- a/verification/tla/coverage-baseline.json +++ b/verification/tla/coverage-baseline.json @@ -151,12 +151,18 @@ "A|StopNext" ], "Disposal": [ + "A|Begin", "A|Death", - "A|Drop", + "A|Defer", + "A|Done", "A|Exit", "A|Expire", - "A|Release", + "A|Retire", + "A|Run", + "A|Settle", "A|Submit", - "A|Wake" + "A|Take", + "A|Wait", + "A|Woke" ] } diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index b3f30b8cc..f4e6ee91b 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -287,6 +287,7 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDisposal", "WakeBeforeTransition", "WakeAfterTransition"), ("MCDisposal", "DropWhileHeld", "NoDropWhileHeld"), ("MCDisposal", "BatchSerial", "TimelyWake"), + ("MCDisposal", "NoStartOnTake", "TimelyWake"), ("MCDeferredJobs", "NoInstanceCheck", "NoLateResult"), ("MCDeferredJobs", "NoCancelFence", "NoStaleAfterCancel"), ("MCDeferredJobs", "DoubleTerminal", "AtMostOneTerminal"), @@ -381,30 +382,51 @@ def start_controls(tlc): ] -def disposal_trace_controls(traces, tlc): - """Edits of recorded disposal traces that Disposal.tla must reject.""" +def disposal_trace_controls(traces, tlc, directory): + """Disposal traces Disposal.tla must reject: edits of recorded traces, + and traces recorded from deliberately broken code (`controls/`).""" by_name = {t["name"]: t for t in traces if t.get("schema") == "disposal-v1"} def wake_before_expire(events): - wake = next(i for i, e in enumerate(events) if e["a"] == "Wake") - expire = next(i for i, e in enumerate(events) - if e["a"] == "Expire" and e["id"] == events[wake]["id"]) - events.insert(expire, events.pop(wake)) + expire = next(i for i, e in enumerate(events) if e["a"] == "Expire" and e["x"] == "wake") + run = next(i for i, e in enumerate(events) + if e["a"] == "Run" and e["id"] == events[expire]["id"]) + events.insert(expire, events.pop(run)) require_rejected(by_name["disposal_expired_callers"], wake_before_expire, "a wake-up run before its slot's transition", tlc) - def drop_before_submit(events): - drop = next(i for i, e in enumerate(events) if e["a"] == "Drop") + def run_before_submit(events): + run = next(i for i, e in enumerate(events) if e["a"] == "Run") submit = next(i for i, e in enumerate(events) if e["a"] == "Submit") - events.insert(submit, events.pop(drop)) - require_rejected(by_name["disposal_released_payloads"], drop_before_submit, - "a payload dropped by a worker before the hand-off", tlc) + events.insert(submit, events.pop(run)) + require_rejected(by_name["disposal_released_payloads"], run_before_submit, + "a payload run by a worker before the hand-off", tlc) def drop_while_held(events): - next(e for e in events if e["a"] == "Drop" and e["x"] == "held")["x"] = "dropped" + next(e for e in events if e["a"] == "Settle" and e["x"] == "held")["x"] = "dropped" require_rejected(by_name["disposal_held_until_death"], drop_while_held, "a handed-off payload dropped while the owner's hold is on", tlc) + def missing_start(events): + next(e for e in events if e["a"] in {"Submit", "Take"} and e["x"] == "spawn")["x"] = "" + spawning = next(t for t in by_name.values() + if any(e["a"] in {"Submit", "Take"} and e["x"] == "spawn" for e in t["events"])) + require_rejected(spawning, missing_start, + "a worker start the start rule requires is not decided", tlc) + + def serial_in_one_entry(events): + # Two items of one visit run by one worker after one Take. + takes = [i for i, e in enumerate(events) if e["a"] == "Take"] + events.pop(takes[1]) + require_rejected(by_name["disposal_expired_callers"], serial_in_one_entry, + "a worker runs an item it never took", tlc) + + # Recorded from code with `disposal.batch_serial` (the shared-batch bug). + for path in sorted((directory / "controls").glob("disposal_*.json")): + control = json.loads(path.read_text()) + require_rejected(control, lambda events: None, + f"recorded from broken code: {control['name']}", tlc) + def reply_trace_controls(traces, tlc): """Edits of recorded reply-slot traces that DeferredReplies.tla must @@ -829,7 +851,7 @@ def decide_at_once(events): job_trace_controls(traces, args.tlc) coordinator_trace_controls(traces, args.tlc) reply_trace_controls(traces, args.tlc) - disposal_trace_controls(traces, args.tlc) + disposal_trace_controls(traces, args.tlc, args.traces) lifecycle_trace_controls(traces, args.tlc) for module, fault, invariant in LIFECYCLE_CONTROLS: named_fault(module, fault, invariant, args.tlc) diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index 5162cc228..3d14e4f42 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -152,48 +152,79 @@ def generate_replies(trace, workdir): def generate_disposal(trace, workdir): - """The deadline service's hand-off and the disposal workers against - Disposal.tla. Expire ids number the requests 1..N, Release ids the - payloads 1..P, each in order of appearance. Numbering is over the whole - log, so a worker step logged before its hand-off keeps its request or - payload and the model rejects it; one never handed off gets 0 (no item).""" + """The deadline service's hand-off and the disposal pool against + Disposal.tla. The leading `Pool` event gives the idle workers when + recording started (InitIdle = 1..K; a worker that appears without + `Begin` is one of them). Requests (Expire), payloads (Defer drop), other + items (Defer other) and started workers (Begin) are numbered in order of + appearance, over the whole log, so an event out of order keeps its item + and the model rejects it; an item never handed off gets kind "none".""" for model in ("Disposal.tla", "TraceDisposal.tla"): shutil.copy(os.path.join(HERE, model), workdir) - reqs, pays, records = {}, {}, [] - for e in trace["events"]: - a, raw = e["a"], int(e["id"]) - if a == "Expire": - if raw in reqs: + events = list(trace["events"]) + if not events or events[0]["a"] != "Pool": + raise ValueError("a disposal trace starts with the pool's state (Pool)") + pool = events.pop(0) + idle, started = int(pool["w"]), int(pool["r"]) + if idle != started or pool["x"]: + raise ValueError(f"recording must start with the pool quiescent: {pool}") + reqs, items, counts = {}, {}, {"drop": 0, "other": 0} + for e in events: + if e["a"] == "Expire": + r = int(e["r"]) + if r in reqs: raise ValueError(f"a slot expired twice: {e}") - reqs[raw] = len(reqs) + 1 - elif a == "Release": - if raw in pays: - raise ValueError(f"a payload released twice: {e}") - pays[raw] = len(pays) + 1 - for e in trace["events"]: - a, raw = e["a"], int(e["id"]) - if a in {"Expire", "Wake"}: - ident = reqs.get(raw, 0) - elif a in {"Release", "Drop"}: - ident = pays.get(raw, 0) + reqs[r] = len(reqs) + 1 + if int(e["id"]): + items[int(e["id"])] = ("wake", reqs[r]) + elif e["a"] == "Defer": + kind = e["x"] + counts[kind] += 1 + items[int(e["id"])] = (kind, counts[kind]) + begun = {int(e["w"]) for e in events if e["a"] == "Begin"} + workers, preexisting = {}, 0 + for e in events: + w = int(e.get("w", 0)) + if not w or w in workers: + continue + if w in begun: + workers[w] = None # numbered after the pre-existing ones else: - ident = 0 - records.append("[a |-> {}, id |-> {}, x |-> {}]".format( - tla_string(a), ident, tla_string(e["x"]))) + preexisting += 1 + if preexisting > idle: + raise ValueError(f"more pre-existing workers than were idle: {e}") + workers[w] = preexisting + following = idle + for w in workers: + if workers[w] is None: + following += 1 + workers[w] = following + records = [] + for e in events: + a = e["a"] + if a == "Expire": + k, n = ("wake" if e["x"] == "wake" else "none"), reqs[int(e["r"])] + elif a in {"Defer", "Take", "Run", "Settle", "Done"}: + k, n = items.get(int(e["id"]), ("none", 0)) + else: + k, n = "none", 0 + records.append("[a |-> {}, w |-> {}, k |-> {}, n |-> {}, x |-> {}]".format( + tla_string(a), workers.get(int(e.get("w", 0)), 0), tla_string(k), n, + tla_string(e["x"]))) if not records: raise ValueError("a disposal trace needs at least one event") with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: out.write("---- MODULE TraceRun ----\nEXTENDS TraceDisposal\n" - "TraceBlocking == {}\nTraceLog == <<\n " + f"TraceBlocking == {{}}\nTraceInitIdle == 1..{idle}\nTraceLog == <<\n " + ",\n ".join(records) + "\n>>\n====\n") with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: - # Deadlines are chosen by TLC; one tick makes every slot due. One - # worker per item, plus one, so no recorded run waits for a worker. + # Deadlines are chosen by TLC; one tick makes every slot due. The + # worker cap is the code's (MAX_WORKERS). out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" - f" N = {len(reqs)}\n P = {len(pays)}\n MaxTime = 1\n" - f" Workers = {len(reqs) + len(pays) + 1}\n" - " Blocking <- TraceBlocking\n Fault = \"None\"\n" - " Trace <- TraceLog\n" + f" N = {len(reqs)}\n P = {counts['drop']}\n O = {counts['other']}\n" + " MaxTime = 1\n Workers = 64\n" + " Blocking <- TraceBlocking\n InitIdle <- TraceInitIdle\n" + " Fault = \"None\"\n Trace <- TraceLog\n" "INVARIANT Safety\nINVARIANT TraceIncomplete\n") From a6a25363bdf2f284c353d2197654f061e96d6963 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 20:52:57 -0300 Subject: [PATCH 12/13] docs: name what still runs on the deadline thread; pre-exit drops; disposal trace docs --- CHANGELOG.md | 13 +++++--- concurrency/src/reply/deadline.rs | 6 ++-- docs/design/deferred-replies.md | 52 ++++++++++++++++++++--------- docs/design/semantics.md | 33 +++++++++++-------- verification/tla/README.md | 55 +++++++++++++++++++------------ 5 files changed, 101 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50fb8ae9a..880f55fe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,10 +90,15 @@ on requests that do not expire. New regression tests (tasks and threads), a Loom case for the handed-off wake-up, an `InlineDisposal` negative control in `DeferredReplies.tla`, and a new trace-validated model, - `Disposal.tla` (expiry and hand-off, per-item queueing, worker wake-ups - after the transition, payload drops held until the death point), with the - `disposal_trace` scenarios, a `disposal.item` failpoint and four negative - controls including the shared-batch bug. + `Disposal.tla` (expiry and hand-off, per-item queueing, the pool's start + rule, idle count and cap, worker wake-ups after the transition, two-step + payload drops held until the death point), validated against traces that + record every hand-off and worker step (`disposal_trace`, with + `disposal.item`, `disposal.keep_alive` and `disposal.batch_serial` + failpoints; the last records a control trace from code with the + shared-batch bug, which must be rejected) and five negative controls. + Empty payloads are no longer handed off, and a due entry is dropped in + place unless it may hold an unread reply. - Regenerate the 11 planted-bug patches (04, 10-17, 19, 20) that no longer applied, each planting the same fault where the code now lives, and check in the required `model-sync` job that every planted patch still applies. diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 0b83edd6b..5f6636f31 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -12,9 +12,9 @@ //! due entries even when no caller polls, so credits return on time: //! //! - real time: one process-wide service thread with its own timer, so -//! per-test Tokio runtimes cannot break it. It runs no user code: what an -//! expiry releases (wakers, payload destructors) goes to a disposal worker -//! (`super::disposal`); +//! per-test Tokio runtimes cannot break it. What an expiry releases +//! (wakers, payload destructors) goes to a disposal worker +//! (`super::disposal`, which lists what still runs inline); //! - under `spawned_sim`: a task of the simulated runtime per actor heap, so //! clock domains never mix and virtual time drives it. diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index f3c6efced..66625905c 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -298,20 +298,25 @@ handles a call even after the caller timed out, and the late reply is dropped. during the visit and handed to the disposal workers (`reply/disposal.rs`): the caller's waker and the token's watchers, payloads routed to the owner's failure sink (a held or refused job result, a discarded emit, - `done` and its captures), the due entries, and the last references to - jobs, runs and the registry (a reference that is not the last is dropped - in place: it runs no user code). Each item is queued on its own, so a - destructor or waker that blocks holds only the worker running it: no - other expiry, credit return or wake-up waits for it, of another actor or - of the same one (two callers of one actor expiring in one visit are woken - independently). Disposal workers form an elastic pool: when queued items + `done` and its captures), a due slot that may hold an unread reply, and + the last references to jobs, runs and the registry (a reference that is + not the last, or an entry with nothing of the user's left, is dropped in + place; empty payloads, the usual case, are not handed off at all). Each + item is queued on its own, so a destructor or waker that blocks holds only + the worker running it: while fewer than 64 callbacks are blocked at once, + no other expiry, credit return or wake-up waits for it, of another actor + or of the same one (two callers of one actor expiring in one visit are + woken independently). Disposal workers form an elastic pool: when queued items outnumber idle workers one more starts (one start at a time, up to 64; idle ones exit after 10 s), and each worker re-checks when it takes an item. Reaching the cap logs `disposal_workers_capped` once. Modelled in - `verification/tla/Disposal.tla` (trace-validated against - `concurrency/tests/disposal_trace.rs`): no wake-up before the transition - and credit return, no payload drop while held, and no expiry or wake-up - waiting for a blocked item; its controls include the shared-batch bug. + `verification/tla/Disposal.tla`, including the pool's start rule, idle + count and cap (trace-validated against `concurrency/tests/disposal_trace.rs`, + which records every hand-off and every worker step): no wake-up before the + transition and credit return, no payload drop decided while held, and no + expiry or wake-up waiting for a blocked item below the cap; its controls + include the shared-batch bug, as a model fault and as a trace recorded + from the code with that bug switched on (`disposal.batch_serial`). - *Why a pool and not the owner.* Deferring disposal to the owning actor would tie a caller's wake-up to the owner's next turn (a wedged owner would hang its callers past their deadline), and deferring it to the @@ -333,9 +338,23 @@ handles a call even after the caller timed out, and the late reply is dropped. - *Cost.* None on a request or reply that does not expire: the hand-off is in the service's own path, and the shared helpers test a thread-local only when they already have a waker or a payload to dispose of. - - *Remaining assumption.* The owner's own executor wake (a runtime waker or - condvar) still runs on the service thread; it is framework code. With - 64 callbacks blocked at once, later items queue behind them. + - *Pre-exit drops.* A worker decides a payload's fate under the hold + lock (`DropFailures::settle`) and runs the destructor after releasing + it, like an inline discard: a drop decided just before the owner's exit + began may still be running while the exit proceeds. It is pre-exit work + (§5); the exit never waits for it, since such a destructor could itself + wait for this actor's callers. The guarantee is that no drop is decided + after the hold began (the model's `NoDropWhileHeld`, on the decision). + - *What still runs on the service thread.* The owner's own executor wake + (enqueueing a job's message: a runtime waker or condvar), dropping (not + waking) a job's stored wakers when its last reference goes, the `tracing` + subscriber for a contained panic or the cap warning, a panic payload's + destructor (`dispose_panic`), and, only if no disposal thread exists and + none can be spawned, the deferred work itself. If an unwinding job + notification drops its run there, that drop is inline too. + - *Limits.* With 64 callbacks blocked at once, later items queue behind + them. A burst of quick items can start workers that then sit idle (one + start in flight at a time; idle workers exit after 10 s). Ordinary request/Response runtime timers and send_after/send_interval drivers are separate (see semantics.md, section 10). - **Simulation.** Under `spawned_sim` the service is a task of the simulated @@ -1167,8 +1186,9 @@ semantics above. service thread (not a task on the `ActorSystem` control runtime); a dead owner is forgotten at its registry close. Under `spawned_sim` each owner's heap is served by a task of the simulated runtime, so clock domains never - mix. The thread runs no user code: wakers and payload destructors an expiry - releases go to an elastic disposal pool (§2, "Callback progress"). + mix. The wakers and payload destructors an expiry releases go to an + elastic disposal pool, not the thread (§2, "Callback progress", which + lists what still runs inline). - **Cleanup deadline.** Armed when exit begins: the supervisor's `shutdown_timeout` for a supervised child, the remaining system grace during `ActorSystem::shutdown`, otherwise `DEFAULT_STOP_DEADLINE` (5 s). diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 446f296d3..8711e3da3 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -160,12 +160,13 @@ the expiry runs is delivered (`Delivery::Delivered`, and the caller gets the value), as an OTP reply already in the caller's mailbox when its `after` clause would fire is received. Timer resolution and when the runtime or deadline service can process expiry determine that window; this is not a -hard real-time bound. The deadline service runs no user code: the wakers and -payload destructors an expiry releases run on disposal workers, one item at a -time, so a blocking destructor or waker (of another actor, or of another -caller of the same actor) does not delay an expiry, its deferral-credit -return or its caller's wake-up -([`deadline_service`][deadline_service]). +hard real-time bound. The deadline service hands the wakers and payload +destructors an expiry releases to disposal workers, one item at a time, so +while fewer than 64 such callbacks are blocked at once, a blocking +destructor or waker (of another actor, or of another caller of the same +actor) does not delay an expiry, its deferral-credit return or its caller's +wake-up ([`deadline_service`][deadline_service]). What still runs on the +service thread is listed in deferred-replies.md §2. Which clock expires a waiting caller is part of the contract: `request`, `request_with_timeout` and `Response::from(rx)` arm the runtime's timer, so they follow paused Tokio time; a `ReplyReceiver` awaited directly @@ -768,14 +769,18 @@ stopped after `Shutdown`), - **Preemption.** A running handler, a blocking call or a non-returning hook is never interrupted; stop, shutdown deadlines and cleanup timeouts only signal and report. Rust cannot kill a thread. -- **Deadline-service isolation past 64 blocked callbacks.** The shared - deadline-service thread runs no user code (section 3): wakers and - payload destructors an expiry releases go to a pool of at most 64 disposal - workers. With that many callbacks blocked at once, later disposals (and so - their callers' wake-ups) queue behind them; expiry and deferral-credit - return do not. A discarded scheduled emit's per-job emit credit returns - with its message's disposal, so it can queue too. No callback is - preempted. +- **Deadline-service isolation past 64 blocked callbacks, and its inline + remainder.** Wakers and payload destructors an expiry releases go to a + pool of at most 64 disposal workers (section 3). With that many callbacks + blocked at once, later disposals (and so their callers' wake-ups) queue + behind them; expiry and deferral-credit return do not. A discarded + scheduled emit's per-job emit credit returns with its message's disposal, + so it can queue too. A few things still run on the service thread: the + owner's executor wake, dropping a job's stored wakers, `tracing` + subscriber calls, a panic payload's destructor, and the work itself if no + disposal thread can be spawned (deferred-replies.md §2). A payload drop + decided just before the owner's exit may still run during it (pre-exit + work; the exit never waits for it). No callback is preempted. - **Bounded memory.** Mailboxes are unbounded (D5); there are no per-actor memory limits (rank 18a). - **Deadlock detection.** A request cycle, or a handler requesting its own diff --git a/verification/tla/README.md b/verification/tla/README.md index 0704431ca..6b0afb50d 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -17,7 +17,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `TraceCoordinator.tla` | Trace validation of `Coordinator.tla`: replays the shutdown coordinator events recorded from real `ActorSystem::shutdown` runs (`coordinator_trace`). | | `NestedSupervision.tla`, `MCNested.tla/.cfg` | Dynamic subtree ownership and shutdown delivery; parent scope completion waits for descendant completion. Bounded three-node/two-generation model with fair cleanup. | | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with exit-pending drops (unwound or dropped while exiting, answered at death with the final reason), a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Fourteen named negative controls. | -| `Disposal.tla`, `MCDisposal{,Drop}.tla/.cfg`, `TraceDisposal.tla` | The deadline service's hand-off of user code to disposal workers (companion of `DeferredReplies.tla`, which takes the service's expiry as one step): per-entry `Expire` (transition and credit return on the service thread), payload `Release`, `Submit` one queue entry per item, an elastic worker pool (start while queued entries outnumber idle workers, take, run, retire), wake-ups run by workers, payload drops that re-check the owner's hold (held until `Death`), and items that never return (`Blocking`). Invariants: credit with the transition, no wake-up before the transition, no payload drop while held, timely expiry and timely wake-up whatever blocks; liveness: every non-blocking wake-up runs, every deadline is serviced. Four named negative controls; trace-validated. See [Disposal](#disposal-trace-validation). | +| `Disposal.tla`, `MCDisposal{,Drop,Capped}.tla/.cfg`, `TraceDisposal.tla` | The deadline service's hand-off of user code to disposal workers (companion of `DeferredReplies.tla`, which takes the service's expiry as one step): per-entry `Expire` (transition and credit return on the service thread), payload `Release` and `Other` items, `Submit` one queue entry per item, and the pool as `disposal.rs` runs it: worker states (active, idle, holding an entry, running an item), the idle count, the start rule after `Submit` and every `Take` (one start in flight, up to the cap), `Wait`/`Woke`/`Retire`; wake-ups run by workers; payload drops in two steps (`Settle` decides under the hold lock: held until `Death`, or dropped; then the destructor until `Done`); items that never return (`Blocking`). Invariants: credit with the transition, no wake-up before the transition, no payload drop decided while held, the cap, timely expiry and timely wake-up whatever blocks (`MCDisposalCapped`, one worker: late only while every worker is stuck); liveness: every non-blocking wake-up runs, every deadline is serviced. Five named negative controls; trace-validated, including a trace recorded from code with the shared-batch bug. See [Disposal](#disposal-trace-validation). | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits; `MCDeferredJobsHeld` the one with delayed emits (`emit_at`) and held results (completion, held notification, scheduled emits and credit ownership as separate steps). Twelve named negative controls; `MCDeferredJobsEmitRace` checks retained emit admission at held-result release (306 states). | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Nineteen negative-control faults (19 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point, a parent exit reaching `exit_received`, an exit signal past the sender's death point (`NoSignalFence`) and publication before the queued requests were answered (`PublishBeforeDischarge`). Death is two steps, as in `actor_cleanup.rs`: the death point (`Die`/`RuntimeDrop`: signal fence, seal, table closed) and `Discharge` (queued and exit-pending requests answered), with other actors' steps in between; `Publish` waits for `Discharge`. Per PR: `MCLifecycle` (10.0M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance); manual pre-release: `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | | `Start.tla`, `MCStart{,Names,Cap,Timeout,Continue,Scope,Live,Deep}.tla/.cfg` | Unified start API, 0.9 design (approved after three review rounds) ([design](../../docs/design/unified-start.md), not implemented; TLC-only in `model-map.toml` until trace validation lands with the implementation): `spawn`, `start`, `start_link`, `start_monitor`; placement inheritance; an atomic stage-1 commit with the owner gate; an immutable startup commitment and a waiter; death as D, release, then publication, for the child and the parent; the final reason fixed at D; fallible init with held-signal precedence; the continuation; S8 scope ownership with the polite Shutdown and the scope deadline. 40 negative controls (37 safety, 3 liveness); see [Unified start](#unified-start-09-design). Per PR: `MCStart`, `MCStartNames`, `MCStartCap`, `MCStartTimeout`, `MCStartContinue`, `MCStartScope`, `MCStartLive` (CI leg `start`); weekly: `MCStartDeep`. | @@ -715,6 +715,7 @@ exactly the property it targets: | `Disposal` (`MCDisposal`) | `WakeBeforeTransition` (the wake-up handed off before the transition commits) | `WakeAfterTransition` | | `Disposal` (`MCDisposal`) | `DropWhileHeld` (the worker drops a payload without re-checking the owner's hold) | `NoDropWhileHeld` | | `Disposal` (`MCDisposal`) | `BatchSerial` (a visit's batch is one queue entry run serially; the shared-batch bug of PR #282's first revision) | `TimelyWake` | +| `Disposal` (`MCDisposal`) | `NoStartOnTake` (the start rule checked only at `Submit`: a queued item stranded behind blocked workers) | `TimelyWake` | | `DeferredJobs` | `NoInstanceCheck` | `NoLateResult` | | `DeferredJobs` | `NoCancelFence` | `NoStaleAfterCancel` | | `DeferredJobs` | `DoubleTerminal` | `AtMostOneTerminal` | @@ -803,26 +804,38 @@ turn, and a `Stopped` discharge of the panicking turn's token before its ### Disposal trace validation -While `lifecycle_trace::record_disposal()` is on, the code records the -`Disposal.tla` actions at their linearization points: `Expire` (on the -service thread, after `Slot::expire_collect`'s transition and credit return; -`x` says whether a wake-up was handed off), `Release` (a payload deferred -from the service thread, in one section with `DropFailures`' hold check), -`Submit` (`deadline.rs`, the visit's work handed to the pool), `Wake` (a -worker runs a handed-off wake-up), `Drop` (a worker settles a payload, in -one section with the hold check; `x`: `held` or `dropped`), `Exit` -(`DropFailures::hold`) and `Death` (`DropFailures::release`). Ids are fresh -per `Expire`/`Release`; the checker numbers requests and payloads. The -workers' start, take and retire, and the clock, are silent; `Commit` (the -`WakeBeforeTransition` fault device) has no event. Recording is -process-wide, so `concurrency/tests/disposal_trace.rs` runs its scenarios -one at a time: expired callers woken by workers (two waiting, one never -polled), payloads released by a job deadline and a discarded scheduled emit -and dropped by workers, and a payload handed off before the owner's exit -whose worker is delayed (`disposal.item` failpoint) until `stopped()` runs: -it is held and dropped at the death point. `negative_checks.py` requires -three edited traces to be rejected (a wake-up before its expiry, a payload -drop before the hand-off, and a handed-off payload dropped while held). +While `lifecycle_trace::record_disposal()` is on, the code records every +`Disposal.tla` action at its linearization point. The first event, `Pool`, +is the pool's state when recording starts (the checker requires it +quiescent and makes its idle workers the model's `InitIdle`). On the +service thread: `Expire` (after `Slot::expire_collect`'s transition and +credit return; `x` says whether a wake-up was handed off), `Defer` (every +other hand-off: a payload, `x: drop`, in one section with `DropFailures`' +hold check; anything else, `x: other`: a last reference, emitter wakers, a +slot with an unread reply) and `Submit` (the items queued, `x: spawn` if +the start rule started a worker). In the pool, under its lock: `Begin`, +`Take` (with the start decision), `Wait`, `Woke` and `Retire`; outside it: +`Run`, `Settle` (a payload's hold check, `x: held` or `dropped`) and `Done`. +The owner: `Exit` (`DropFailures::hold`) and `Death` (`DropFailures::release`). +Workers and items carry ids; the checker numbers them. Only the clock is +silent; `Commit` (the `WakeBeforeTransition` fault device) has no event. +Because every worker step names its worker and item, a worker running an +item it never took (serial execution of a batch) is rejected, and because +the start decisions are recorded and the model's start rule must agree, a +start that the rule requires but the code skips (stranding) is rejected. + +Recording is process-wide, so `concurrency/tests/disposal_trace.rs` runs its +scenarios one at a time: expired callers woken by workers (two waiting, one +never polled), payloads released by a job deadline and a discarded scheduled +emit and dropped by workers, a payload handed off before the owner's exit +whose worker is delayed (`disposal.item` failpoint) until `stopped()` runs +(held, and dropped at the death point), and idle workers retiring +(`disposal.keep_alive`) and new ones starting. A control is recorded from +the code with the shared-batch bug switched back on (`disposal.batch_serial`) +into `controls/`; `negative_checks.py` requires it to be rejected, along with +five edited traces (a wake-up before its expiry, a payload run before the +hand-off, a handed-off payload dropped while held, a required worker start +not taken, and a worker running an item it never took). ## Unified start (0.9 design) From ec43dde61a9ceb380c819dd7d20525399a0af326 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 21:51:31 -0300 Subject: [PATCH 13/13] perf(disposal): decide a due entry's hand-off without new per-reply-type code Fold the decision into Expirable::expire (it returns whether the last drop could run user code) and read only the slot word in one non-generic function, so the generic Slot code instantiated in callers' crates is unchanged. An extra per-type method had shifted LLVM's inlining of the tasks receive loop (+5.5% on async_ct_send). --- concurrency/src/reply/deadline.rs | 16 ++++----- concurrency/src/reply/jobs.rs | 17 ++++++--- concurrency/src/reply/registry.rs | 58 ++++++++++++++++++++----------- concurrency/src/tasks/jobs.rs | 2 +- concurrency/src/threads/jobs.rs | 2 +- 5 files changed, 60 insertions(+), 35 deletions(-) diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 5f6636f31..21abd4bac 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -27,14 +27,14 @@ use std::sync::{Arc, Weak}; pub(crate) trait Expirable: Send + Sync { /// The current deadline while the item is pending, else `None`. fn pending_deadline(&self) -> Option; - /// The deadline passed (`now >= deadline`): expire the item. - fn expire(&self, now: u64); - /// Drop the service's reference after the visit. It may be the last: - /// an item whose last drop can run user code hands that drop off - /// ([`super::disposal`]); others drop in place, with no hand-off. - fn dispose(self: Arc) { - drop(self); - } + /// Service a popped entry: if `due` (its deadline passed, `now >= + /// deadline`), expire it. Returns whether dropping the last reference + /// could still run user code (an unread reply): the service hands such a + /// drop off ([`super::disposal`]) and drops the others in place, with no + /// hand-off. Job entries route their user payloads through the owner's + /// drop sink (deferred there), so they return `false`. One method, not + /// two: each is instantiated per reply type in the caller's crate. + fn expire(&self, now: u64, due: bool) -> bool; } struct Entry { diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 5a3e67cc8..4bea803c1 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -1012,11 +1012,14 @@ impl Expirable for JobDeadline { fn pending_deadline(&self) -> Option { self.state.core.is_pending().then_some(self.at) } - fn expire(&self, _now: u64) { - let notify = lock(&self.notify).take(); - if let Some(notify) = notify { - notify(); + fn expire(&self, _now: u64, due: bool) -> bool { + if due { + let notify = lock(&self.notify).take(); + if let Some(notify) = notify { + notify(); + } } + false } } @@ -1131,13 +1134,17 @@ impl Expirable for ScheduledEmit { fn pending_deadline(&self) -> Option { lock(&self.send).is_some().then_some(self.at) } - fn expire(&self, _now: u64) { + fn expire(&self, _now: u64, due: bool) -> bool { + if !due { + return false; + } fail_point!("job.emit_due"); if let Some(job) = self.job.upgrade() { self.settle(&job, true); // Possibly the last reference (its other scheduled emits). super::disposal::defer_drop_arc(job); } + false } } diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 73931faa5..3b5d5a43a 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -54,26 +54,21 @@ pub(crate) trait Closeable: Send + Sync { /// with the wake-up its close collected, if it closed it. type PendingWake = (Arc, Option); -impl Expirable for Slot { - /// Only a `REPLIED` slot can hold a value (an unread reply, user code - /// when destroyed), so only its drop is handed off, and that of a slot - /// not yet terminal (a reply still `WRITING` may store one). Any other - /// terminal state is final and holds no value: dropped in place. - fn dispose(self: Arc) { - if !self.is_terminal() - || Terminal::of(super::slot::state(self.word())) == Some(Terminal::Replied) - { - super::disposal::defer_drop(self); - } - } +impl Expirable for Slot { fn pending_deadline(&self) -> Option { let deadline = self.deadline(); (!self.is_terminal() && deadline != NEVER).then_some(deadline) } /// The transition and credit return happen here; a parked blocking /// receiver is unparked here too (no user code); the wakers are handed - /// to a disposal worker when this runs on the deadline thread. - fn expire(&self, now: u64) { + /// to a disposal worker when this runs on the deadline thread. Only a + /// `REPLIED` slot can hold a value (an unread reply, user code when + /// destroyed), or one not yet terminal (a reply still `WRITING` may + /// store one); any other terminal state is final and holds none. + fn expire(&self, now: u64, due: bool) -> bool { + if !due { + return may_hold_value(self); + } if let Some(mut wake) = self.expire_collect(now) { wake.notify_parked(); // Disposal.tla `Expire`: after the transition and its credit @@ -94,9 +89,26 @@ impl Expirable for Slot { }; crate::lifecycle_trace::disposal("Expire", 0, item, request, kind); } + may_hold_value(self) + } +} + +/// Whether dropping `slot`'s last reference could run user code: it is +/// `REPLIED` (an unread value) or not yet terminal. +/// Not generic (one copy, in this crate): only the slot's word is read. +#[inline(never)] +fn may_hold_word(word: u64) -> bool { + let state = super::slot::state(word); + match Terminal::of(state) { + None => true, + Some(terminal) => terminal == Terminal::Replied, } } +fn may_hold_value(slot: &Slot) -> bool { + may_hold_word(slot.word()) +} + impl Closeable for Slot { fn close_collect(&self) -> Option { Slot::close_collect(self) @@ -336,11 +348,14 @@ impl Registry { let due = lock(&self.lists).deadlines.due(now); for item in due { contain_entry(|| { - if item.pending_deadline().is_some_and(|d| d <= now) { - item.expire(now); + let due = item.pending_deadline().is_some_and(|d| d <= now); + // Possibly the last reference: handed off only if dropping + // it could run user code (see `Expirable::expire`). + if item.expire(now, due) { + super::disposal::defer_drop(item); + } else { + drop(item); } - // Possibly the last reference (see `Expirable::dispose`). - item.dispose(); }); } lock(&self.lists).deadlines.earliest() @@ -876,8 +891,11 @@ mod tests { fn pending_deadline(&self) -> Option { Some(self.0) } - fn expire(&self, _: u64) { - let _ = self.1.send(()); + fn expire(&self, _: u64, due: bool) -> bool { + if due { + let _ = self.1.send(()); + } + false } } assert!(spawned_rt::tasks::Handle::try_current().is_err()); diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index e7962e1d6..fa83ae99a 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -728,7 +728,7 @@ impl Context { // The run owns it; the heap only refers to it. let _ = run.deadline.set(deadline.clone()); if at <= now_ticks() { - deadline.expire(at); + deadline.expire(at, true); } else { let item: Weak = Arc::downgrade(&deadline) as Weak; self.shared.replies.track(at, item); diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index ca7e5da37..19a85441f 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -499,7 +499,7 @@ impl Context { // The run owns it; the heap only refers to it. let _ = run.deadline.set(deadline.clone()); if at <= now_ticks() { - deadline.expire(at); + deadline.expire(at, true); } else { let item: Weak = Arc::downgrade(&deadline) as Weak; self.shared.replies.track(at, item);