diff --git a/CHANGELOG.md b/CHANGELOG.md index d11e25f2..19b8f5e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,10 @@ arms); the model releases a group slot at scope completion, as the code does. Not yet: scope ownership (S8), supervisor `ignore` and `ChildSpec::options`. +- Test kit: `Mock::on_started`, `start_delay`, `drop_flag` (unified start + design 10) to test code that starts a dependency. Ten unified-start + planted bugs (`verification/planted/patches/25-34`); the runner's trace + layer also records the lifecycle and start traces. - `model-sync`: a design-only TLA+ model (`design_only = true`: Start, Introspection, DynamicSupervisor) stays allowlisted as TLC-only only while diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index fa5ddf1c..12785338 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -196,6 +196,12 @@ pub const SITES: &[Site] = &[ the child is killed before started() (`KilledEarly`) and its \ start answers `InitExited { Killed }`.", }, + Site { + name: "start.init_timer_fire", + has_error: false, + description: "The init timer expired and is about to reach the startup cell. \ + Delay lets the child commit ready first: the timer must lose.", + }, Site { name: "actor.thread_spawn", has_error: true, diff --git a/concurrency/src/start.rs b/concurrency/src/start.rs index 5554f45d..8c911bde 100644 --- a/concurrency/src/start.rs +++ b/concurrency/src/start.rs @@ -1094,6 +1094,7 @@ pub(crate) fn spawn_init_timer_async( let sleep = std::pin::pin!(spawned_rt::tasks::sleep(after)); let cancelled = std::pin::pin!(theirs.cancelled()); if let futures::future::Either::Left(_) = futures::future::select(sleep, cancelled).await { + fail_point!("start.init_timer_fire", async); fire_init_timer(&cell, &handle); } })); @@ -1120,7 +1121,13 @@ pub(crate) fn spawn_init_timer(cell: &Arc, handle: ChildHandle, aft .queue .lock() .unwrap_or_else(|p| p.into_inner()) - .insert(key, Box::new(move || fire_init_timer(&cell, &handle))); + .insert( + key, + Box::new(move || { + fail_point!("start.init_timer_fire"); + fire_init_timer(&cell, &handle) + }), + ); timers.wake.notify_one(); cancel.on_cancel(Box::new(move || { let removed = timers diff --git a/concurrency/tests/start_races.rs b/concurrency/tests/start_races.rs new file mode 100644 index 00000000..563a4b33 --- /dev/null +++ b/concurrency/tests/start_races.rs @@ -0,0 +1,325 @@ +//! Targeted start races (unified start design 16.3): each pins one +//! interleaving of the startup decision with an outside event, by explicit +//! waits (gates the actors open and close) or a failpoint, never by timing. +//! `sim_test`s also run under the simulator (`--cfg spawned_sim`), where the +//! schedule explorer varies the remaining interleavings. +#![cfg(feature = "async")] +#![cfg_attr(spawned_sim, allow(unused_imports, dead_code))] + +use spawned_concurrency::{ + message::Message, + system::ActorSystem, + tasks::{Actor, ActorStart, Context, Handler, StartOptions}, + ExitReason, PendingReply, StartError, +}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::sync::oneshot; + +const LIMIT: Duration = Duration::from_secs(10); + +struct Ping; +impl Message for Ping { + type Result = (); +} + +/// A child whose `started()` reports that it began, then waits on a gate. +struct Gated { + entered: Option>, + gate: Option>, + trap: bool, + started: Arc, +} + +impl Gated { + fn new(trap: bool) -> (Self, oneshot::Receiver<()>, oneshot::Sender<()>) { + let (entered_tx, entered) = oneshot::channel(); + let (release, gate) = oneshot::channel(); + let child = Self { + entered: Some(entered_tx), + gate: Some(gate), + trap, + started: Arc::default(), + }; + (child, entered, release) + } +} + +impl Actor for Gated { + async fn started(&mut self, ctx: &Context) { + self.started.fetch_add(1, Ordering::SeqCst); + if self.trap { + ctx.trap_exit(true); + } + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(gate) = self.gate.take() { + let _ = gate.await; + } + } +} + +impl Handler for Gated { + async fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +/// Planted bug 33: a kill outranks a held untrapped signal (design 2.3 +/// rows 2 and 3): a non-trapping child that gets an exit signal and then a +/// kill while `started()` runs fails its start `Killed`. +#[spawned_macros::sim_test] +async fn kill_outranks_a_held_signal_during_started() { + let (child, entered, release) = Gated::new(false); + let starting = child.start(); + let handle = starting.actor_ref().unwrap().child_handle(); + entered.await.unwrap(); + handle.exit(ExitReason::Shutdown(None)); + handle.kill(); + release.send(()).unwrap(); + let result = starting.await; + assert!( + matches!( + result, + Err(StartError::InitExited { + reason: ExitReason::Killed, + pending: None + }) + ), + "{result:?}" + ); + assert_eq!(handle.wait_exit_async().await, ExitReason::Killed); +} + +/// A parent that begins its own exit, then tries a `start_link` in the same +/// turn. +struct StopsThenLinks { + child_started: Arc, +} +impl Actor for StopsThenLinks {} + +struct StopThenLink; +impl Message for StopThenLink { + type Result = Result<(), StartError>; +} +impl Handler for StopsThenLinks { + async fn handle( + &mut self, + _: StopThenLink, + reply: PendingReply<'_, Result<(), StartError>>, + ctx: &Context, + ) { + ctx.stop(); + let (mut child, _entered, _release) = Gated::new(false); + child.started = self.child_started.clone(); + reply.reply(child.start_link(ctx).await.map(|_| ())); + } +} + +/// Planted bug 32: a parent whose exit has begun (its own `ctx.stop()`, +/// still in its turn: requested, not dead) refuses `start_link` at stage 1 +/// (`ParentExited`); no child runs. +#[spawned_macros::sim_test] +async fn start_link_through_an_exiting_parent_is_refused() { + let child_started = Arc::new(AtomicUsize::new(0)); + let parent = StopsThenLinks { + child_started: child_started.clone(), + } + .start() + .await + .unwrap() + .started() + .unwrap(); + let result = parent.request(StopThenLink).await.unwrap(); + assert!( + matches!(result, Err(StartError::ParentExited)), + "{result:?}" + ); + parent.wait_exit().await; + assert_eq!(child_started.load(Ordering::SeqCst), 0); +} + +/// A root whose `stopped()` reports and then waits on a gate. +struct SlowStop { + entered: Mutex>>, + gate: Mutex>>, +} +impl Actor for SlowStop { + async fn started(&mut self, ctx: &Context) { + // Trapping: the system's Shutdown runs stopped() (E3). + ctx.trap_exit(true); + } + async fn stopped(&mut self, _: &Context) { + let entered = self.entered.lock().unwrap().take(); + if let Some(entered) = entered { + let _ = entered.send(()); + } + let gate = self.gate.lock().unwrap().take(); + if let Some(gate) = gate { + let _ = gate.await; + } + } +} + +/// Planted bug 34: row 9. The group's admission closes (system shutdown +/// began) while a managed child is in `started()` and before its own +/// Shutdown reaches it (the system stops roots newest first; a newer root's +/// slow `stopped()` holds the coordinator): the child's ready return +/// commits nothing and its start fails `Shutdown`. +#[spawned_macros::sim_test] +async fn ready_after_the_group_closed_fails_the_start() { + let system = Arc::new(ActorSystem::new().unwrap()); + let group = system.async_group("start-races-closing", 4).unwrap(); + let (child, child_entered, release_child) = Gated::new(true); + let starting = child.with_options(&group).start(); + child_entered.await.unwrap(); + let (stop_entered_tx, stop_entered) = oneshot::channel(); + let (release_stop, stop_gate) = oneshot::channel(); + let newer = SlowStop { + entered: Mutex::new(Some(stop_entered_tx)), + gate: Mutex::new(Some(stop_gate)), + } + .with_options(&group) + .start() + .await + .unwrap() + .started() + .unwrap(); + let shutdown = { + let system = system.clone(); + spawned_rt::tasks::spawn(async move { system.shutdown(LIMIT).await.complete }) + }; + // The newer root is in stopped(): admission is closed, the child is not + // signalled yet. + stop_entered.await.unwrap(); + release_child.send(()).unwrap(); + let result = starting.await; + assert!( + matches!( + result, + Err(StartError::InitExited { + reason: ExitReason::Shutdown(None), + pending: None + }) + ), + "{result:?}" + ); + release_stop.send(()).unwrap(); + newer.wait_exit().await; + assert!(shutdown.await.unwrap()); +} + +/// A child whose state drop is slow and observable. +struct SlowDrop { + dropped: Arc, +} +impl Actor for SlowDrop { + async fn started(&mut self, _: &Context) -> Result<(), String> { + Err("no config".into()) + } +} +impl Drop for SlowDrop { + fn drop(&mut self) { + std::thread::sleep(Duration::from_millis(50)); + self.dropped.store(true, Ordering::SeqCst); + } +} + +/// Planted bug 28: a failed start is answered only after the child's +/// publication: its state is already dropped (design 5.5, +/// `FailureAfterPublication`). Real threads: the waiter runs on another +/// worker while the child's slow destructor runs (on one thread the drop +/// always finishes before the waiter is polled). +#[cfg(not(spawned_sim))] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_failed_start_answers_after_the_state_drop() { + let dropped = Arc::new(AtomicBool::new(false)); + let result = SlowDrop { + dropped: dropped.clone(), + } + .start() + .await; + assert!( + matches!(result, Err(StartError::InitFailed { pending: None, .. })), + "{result:?}" + ); + assert!( + dropped.load(Ordering::SeqCst), + "answered before the state drop" + ); +} + +/// A child whose `started()` takes exactly `after`. +struct TakesExactly(Duration); +impl Actor for TakesExactly { + async fn started(&mut self, _: &Context) { + spawned_rt::tasks::sleep(self.0).await; + } +} +impl Handler for TakesExactly { + async fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +/// Planted bug 25 (simulator race): `started()` returns at the init +/// timer's own instant, so the schedule explorer orders the ready commit +/// and the timer's expiry both ways. Whichever wins, a child answered +/// `Started` is never killed by the timer (`TimeoutNeverKillsReady`). +#[spawned_macros::sim_test] +async fn the_init_timer_never_kills_a_ready_child() { + let after = Duration::from_millis(50); + let result = TakesExactly(after) + .with_options(StartOptions::new().init_timeout(after)) + .start() + .await; + match result { + Ok(outcome) => { + let actor = outcome.started().unwrap(); + spawned_rt::tasks::sleep(Duration::from_millis(20)).await; + assert!( + actor.request(Ping).await.is_ok(), + "a child answered Started was killed by the init timer" + ); + actor.child_handle().stop(); + actor.wait_exit().await; + } + Err(StartError::InitTimeout { .. }) => {} + other => panic!("{other:?}"), + } +} + +/// Planted bug 25 (failpoint): the timer fires before the ready commit but +/// reaches the cell after it (`start.init_timer_fire` delays it). The ready +/// commitment stands; the timer loses and kills nothing. +#[cfg(all(feature = "failpoints", not(spawned_sim)))] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_timer_reaching_the_cell_after_ready_loses() { + use spawned_concurrency::failpoints::{self, Action, FailPoint, FailScenario}; + let _scenario = FailScenario::setup(); + failpoints::configure( + "start.init_timer_fire", + FailPoint::new(Action::Delay(Duration::from_millis(400))).times(1), + ) + .unwrap(); + let actor = TakesExactly(Duration::from_millis(100)) + .with_options(StartOptions::new().init_timeout(Duration::from_millis(20))) + .start() + .await + .unwrap() + .started() + .unwrap(); + // Until the delayed timer has reached the cell. + let deadline = std::time::Instant::now() + LIMIT; + while failpoints::stats("start.init_timer_fire").fired < 1 { + assert!(std::time::Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(5)).await; + } + tokio::time::sleep(Duration::from_millis(600)).await; + assert!( + actor.request(Ping).await.is_ok(), + "the late timer killed a ready child" + ); +} diff --git a/concurrency/tests/start_trace.rs b/concurrency/tests/start_trace.rs index 18567660..96538b7f 100644 --- a/concurrency/tests/start_trace.rs +++ b/concurrency/tests/start_trace.rs @@ -1194,3 +1194,44 @@ async fn init_stop_with_shutdown_payload() { p.request(Stop).await.unwrap(); p.wait_exit().await; } + +/// A child whose `started()` fails and whose destructor is slow. +struct SlowDropChild; +impl Actor for SlowDropChild { + async fn started(&mut self, _: &Context) -> Result<(), String> { + Err("no config".into()) + } +} +impl Drop for SlowDropChild { + fn drop(&mut self) { + std::thread::sleep(Duration::from_millis(50)); + } +} + +/// A failed start_link is answered after the child's publication, even +/// with a slow destructor and the waiter on another worker (design 5.5; +/// the model's `FailureAfterPublication`). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn failed_start_answers_after_a_slow_drop() { + let _serial = SERIAL.lock().await; + let (p, _) = parent(true).await; + lifecycle_trace::record_start_parent(&p.child_handle()); + let result = SlowDropChild.start_link(&p.context()).await; + assert!( + matches!(result, Err(StartError::InitFailed { pending: None, .. })), + "{result:?}" + ); + children_done(1).await; + let events = write("failed_start_answers_after_a_slow_drop", "U", true, 1000); + has( + &events, + &[ + "SResolve:error", + "SDie:Normal", + "SPublish:divert", + "SAnswer:failed", + ], + ); + p.request(Stop).await.unwrap(); + p.wait_exit().await; +} diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 0a6a57fc..40d54f0e 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -1858,6 +1858,13 @@ out.expect_no_message(Duration::from_millis(20)).await; messages; a reply closure that panics logs its message as `Crashed`. - The log keeps one entry per message for the mock's lifetime, so it suits unit tests rather than soak runs with millions of messages. +- Start behaviour (0.9): `on_started(|ctx| Init::error("no backend"))` makes + the mock's `started()` return that `Init` (continue, error, stop, ignore), + to test how the code under test handles a dependency that fails to start; + `start_delay(d)` makes `started()` take `d` first (virtual time under the + simulator in tasks mode), for `init_timeout` and waiting starts; + `drop_flag(flag)` sets `flag` when the mock's state is dropped, so a test + can check that a failed start answered only after the state was gone. `MockRef` dereferences to `ActorRef`. Its own methods win over protocol methods with the same name; call those through `actor_ref()` or the `XRef`. diff --git a/docs/TESTING.md b/docs/TESTING.md index 76c7049c..6228804d 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -144,7 +144,7 @@ section maps each one to what spawned does and lists where it is still behind. | Fault injection | Failpoints at named sites in the lifecycle code; suites that panic in every user callback and drop every public future at every await point | | Formal specifications | TLA+ models of every lifecycle protocol, each invariant with a negative control that must fail; Apalache proofs of the safety invariants | | Specs tied to the code | Trace validation of real runs against the models, trace coverage, and the required `model-sync` check: code that changes modelled behaviour without its model fails CI | -| Proving the tests work | The [planted-bug matrix](#planted-bug-sensitivity) (all 24 realistic bugs caught, each by two or more layers), `cargo-mutants`, and the [semantics coverage](#semantics-coverage) matrix (a check that fails for all 50 guarantees) | +| Proving the tests work | The [planted-bug matrix](#planted-bug-sensitivity) (all 24 realistic 0.8 bugs and the 10 for the 0.9 unified start caught, each by two or more layers), `cargo-mutants`, and the [semantics coverage](#semantics-coverage) matrix (a check that fails for all 50 guarantees) | | Fuzzing, metamorphic testing | The coverage-guided [scenario fuzzer](#scenario-fuzzing-fuzz) with contract oracles; one workload on every backend must give the same result | | Linearizability checking | A Porcupine-style checker over concurrent request histories | | Memory-model checking | Loom, ThreadSanitizer, LeakSanitizer and Kani proofs | @@ -1558,6 +1558,9 @@ python3 -m unittest scripts/tests/test_model_sync.py # self-tests and Mutation testing breaks one operator at a time. To measure whether the stack catches the bugs a reviewer would actually miss, 24 realistic bugs were written +(plus 10 for the 0.9 unified start, patches 25-34: the init timer, the +notice disposition table, the answer after publication, the held administrative stop, +held signals, the stage-1 parent check, the decision precedence and row 9) by hand (`verification/planted/patches/`, one patch each, listed in `INDEX.tsv`) and each was run, alone, against six layers: @@ -1572,7 +1575,7 @@ by hand (`verification/planted/patches/`, one patch each, listed in ```sh verification/planted/run.sh -l TSCFVK -f 900 -o /tmp/planted baseline # control: every layer must miss -verification/planted/run.sh -l TSCFVK -f 600 -o /tmp/planted # all 24 patches +verification/planted/run.sh -l TSCFVK -f 600 -o /tmp/planted # all 34 patches verification/planted/run.sh -l TS -o /tmp/planted verification/planted/patches/09-*.patch ``` diff --git a/testing/src/shared.rs b/testing/src/shared.rs index ba712441..bbd5e22a 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -27,9 +27,37 @@ macro_rules! mock_builder { Self { rules: Rules::default(), inbox: Arc::new(Inbox::default()), + start: StartHooks::default(), } } + /// What the mock's `started()` returns (default: ready): an + /// `Init` (continue, error, stop, ignore) to test a caller's start + /// handling. Runs after [`start_delay`](Self::start_delay). + pub fn on_started( + mut self, + init: impl FnOnce(&Context) -> Init + Send + 'static, + ) -> Self { + self.start.on_started = Some(Box::new(init)); + self + } + + /// Make `started()` take `delay` before it returns (virtual time + /// under the simulator in tasks mode): a slow initialiser, for + /// `init_timeout` and waiting starts. + pub fn start_delay(mut self, delay: Duration) -> Self { + self.start.delay = Some(delay); + self + } + + /// Set `flag` when the mock's state is dropped: lets a test check + /// that a failed start was answered only after the state was gone + /// (publication order). + pub fn drop_flag(mut self, flag: Arc) -> Self { + self.start.drop_flag = Some(flag); + self + } + /// Answer each `M` with `reply(&msg)`. The message is still logged /// for `expect`. For fire-and-forget messages return `()`. pub fn on( diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index 0850c0a5..836a4458 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -18,7 +18,7 @@ use spawned_concurrency::message::Message; use spawned_concurrency::supervision::{ChildState, SupervisedChild, Supervisor, SupervisorReport}; use spawned_concurrency::system::{ActorSystem, ExecutionGroup, ShutdownReport}; use spawned_concurrency::tasks::{ - Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, StartOptions, + Actor, ActorRef, ActorStart, Context, Handler, Init, Receiver, Recipient, StartOptions, }; use spawned_concurrency::{ChildHandle, ExitReason}; use spawned_concurrency::{Deadline, PendingReply, ReplyReceiver}; @@ -54,6 +54,23 @@ use std::time::Duration; pub struct Mock { rules: Rules, inbox: Arc, + start: StartHooks, +} + +/// The mock's start behaviour (`on_started`, `start_delay`, `drop_flag`). +#[derive(Default)] +struct StartHooks { + on_started: Option) -> Init + Send>>, + delay: Option, + drop_flag: Option>, +} + +impl Drop for Mock { + fn drop(&mut self) { + if let Some(flag) = &self.start.drop_flag { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + } + } } impl Default for Mock { @@ -92,7 +109,18 @@ impl Mock { } } -impl Actor for Mock {} +impl Actor for Mock { + #[allow(refining_impl_trait)] // returns an `Init` + async fn started(&mut self, ctx: &Context) -> Init { + if let Some(delay) = self.start.delay { + sleep(delay).await; + } + match self.start.on_started.take() { + Some(init) => init(ctx), + None => Init::ready(), + } + } +} impl Handler for Mock { fn handle( diff --git a/testing/src/threads.rs b/testing/src/threads.rs index 6e87bd97..9ceea084 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -7,7 +7,7 @@ use crate::shared::SETTLE_TIMEOUT; use spawned_concurrency::error::ActorError; use spawned_concurrency::message::Message; use spawned_concurrency::threads::{ - Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, + Actor, ActorRef, ActorStart, Context, Handler, Init, Receiver, Recipient, }; use spawned_concurrency::{ChildHandle, ExitReason}; use spawned_concurrency::{Deadline, PendingReply, ReplyReceiver}; @@ -22,6 +22,23 @@ use std::time::{Duration, Instant}; pub struct Mock { rules: Rules, inbox: Arc, + start: StartHooks, +} + +/// The mock's start behaviour (`on_started`, `start_delay`, `drop_flag`). +#[derive(Default)] +struct StartHooks { + on_started: Option) -> Init + Send>>, + delay: Option, + drop_flag: Option>, +} + +impl Drop for Mock { + fn drop(&mut self) { + if let Some(flag) = &self.start.drop_flag { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + } + } } impl Default for Mock { @@ -49,7 +66,18 @@ impl Mock { } } -impl Actor for Mock {} +impl Actor for Mock { + #[allow(refining_impl_trait)] // returns an `Init` + fn started(&mut self, ctx: &Context) -> Init { + if let Some(delay) = self.start.delay { + std::thread::sleep(delay); + } + match self.start.on_started.take() { + Some(init) => init(ctx), + None => Init::ready(), + } + } +} impl Handler for Mock { fn handle(&mut self, msg: M, reply: PendingReply<'_, M::Result>, _: &Context) { diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 6d12b54d..e3c497cd 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -654,3 +654,37 @@ async fn in_actor_runs_a_closure_in_a_turn() { tally.context().stop(); tally.join().await; } + +/// `Mock::on_started` and `drop_flag`: a failed start is answered only after +/// the mock's state is dropped (publication order). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mock_on_started_error_answers_after_drop() { + let dropped = Arc::new(AtomicBool::new(false)); + let result = Mock::new() + .on_started(|_| tasks::Init::error("no backend")) + .drop_flag(dropped.clone()) + .start() + .await; + assert!(matches!( + result, + Err(spawned_concurrency::StartError::InitFailed { pending: None, .. }) + )); + assert!( + dropped.load(Ordering::SeqCst), + "answered before the state drop" + ); +} + +/// `Mock::start_delay` with an init timeout: the start times out. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mock_start_delay_hits_init_timeout() { + let result = Mock::new() + .start_delay(Duration::from_millis(200)) + .with_options(tasks::StartOptions::new().init_timeout(Duration::from_millis(20))) + .start() + .await; + assert!(matches!( + result, + Err(spawned_concurrency::StartError::InitTimeout { pending: None }) + )); +} diff --git a/verification/planted/patches/25-start-timer-overrides-ready.patch b/verification/planted/patches/25-start-timer-overrides-ready.patch new file mode 100644 index 00000000..25361869 --- /dev/null +++ b/verification/planted/patches/25-start-timer-overrides-ready.patch @@ -0,0 +1,13 @@ +diff --git a/concurrency/src/start.rs b/concurrency/src/start.rs +index e169bf32..1a3571a7 100644 +--- a/concurrency/src/start.rs ++++ b/concurrency/src/start.rs +@@ -611,7 +611,7 @@ impl StartupCell { + pub(crate) fn time_out(&self) -> bool { + let mut inner = self.lock(); + inner.timer = None; +- let won = self.commit() == Commit::Pending; ++ let won = self.commit() != Commit::TimedOut; + if won { + self.state.store(Commit::TimedOut as u8, Ordering::Release); + inner.committed_at = Some(Instant::now()); diff --git a/verification/planted/patches/26-start-trapping-parent-signal.patch b/verification/planted/patches/26-start-trapping-parent-signal.patch new file mode 100644 index 00000000..cfa42626 --- /dev/null +++ b/verification/planted/patches/26-start-trapping-parent-signal.patch @@ -0,0 +1,13 @@ +diff --git a/concurrency/src/start.rs b/concurrency/src/start.rs +index e169bf32..c2ee47ac 100644 +--- a/concurrency/src/start.rs ++++ b/concurrency/src/start.rs +@@ -959,7 +959,7 @@ fn dispose( + match waiter { + Waiter::Waiting => match kind { + EdgeKind::Monitor => Disposition::Divert, +- EdgeKind::Link if trapping || commit == Commit::TimedOut => Disposition::Divert, ++ EdgeKind::Link if commit == Commit::TimedOut => Disposition::Divert, + // A non-trapping parent: applied if non-Normal (OTP: the link + // kills the caller); a Normal one is ignored, so it is dropped + // here (the model's legitimate drop) rather than delivered. diff --git a/verification/planted/patches/27-start-failed-down-delivered.patch b/verification/planted/patches/27-start-failed-down-delivered.patch new file mode 100644 index 00000000..9beddf6d --- /dev/null +++ b/verification/planted/patches/27-start-failed-down-delivered.patch @@ -0,0 +1,13 @@ +diff --git a/concurrency/src/start.rs b/concurrency/src/start.rs +index e169bf32..8567bc58 100644 +--- a/concurrency/src/start.rs ++++ b/concurrency/src/start.rs +@@ -958,7 +958,7 @@ fn dispose( + } + match waiter { + Waiter::Waiting => match kind { +- EdgeKind::Monitor => Disposition::Divert, ++ EdgeKind::Monitor => Disposition::Deliver, + EdgeKind::Link if trapping || commit == Commit::TimedOut => Disposition::Divert, + // A non-trapping parent: applied if non-Normal (OTP: the link + // kills the caller); a Normal one is ignored, so it is dropped diff --git a/verification/planted/patches/28-start-answer-before-publication.patch b/verification/planted/patches/28-start-answer-before-publication.patch new file mode 100644 index 00000000..163a271b --- /dev/null +++ b/verification/planted/patches/28-start-answer-before-publication.patch @@ -0,0 +1,12 @@ +diff --git a/concurrency/src/start.rs b/concurrency/src/start.rs +index e169bf32..7729d0b9 100644 +--- a/concurrency/src/start.rs ++++ b/concurrency/src/start.rs +@@ -727,6 +727,7 @@ impl StartupCell { + _ => { + let (pending, reason) = match &inner.published { + Some(final_reason) => (None, final_reason.clone()), ++ None if inner.requested.is_some() => (None, inner.requested.clone().unwrap()), + None if inner.publishing => return self.wait_or_abandon(inner, waiter, None), + None => { + let deadline = inner diff --git a/verification/planted/patches/29-start-admin-stop-lost.patch b/verification/planted/patches/29-start-admin-stop-lost.patch new file mode 100644 index 00000000..40f7ad5c --- /dev/null +++ b/verification/planted/patches/29-start-admin-stop-lost.patch @@ -0,0 +1,15 @@ +diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs +index 5add5c2a..53b0c026 100644 +--- a/concurrency/src/link.rs ++++ b/concurrency/src/link.rs +@@ -361,9 +361,7 @@ impl ExitRequest { + // A held administrative stop now begins an ordinary exit: the + // run loop (already woken) dispatches nothing and runs stopped(). + // (Its `SExit` is recorded right after the ready `SResolve`.) +- if let Some(reason) = admin { +- self.reason = Some(reason); +- } ++ let _ = admin; + } else { + self.frozen = true; + self.skip_hook = true; diff --git a/verification/planted/patches/30-start-admin-stop-fails-init.patch b/verification/planted/patches/30-start-admin-stop-fails-init.patch new file mode 100644 index 00000000..0a2f3f92 --- /dev/null +++ b/verification/planted/patches/30-start-admin-stop-fails-init.patch @@ -0,0 +1,14 @@ +diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs +index 5add5c2a..7f78c7a4 100644 +--- a/concurrency/src/link.rs ++++ b/concurrency/src/link.rs +@@ -176,7 +176,8 @@ impl ExitRequest { + if self.admin.is_none() && self.reason.is_none() { + #[cfg(feature = "lifecycle-trace")] + crate::lifecycle_trace::start_admin(self.trace); +- self.admin = Some(reason); ++ self.admin = Some(reason.clone()); ++ self.reason = Some(reason); + } + return true; + } diff --git a/verification/planted/patches/31-start-trapping-fails-on-signal.patch b/verification/planted/patches/31-start-trapping-fails-on-signal.patch new file mode 100644 index 00000000..2d615944 --- /dev/null +++ b/verification/planted/patches/31-start-trapping-fails-on-signal.patch @@ -0,0 +1,13 @@ +diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs +index 5add5c2a..92c36109 100644 +--- a/concurrency/src/link.rs ++++ b/concurrency/src/link.rs +@@ -312,7 +312,7 @@ impl ExitRequest { + // Row 3: the first held non-Normal signal to a non-trapping child. + // (An administrative stop is not a row: it is applied after a ready + // commitment, below.) +- let row3 = if trapping { ++ let row3 = if false { + None + } else { + held.iter().find(|r| **r != ExitReason::Normal).cloned() diff --git a/verification/planted/patches/32-start-link-to-exiting-parent.patch b/verification/planted/patches/32-start-link-to-exiting-parent.patch new file mode 100644 index 00000000..45925919 --- /dev/null +++ b/verification/planted/patches/32-start-link-to-exiting-parent.patch @@ -0,0 +1,13 @@ +diff --git a/concurrency/src/tasks/actor/lifecycle.rs b/concurrency/src/tasks/actor/lifecycle.rs +index c3cb4b99..10cd89fa 100644 +--- a/concurrency/src/tasks/actor/lifecycle.rs ++++ b/concurrency/src/tasks/actor/lifecycle.rs +@@ -239,7 +239,7 @@ impl ActorRef { + &links, + || { + let request = link::lock_request(&p.request); +- if request.reason().is_some() || request.is_dead() { ++ if request.is_dead() { + trace::commit_point(Some("parentexited")); + return false; + } diff --git a/verification/planted/patches/33-start-kill-below-signal.patch b/verification/planted/patches/33-start-kill-below-signal.patch new file mode 100644 index 00000000..f4cbb437 --- /dev/null +++ b/verification/planted/patches/33-start-kill-below-signal.patch @@ -0,0 +1,13 @@ +diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs +index 5add5c2a..67841882 100644 +--- a/concurrency/src/link.rs ++++ b/concurrency/src/link.rs +@@ -321,7 +321,7 @@ impl ExitRequest { + let row = if timed_out { + self.reason = Some(ExitReason::Killed); + Row::Timer +- } else if self.killed { ++ } else if self.killed && row3.is_none() { + self.reason = Some(ExitReason::Killed); + Row::Kill + } else if let Some(reason) = row3 { diff --git a/verification/planted/patches/34-start-ready-in-closing-group.patch b/verification/planted/patches/34-start-ready-in-closing-group.patch new file mode 100644 index 00000000..4ea38055 --- /dev/null +++ b/verification/planted/patches/34-start-ready-in-closing-group.patch @@ -0,0 +1,15 @@ +diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs +index b1354adf..bb654d6f 100644 +--- a/concurrency/src/system.rs ++++ b/concurrency/src/system.rs +@@ -1249,9 +1249,7 @@ impl Scope { + let group = self.group.upgrade(); + let group_state = group.as_ref().map(|g| lock(&g.state)); + let mut scope = lock(&self.state); +- let open = parent.as_deref() != Some(&false) +- && group_state.as_ref().is_some_and(|g| !g.closing) +- && scope.open; ++ let open = parent.as_deref() != Some(&false) && group_state.is_some() && scope.open; + let (result, ready) = decide(open); + if ready { + scope.ready_at = Some(Instant::now()); diff --git a/verification/planted/patches/INDEX.tsv b/verification/planted/patches/INDEX.tsv index 5d512a88..e98a7126 100644 --- a/verification/planted/patches/INDEX.tsv +++ b/verification/planted/patches/INDEX.tsv @@ -23,3 +23,13 @@ id name file bug 22 timer-interval-outlives-actor tasks/time.rs send_interval keeps ticking after its actor stops (only its own token stops it) 23 sys-shutdown-forward-order system.rs ActorSystem shutdown stops targets in admission order instead of reverse 24 sys-targets-unsorted system.rs System shutdown stops all standalone actors before all trees, ignoring admission order +25 start-timer-overrides-ready start.rs The init timer commits TimedOut over a Ready commitment (kills a ready child) +26 start-trapping-parent-signal start.rs A waiting trapping parent gets the failed start's link notice delivered instead of diverted to the answer +27 start-failed-down-delivered start.rs A failed start_monitor's Down reaches the waiting parent instead of being consumed by the error +28 start-answer-before-publication start.rs A failed start is answered at its decision, before the child's publication (state not yet dropped) +29 start-admin-stop-lost link.rs An administrative stop held during started() is discarded when the start commits ready +30 start-admin-stop-fails-init link.rs An administrative stop during started() fails the start without stopped() (revision 3's row 3) +31 start-trapping-fails-on-signal link.rs A trapping child fails its start on a held non-Normal signal instead of queueing it +32 start-link-to-exiting-parent tasks/actor/lifecycle.rs A start_link commits its link to a parent already exiting (stage-1 check misses a requested exit) +33 start-kill-below-signal link.rs A held untrapped signal outranks a kill during started() (final reason not Killed) +34 start-ready-in-closing-group system.rs A start commits ready while its execution group is closing (row 9 lost) diff --git a/verification/planted/run.sh b/verification/planted/run.sh index 775bbb15..1c80016b 100755 --- a/verification/planted/run.sh +++ b/verification/planted/run.sh @@ -153,6 +153,7 @@ layer_V() { LIFECYCLE_TRACE_DIR="$traces" cargo test -p spawned-concurrency \ --features lifecycle-trace,failpoints --no-fail-fast --test supervision_trace \ --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace \ + --test lifecycle_trace --test start_trace \ > "$log" 2>&1 if grep -qE '^error(\[E[0-9]+\])?:' "$log" && ! grep -q '^test result' "$log"; then verdict=build; secs=0; detail=""; return