From 8badc8f4f04b4c892b6e7125edd90756ed375e38 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 17 Nov 2026 23:03:37 -0300 Subject: [PATCH] feat(metrics): disposal worker pool gauges Add spawned_disposal_queue_entries, disposal_workers_alive and disposal_workers_idle, read at scrape under the pool's lock (no change on the hand-off path), and spawned_disposal_workers_capped_total, counted once per cap episode beside the existing warning (a no-op without metrics). tests/metrics_retained.rs expires deadlines in both backends with non-blocking (waiting callers' wake-ups) and blocking (a discarded emit whose destructor sleeps) disposal and checks the queue drains and every worker retires after its keep-alive (disposal.keep_alive failpoint with failpoints, the real 10 s otherwise). The soak harness judges the three gauges. Removes the GUIDE note that deferred them to #282. --- CHANGELOG.md | 9 + concurrency/benches/soak/main.rs | 3 + concurrency/benches/soak/report.rs | 5 +- concurrency/benches/soak/thresholds.toml | 10 + concurrency/src/instrumentation.rs | 2 + concurrency/src/metrics.rs | 22 +- concurrency/src/reply/disposal.rs | 32 ++- concurrency/tests/metrics_retained.rs | 261 ++++++++++++++++++++++- docs/GUIDE.md | 22 +- docs/TESTING.md | 1 + 10 files changed, 348 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 362bfe88..c8eb616f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,15 @@ its baseline (`tests/metrics_retained.rs`), which doubles as a leak test. `timers_alive` is how the threads-mode timer callback leak above was found. +- `metrics`: the disposal worker pool's gauges, `spawned_disposal_queue_entries`, + `disposal_workers_alive` and `disposal_workers_idle` (read at scrape under + the pool's lock), and the counter `spawned_disposal_workers_capped_total` + (times all 64 workers were busy, once per episode, beside the existing + warning). Compiled out without the feature; no change on the hand-off path + otherwise. A test expires deadlines in both backends with blocking and + non-blocking disposal and checks the queue drains and every worker retires + after its keep-alive (`disposal.keep_alive` failpoint with `failpoints`). + The soak harness judges the three gauges. - Fix the flaky `supervision::transient_startup_timeout_retries_after_actual_cleanup` (a 20 ms startup deadline raced the fast starts on a loaded host, which also made cargo-mutants report surviving mutants as caught): the replacement's diff --git a/concurrency/benches/soak/main.rs b/concurrency/benches/soak/main.rs index b6f17fbb..f4535191 100644 --- a/concurrency/benches/soak/main.rs +++ b/concurrency/benches/soak/main.rs @@ -278,6 +278,9 @@ impl Recorder { m.link_records_alive as f64, m.registered_names as f64, m.timers_alive as f64, + m.disposal_queue_entries as f64, + m.disposal_workers_alive as f64, + m.disposal_workers_idle as f64, ]; debug_assert_eq!(values.len(), SERIES.len()); let sample = Sample { diff --git a/concurrency/benches/soak/report.rs b/concurrency/benches/soak/report.rs index 9d982b3c..d690b86d 100644 --- a/concurrency/benches/soak/report.rs +++ b/concurrency/benches/soak/report.rs @@ -27,6 +27,9 @@ pub const SERIES: &[&str] = &[ "link_records_alive", "registered_names", "timers_alive", + "disposal_queue_entries", + "disposal_workers_alive", + "disposal_workers_idle", ]; /// Flat `key = number` lines (valid TOML with dotted keys); `#` comments. @@ -81,7 +84,7 @@ impl Phase { pub struct Sample { pub elapsed: f64, pub phase: Phase, - pub values: [f64; 23], + pub values: [f64; 26], pub started: u64, pub scenarios: u64, pub errors: u64, diff --git a/concurrency/benches/soak/thresholds.toml b/concurrency/benches/soak/thresholds.toml index 44c9d361..6490acb3 100644 --- a/concurrency/benches/soak/thresholds.toml +++ b/concurrency/benches/soak/thresholds.toml @@ -40,6 +40,9 @@ trend.monitors_alive = 10 trend.link_records_alive = 10 trend.registered_names = 10 trend.timers_alive = 10 +trend.disposal_queue_entries = 10 +trend.disposal_workers_alive = 10 +trend.disposal_workers_idle = 10 # Minimum growth: a trend also needs slope x steady-window hours above this # many units (MiB for rss_mb and heap_mb) to fail. Bounded pools (Tokio's @@ -70,6 +73,9 @@ growth.monitors_alive = 5 growth.link_records_alive = 5 growth.registered_names = 5 growth.timers_alive = 5 +growth.disposal_queue_entries = 5 +growth.disposal_workers_alive = 5 +growth.disposal_workers_idle = 5 # Quiesce: after the workload stops and the long-lived ActorSystem shuts # down, each series must return to within `slack` of its baseline (sampled @@ -98,3 +104,7 @@ quiesce.monitors_alive = 0 quiesce.link_records_alive = 0 quiesce.registered_names = 0 quiesce.timers_alive = 0 +# Idle disposal workers retire after their 10 s keep-alive, within the timeout. +quiesce.disposal_queue_entries = 0 +quiesce.disposal_workers_alive = 0 +quiesce.disposal_workers_idle = 0 diff --git a/concurrency/src/instrumentation.rs b/concurrency/src/instrumentation.rs index 83b7f863..74ae4f27 100644 --- a/concurrency/src/instrumentation.rs +++ b/concurrency/src/instrumentation.rs @@ -84,6 +84,8 @@ mod disabled { pub(crate) fn job_emitted() {} #[inline(always)] pub(crate) fn jobs_unfinished(_: usize) {} + #[inline(always)] + pub(crate) fn disposal_capped() {} } /// Retained-structure gauges: zero-sized, no-op guards without `metrics`. diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index 4c7912c6..a49bf8ce 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -247,10 +247,15 @@ counters!( link_records_alive, registered_names, timers_alive, + disposal_queue_entries, + disposal_workers_alive, + disposal_workers_idle, + disposal_workers_capped_total, ); -/// Process aggregates (see [`Snapshot`]). The deadline service's sizes are -/// read here, under its own lock; every other value is an atomic load. +/// Process aggregates (see [`Snapshot`]). The deadline service's and the +/// disposal pool's sizes are read here, each under its own lock; every other +/// value is an atomic load. pub fn snapshot() -> Snapshot { let (entries, actors) = crate::reply::deadline::service_sizes(); COUNTERS @@ -259,6 +264,14 @@ pub fn snapshot() -> Snapshot { COUNTERS .deadline_service_actors .store(actors as u64, Relaxed); + let (queued, workers, idle) = crate::reply::disposal::pool_sizes(); + COUNTERS + .disposal_queue_entries + .store(queued as u64, Relaxed); + COUNTERS + .disposal_workers_alive + .store(workers as u64, Relaxed); + COUNTERS.disposal_workers_idle.store(idle as u64, Relaxed); read_snapshot() } static QUEUE_WAIT: Histogram = Histogram::new(); @@ -392,6 +405,11 @@ pub(crate) mod hooks { .jobs_unfinished_at_stop_total .fetch_add(n as u64, Relaxed); } + /// Every disposal worker was busy at the cap (counted once per episode, + /// like the `disposal_workers_capped` warning). + pub(crate) fn disposal_capped() { + COUNTERS.disposal_workers_capped_total.fetch_add(1, Relaxed); + } } /// Retained-structure gauges: a zero-sized [`Live`](live::Live) field in each diff --git a/concurrency/src/reply/disposal.rs b/concurrency/src/reply/disposal.rs index 90c6106f..1b9948e9 100644 --- a/concurrency/src/reply/disposal.rs +++ b/concurrency/src/reply/disposal.rs @@ -240,11 +240,24 @@ struct Pool { ready: Condvar, } +static POOL: OnceLock> = OnceLock::new(); + fn pool() -> &'static Arc { - static POOL: OnceLock> = OnceLock::new(); POOL.get_or_init(Default::default) } +/// The pool's `(queued items, workers, idle workers)`: workers include one +/// still starting, idle ones one woken and not yet back under the lock. Read +/// on a metrics scrape, under the pool's lock; `(0, 0, 0)` before the first +/// hand-off. +#[cfg(feature = "metrics")] +pub(crate) fn pool_sizes() -> (usize, usize, usize) { + POOL.get().map_or((0, 0, 0), |pool| { + let state = lock(pool); + (state.queue.len(), state.workers, state.idle) + }) +} + fn lock(pool: &Pool) -> std::sync::MutexGuard<'_, State> { pool.state.lock().unwrap_or_else(|p| p.into_inner()) } @@ -291,14 +304,17 @@ pub(crate) fn submit(work: Vec) { fn start_worker(pool: &Arc, start: Start) { match start { Start::No => {} - Start::Capped => crate::actor_cleanup::log_safely(|| { - tracing::warn!( - event = "disposal_workers_capped", - max = MAX_WORKERS, - "every deadline disposal worker is busy: later wake-ups and \ + Start::Capped => { + crate::instrumentation::hooks::disposal_capped(); + crate::actor_cleanup::log_safely(|| { + tracing::warn!( + event = "disposal_workers_capped", + max = MAX_WORKERS, + "every deadline disposal worker is busy: later wake-ups and \ payload drops queue until one returns" - ); - }), + ); + }) + } Start::Yes => { let worker = pool.clone(); let spawned = std::thread::Builder::new() diff --git a/concurrency/tests/metrics_retained.rs b/concurrency/tests/metrics_retained.rs index 4443a4fd..0db26785 100644 --- a/concurrency/tests/metrics_retained.rs +++ b/concurrency/tests/metrics_retained.rs @@ -512,10 +512,269 @@ fn threads_timers_do_not_accumulate_on_a_running_actor() { #[test] fn retained_gauges_are_exported() { let text = metrics::render_prometheus(); - for (name, _) in GAUGES { + for (name, _) in GAUGES.iter().chain(DISPOSAL) { assert!( text.contains(&format!("# TYPE spawned_{name} gauge\n")), "spawned_{name} is not exported as a gauge" ); } + assert!( + text.contains("# TYPE spawned_disposal_workers_capped_total counter\n"), + "spawned_disposal_workers_capped_total is not exported as a counter" + ); +} + +// --------------------------------------------------------------------------- +// disposal pool +// --------------------------------------------------------------------------- + +/// The disposal pool's gauges. Kept out of `GAUGES`: idle workers outlive +/// the work that started them by their keep-alive (10 s), so the other tests +/// do not wait for them. +const DISPOSAL: &[Gauge] = &[ + ("disposal_queue_entries", |s| s.disposal_queue_entries), + ("disposal_workers_alive", |s| s.disposal_workers_alive), + ("disposal_workers_idle", |s| s.disposal_workers_idle), +]; + +fn read_disposal() -> Vec { + let s = metrics::snapshot(); + DISPOSAL.iter().map(|(_, g)| g(&s)).collect() +} + +/// Wait (bounded by `limit`) until no item is queued and every worker has +/// retired, the pool's resting state. +fn disposal_pool_rests(limit: Duration, what: &str) { + let end = Instant::now() + limit; + loop { + let now = read_disposal(); + if now == [0, 0, 0] { + return; + } + assert!( + Instant::now() < end, + "{what}: disposal pool did not drain and retire within {limit:?}: {:?}", + DISPOSAL + .iter() + .map(|(n, _)| *n) + .zip(now) + .collect::>() + ); + std::thread::sleep(Duration::from_millis(5)); + } +} + +/// Requests that expire while their callers wait: the deadline service hands +/// each caller's wake-up to a disposal worker (non-blocking items). +const EXPIRE: Duration = Duration::from_millis(30); + +/// A message whose destructor blocks for a while: discarded on the deadline +/// service, it occupies a disposal worker (a blocking item). +struct Slow(Option>); +impl Message for Slow { + type Result = (); +} +impl Drop for Slow { + fn drop(&mut self) { + if let Some(dropped) = self.0.take() { + std::thread::sleep(Duration::from_millis(50)); + dropped.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } +} +/// Start a blocking job whose deadline (20 ms) passes before its scheduled +/// `Slow` emit (40 ms): the service discards the emit and its destructor +/// runs on a disposal worker. +struct StallJob(std::sync::Arc); +impl Message for StallJob { + type Result = (); +} +const JOB_DEADLINE: Duration = Duration::from_millis(20); +const EMIT_AFTER: Duration = Duration::from_millis(40); +const JOB_RUNS: Duration = Duration::from_millis(100); + +#[derive(Default)] +struct AsyncExpirer(Vec>); +impl tasks::Actor for AsyncExpirer {} +impl tasks::Handler for AsyncExpirer { + async fn handle(&mut self, _: Park, reply: PendingReply<'_, u64>, _: &tasks::Context) { + self.0.push(reply.defer().unwrap()); + } +} +impl tasks::Handler for AsyncExpirer { + async fn handle( + &mut self, + StallJob(dropped): StallJob, + reply: PendingReply<'_, ()>, + ctx: &tasks::Context, + ) { + ctx.pipe_blocking( + tasks::JobOptions::new().deadline(JOB_DEADLINE), + move |job| { + let at = tasks::Instant::now() + EMIT_AFTER; + job.emit_at_blocking(at, Slow(Some(dropped))).unwrap(); + std::thread::sleep(JOB_RUNS); + }, + |_: tasks::JobOutcome<()>| JobDone, + ) + .unwrap(); + reply.reply(()); + } +} +impl tasks::Handler for AsyncExpirer { + async fn handle(&mut self, mut msg: Slow, r: PendingReply<'_, ()>, _: &tasks::Context) { + msg.0 = None; // delivered, not discarded: not the scenario + r.reply(()); + } +} +impl tasks::Handler for AsyncExpirer { + async fn handle(&mut self, _: JobDone, r: PendingReply<'_, ()>, _: &tasks::Context) { + r.reply(()); + } +} + +async fn async_expiry(dropped: std::sync::Arc) { + use tasks::ActorStart; + let owner = AsyncExpirer::default().start(); + owner.request(StallJob(dropped)).await.unwrap(); + let expiring: Vec<_> = (0..3) + .map(|_| owner.send_request(Park, Deadline::after(EXPIRE)).unwrap()) + .collect(); + for rx in expiring { + assert!(matches!( + rx.await, + Err(spawned_concurrency::ActorError::RequestTimeout) + )); + } + tokio::time::sleep(JOB_RUNS).await; + owner.context().stop(); + owner.join().await; +} + +#[derive(Default)] +struct NativeExpirer(Vec>); +impl threads::Actor for NativeExpirer {} +impl threads::Handler for NativeExpirer { + fn handle(&mut self, _: Park, reply: PendingReply<'_, u64>, _: &threads::Context) { + self.0.push(reply.defer().unwrap()); + } +} +impl threads::Handler for NativeExpirer { + fn handle( + &mut self, + StallJob(dropped): StallJob, + reply: PendingReply<'_, ()>, + ctx: &threads::Context, + ) { + ctx.pipe_blocking( + threads::JobOptions::new().deadline(JOB_DEADLINE), + move |job| { + let at = threads::Instant::now() + EMIT_AFTER; + job.emit_at_blocking(at, Slow(Some(dropped))).unwrap(); + std::thread::sleep(JOB_RUNS); + }, + |_: threads::JobOutcome<()>| JobDone, + ) + .unwrap(); + reply.reply(()); + } +} +impl threads::Handler for NativeExpirer { + fn handle(&mut self, mut msg: Slow, r: PendingReply<'_, ()>, _: &threads::Context) { + msg.0 = None; + r.reply(()); + } +} +impl threads::Handler for NativeExpirer { + fn handle(&mut self, _: JobDone, r: PendingReply<'_, ()>, _: &threads::Context) { + r.reply(()); + } +} + +fn native_expiry(dropped: std::sync::Arc) { + use threads::ActorStart; + let owner = NativeExpirer::default().start(); + owner.request(StallJob(dropped)).unwrap(); + let expiring: Vec<_> = (0..3) + .map(|_| owner.send_request(Park, Deadline::after(EXPIRE)).unwrap()) + .collect(); + for rx in expiring { + assert!(matches!( + rx.recv(), + Err(spawned_concurrency::ActorError::RequestTimeout) + )); + } + std::thread::sleep(JOB_RUNS); + owner.context().stop(); + owner.join(); +} + +/// Expiring deadlines in both backends hand non-blocking items (waiting +/// callers' wake-ups) and blocking ones (a discarded emit whose destructor +/// sleeps) to the disposal workers: the pool's gauges rise, then the queue +/// drains and every worker retires after its keep-alive. With `failpoints` +/// the keep-alive is shortened (`disposal.keep_alive`); without, the wait +/// covers the real 10 s. +#[test] +fn disposal_pool_gauges_return_to_baseline() { + let _serial = SERIAL.lock().unwrap_or_else(|p| p.into_inner()); + #[cfg(feature = "failpoints")] + let _scenario = { + use spawned_concurrency::failpoints::{self, Action, FailPoint, FailScenario}; + let scenario = FailScenario::setup(); + failpoints::configure("disposal.keep_alive", FailPoint::new(Action::Error)).unwrap(); + scenario + }; + // A worker already waiting keeps the keep-alive it started with. + let keep_alive = Duration::from_secs(20); + disposal_pool_rests(keep_alive, "before the churn"); + let capped_before = metrics::snapshot().disposal_workers_capped_total; + + let rt = runtime(); + let dropped = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let peak = Mutex::new(vec![0u64; DISPOSAL.len()]); + let done = std::sync::atomic::AtomicBool::new(false); + std::thread::scope(|scope| { + scope.spawn(|| { + while !done.load(std::sync::atomic::Ordering::Relaxed) { + let now = read_disposal(); + let mut peak = peak.lock().unwrap(); + for (p, n) in peak.iter_mut().zip(now) { + *p = (*p).max(n); + } + drop(peak); + std::thread::sleep(Duration::from_micros(200)); + } + }); + for _ in 0..2 { + rt.block_on(futures::future::join_all( + (0..ACTORS).map(|_| async_expiry(dropped.clone())), + )); + std::thread::scope(|scope| { + for _ in 0..ACTORS { + let dropped = dropped.clone(); + scope.spawn(move || native_expiry(dropped)); + } + }); + } + // Every blocking item ran (its destructor counted itself). + let end = Instant::now() + Duration::from_secs(10); + while dropped.load(std::sync::atomic::Ordering::SeqCst) < 4 * ACTORS { + assert!(Instant::now() < end, "a discarded emit was never dropped"); + std::thread::sleep(Duration::from_millis(5)); + } + done.store(true, std::sync::atomic::Ordering::Relaxed); + }); + let peak = peak.into_inner().unwrap(); + assert!( + peak[1] >= 2, + "blocking items ran on fewer than two workers: peak {peak:?}" + ); + assert!(peak[2] >= 1, "no worker was ever idle: peak {peak:?}"); + disposal_pool_rests(keep_alive, "after the churn"); + // Far below the cap: never hit. + assert_eq!( + metrics::snapshot().disposal_workers_capped_total, + capped_before + ); } diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 4db9fcbf..21a85a54 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -1631,8 +1631,9 @@ that climbs with uptime while the actor and request counts stay flat points at a leak (or an application that holds framework objects), with no heap profiling build. Each one is counted where its structure is created and destroyed (a zero-sized field in the structure: one relaxed atomic at each -end, no lock), except the deadline service's two sizes, read at scrape under -the service's own lock. All of them are compiled out without the feature. +end, no lock), except the deadline service's two sizes and the disposal +pool's three, read at scrape under the service's or the pool's own lock. All +of them are compiled out without the feature. | Gauge (`spawned_*`) | Counts | Returns to baseline when | |---|---|---| @@ -1648,14 +1649,21 @@ the service's own lock. All of them are compiled out without the feature. | `link_records_alive` | Link records (two per link, one per pending late `NoProc` notice), including records taken at a death point until their notices are delivered. | Links are removed or either end exits. | | `registered_names` | Names in the global registry (`registry::register`, `register_owned`, `ctx.register_name`). | Names are unregistered (managed names: when the owner's cleanup ends). | | `timers_alive` | `send_after` / `send_interval` timers whose task or thread is still running (with it, in `threads` mode, its callback registrations on the actor's and its own stop tokens). | Timers fire or are cancelled, or the actor stops. | +| `disposal_queue_entries` | Items the deadline service handed to the disposal workers (wake-ups and payload drops an expiry released) not yet taken by a worker. Above zero for long only while every worker is busy at the cap. | Workers take them. | +| `disposal_workers_alive` | Disposal worker threads, busy or idle (a starting one included). At most 64. | Idle workers retire after their 10 s keep-alive. | +| `disposal_workers_idle` | Disposal workers waiting for work. | Idle workers retire after their keep-alive. | + +`spawned_disposal_workers_capped_total` counts the times every one of the 64 +disposal workers was busy when more work arrived (once per episode, with the +`disposal_workers_capped` warning): wake-ups and payload drops queued behind +blocked callbacks. A rising count means user wakers or destructors block. Gauges are process-wide sums and never labelled. They have idle baselines above zero (the detached registry, registered names an application keeps, an -interval timer): compare against the same process at rest, or alert on -growth. Process-wide one-time structures (the deadline-service thread, the -global registry map) are not counted. The disposal worker pool proposed with -the deadline-service fix (#282) will add its queue length and worker count -here when it lands. +interval timer, disposal workers within their keep-alive): compare against +the same process at rest, or alert on growth. Process-wide one-time +structures (the deadline-service thread, the global registry map) are not +counted. ### Caller and raw receiver boundaries diff --git a/docs/TESTING.md b/docs/TESTING.md index a231a774..6848b172 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -2036,6 +2036,7 @@ times. | `actors_live` | harness actor states constructed and not yet dropped (started minus fully stopped) | | `actors_active`, `mailbox_depth`, `handlers_active`, `requests_active` | spawned's own `metrics` gauges | | `reply_slots_alive` … `timers_alive` | spawned's retained-structure gauges (GUIDE, "Retained internal structures"): reply slots, tokens, `PendingReplies` entries, reply registries, deadline heap and service entries, service owners, jobs, monitors, link records, names, timers | +| `disposal_queue_entries`, `disposal_workers_alive`, `disposal_workers_idle` | the disposal worker pool's queue and workers (same GUIDE section); idle workers retire within the quiesce timeout | | `actors_started`, `scenarios`, `errors` | progress counters (not judged) | Framework-internal `ActorRef` and `Context` clones are not countable through