diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b5a60e70..99304408f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,20 @@ `callbacks_registered`; the timer thread now holds its registrations and drops them when it fires or is cancelled. Tasks-mode timers were not affected (Tokio removes a cancellation waiter when its future is dropped). +- `metrics`: gauges of spawned's own retained structures, so memory growth + can be attributed to the framework (or ruled out) from one scrape, without a + heap-profiling build: `spawned_reply_slots_alive`, `reply_tokens_alive`, + `pending_replies_entries`, `reply_registries_alive`, + `deadline_heap_entries`, `deadline_service_entries`, + `deadline_service_actors`, `jobs_alive`, `monitors_alive`, + `link_records_alive`, `registered_names` and `timers_alive` (docs/GUIDE.md, + "Retained internal structures"). Counted by a zero-sized field at each + structure's creation and drop (one relaxed atomic each end, no lock); the + deadline service's sizes are read at scrape. Compiled out without the + feature. A churn test in both backends checks that every gauge returns to + its baseline (`tests/metrics_retained.rs`), which doubles as a leak test. + `timers_alive` is how the threads-mode timer callback leak above was + found. - Fix the flaky `supervision::transient_startup_timeout_retries_after_actual_cleanup` (a 20 ms startup deadline raced the fast starts on a loaded host, which also made cargo-mutants report surviving mutants as caught): the replacement's diff --git a/benchmarks/hotpath.py b/benchmarks/hotpath.py index 8c24413ec..c7bfc0be7 100755 --- a/benchmarks/hotpath.py +++ b/benchmarks/hotpath.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Hot-path regression checks: a revision against its base. - hotpath.py gate [--base REV] [--head REV] [--threshold PCT] - hotpath.py abba [--base REV] [--head REV] [--reps N] [--cores LIST] + hotpath.py gate [--base REV] [--head REV] [--threshold PCT] [--features F] + hotpath.py abba [--base REV] [--head REV] [--reps N] [--cores LIST] [--features F] Both build `benchmarks/hotpath/driver.rs` (the head's copy, in both trees, so both sides run the same workload) as a release example of each revision. @@ -48,7 +48,7 @@ def default_base(head): sys.exit("no --base given and no main/origin/main to take a merge base with") -def build(work, label, rev): +def build(work, label, rev, features=None): """Check out `rev` in a worktree, add the driver, build it; return the binary.""" tree = work / label sh(["git", "worktree", "add", "--detach", "--force", str(tree), rev], cwd=ROOT, @@ -56,8 +56,9 @@ def build(work, label, rev): shutil.copy(DRIVER, tree / "concurrency" / "examples" / "hotpath.rs") target = work / f"target-{label}" # one per side: identical crate hashes would collide env = dict(os.environ, CARGO_TARGET_DIR=str(target), CARGO_INCREMENTAL="0") + extra = ["--features", features] if features else [] sh(["cargo", "build", "--release", "--locked", "-q", "-p", "spawned-concurrency", - "--example", "hotpath"], cwd=tree, env=env) + "--example", "hotpath", *extra], cwd=tree, env=env) binary = work / f"hotpath-{label}" shutil.copy(target / "release" / "examples" / "hotpath", binary) return binary @@ -193,6 +194,8 @@ def main(): p.add_argument("--ops", type=int, default=500_000, help="abba: operations per run") p.add_argument("--cores", help="abba: taskset core list, e.g. 12-15") p.add_argument("--csv", help="abba: raw samples") + p.add_argument("--features", help="cargo features for both sides, e.g. metrics " + "(default: the crate's defaults, as CI gates)") args = p.parse_args() args.reps = args.reps or (3 if args.mode == "gate" else 20) @@ -201,8 +204,8 @@ def main(): work = Path(args.work or tempfile.mkdtemp(prefix="hotpath-")).resolve() work.mkdir(parents=True, exist_ok=True) try: - base_bin = build(work, "base", base) - head_bin = build(work, "head", head) + base_bin = build(work, "base", base, args.features) + head_bin = build(work, "head", head, args.features) run = gate if args.mode == "gate" else abba sys.exit(run(args, base_bin, head_bin, base, head)) finally: diff --git a/concurrency/benches/soak/main.rs b/concurrency/benches/soak/main.rs index 0eb5cb1b8..b6f17fbb1 100644 --- a/concurrency/benches/soak/main.rs +++ b/concurrency/benches/soak/main.rs @@ -266,6 +266,18 @@ impl Recorder { m.mailbox_depth as f64, m.handlers_active as f64, m.requests_active as f64, + m.reply_slots_alive as f64, + m.reply_tokens_alive as f64, + m.pending_replies_entries as f64, + m.reply_registries_alive as f64, + m.deadline_heap_entries as f64, + m.deadline_service_entries as f64, + m.deadline_service_actors as f64, + m.jobs_alive as f64, + m.monitors_alive as f64, + m.link_records_alive as f64, + m.registered_names as f64, + m.timers_alive as f64, ]; debug_assert_eq!(values.len(), SERIES.len()); let sample = Sample { diff --git a/concurrency/benches/soak/report.rs b/concurrency/benches/soak/report.rs index 1de0cb36c..9d982b3c8 100644 --- a/concurrency/benches/soak/report.rs +++ b/concurrency/benches/soak/report.rs @@ -15,6 +15,18 @@ pub const SERIES: &[&str] = &[ "mailbox_depth", "handlers_active", "requests_active", + "reply_slots_alive", + "reply_tokens_alive", + "pending_replies_entries", + "reply_registries_alive", + "deadline_heap_entries", + "deadline_service_entries", + "deadline_service_actors", + "jobs_alive", + "monitors_alive", + "link_records_alive", + "registered_names", + "timers_alive", ]; /// Flat `key = number` lines (valid TOML with dotted keys); `#` comments. @@ -69,7 +81,7 @@ impl Phase { pub struct Sample { pub elapsed: f64, pub phase: Phase, - pub values: [f64; 11], + pub values: [f64; 23], pub started: u64, pub scenarios: u64, pub errors: u64, diff --git a/concurrency/benches/soak/thresholds.toml b/concurrency/benches/soak/thresholds.toml index 940e53a33..44c9d361e 100644 --- a/concurrency/benches/soak/thresholds.toml +++ b/concurrency/benches/soak/thresholds.toml @@ -26,6 +26,20 @@ trend.actors_active = 10 trend.mailbox_depth = 100 trend.handlers_active = 10 trend.requests_active = 10 +# spawned's retained-structure gauges (docs/GUIDE.md, "Retained internal +# structures"): counts per hour. +trend.reply_slots_alive = 10 +trend.reply_tokens_alive = 10 +trend.pending_replies_entries = 10 +trend.reply_registries_alive = 10 +trend.deadline_heap_entries = 10 +trend.deadline_service_entries = 10 +trend.deadline_service_actors = 10 +trend.jobs_alive = 10 +trend.monitors_alive = 10 +trend.link_records_alive = 10 +trend.registered_names = 10 +trend.timers_alive = 10 # Minimum growth: a trend also needs slope x steady-window hours above this # many units (MiB for rss_mb and heap_mb) to fail. Bounded pools (Tokio's @@ -44,6 +58,18 @@ growth.actors_active = 5 growth.mailbox_depth = 50 growth.handlers_active = 3 growth.requests_active = 3 +growth.reply_slots_alive = 5 +growth.reply_tokens_alive = 5 +growth.pending_replies_entries = 5 +growth.reply_registries_alive = 5 +growth.deadline_heap_entries = 5 +growth.deadline_service_entries = 5 +growth.deadline_service_actors = 5 +growth.jobs_alive = 5 +growth.monitors_alive = 5 +growth.link_records_alive = 5 +growth.registered_names = 5 +growth.timers_alive = 5 # Quiesce: after the workload stops and the long-lived ActorSystem shuts # down, each series must return to within `slack` of its baseline (sampled @@ -60,3 +86,15 @@ quiesce.actors_active = 0 quiesce.mailbox_depth = 0 quiesce.handlers_active = 0 quiesce.requests_active = 0 +quiesce.reply_slots_alive = 0 +quiesce.reply_tokens_alive = 0 +quiesce.pending_replies_entries = 0 +quiesce.reply_registries_alive = 0 +quiesce.deadline_heap_entries = 0 +quiesce.deadline_service_entries = 0 +quiesce.deadline_service_actors = 0 +quiesce.jobs_alive = 0 +quiesce.monitors_alive = 0 +quiesce.link_records_alive = 0 +quiesce.registered_names = 0 +quiesce.timers_alive = 0 diff --git a/concurrency/src/instrumentation.rs b/concurrency/src/instrumentation.rs index 88a5a524b..83b7f8633 100644 --- a/concurrency/src/instrumentation.rs +++ b/concurrency/src/instrumentation.rs @@ -1,9 +1,38 @@ //! Internal instrumentation; disabled builds use zero-sized no-op guards. + +/// A field counting one live `G` structure (`gauge::*`) from construction to +/// drop; zero-sized, and a no-op without `metrics`. +pub(crate) type Live = live::Live; #[cfg(feature = "metrics")] pub(crate) use crate::metrics::{ - accepted, hooks, reply, ActorObservation, Mailbox, MessageObservation, RequestObservation, + accepted, hooks, live, reply, ActorObservation, Mailbox, MessageObservation, RequestObservation, }; +/// Markers of the retained-structure gauges (`metrics::live`): which +/// `spawned_*` gauge a [`Live`](live::Live) field counts. +pub(crate) mod gauge { + /// `reply_slots_alive`: request reply slots. + pub(crate) enum ReplySlots {} + /// `reply_tokens_alive`: `ReplyTo` tokens (deferred obligations). + pub(crate) enum ReplyTokens {} + /// `reply_registries_alive`: per-instance reply registries. + pub(crate) enum ReplyRegistries {} + /// `pending_replies_entries`: entries in every `PendingReplies`. + pub(crate) enum PendingEntries {} + /// `deadline_heap_entries`: per-instance deadline heap entries. + pub(crate) enum DeadlineEntries {} + /// `jobs_alive`: job states (runs, handles, held results). + pub(crate) enum Jobs {} + /// `monitors_alive`: entries in monitor tables. + pub(crate) enum Monitors {} + /// `link_records_alive`: link records. + pub(crate) enum LinkRecords {} + /// `registered_names`: names in the global registry. + pub(crate) enum Names {} + /// `timers_alive`: `send_after` / `send_interval` timers. + pub(crate) enum Timers {} +} + #[cfg(not(feature = "metrics"))] mod disabled { use crate::{ActorError, ExitReason}; @@ -56,6 +85,22 @@ mod disabled { #[inline(always)] pub(crate) fn jobs_unfinished(_: usize) {} } + + /// Retained-structure gauges: zero-sized, no-op guards without `metrics`. + pub(crate) mod live { + use std::marker::PhantomData; + pub(crate) struct Live(PhantomData G>); + impl Default for Live { + #[inline(always)] + fn default() -> Self { + Self(PhantomData) + } + } + #[inline(always)] + pub(crate) fn added(_: usize) {} + #[inline(always)] + pub(crate) fn removed(_: usize) {} + } } #[cfg(not(feature = "metrics"))] pub(crate) use disabled::*; diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index ff5b35ef3..bf352223a 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -318,6 +318,8 @@ fn apply_signal( mod sync { pub(crate) use std::sync::{Arc, Mutex, MutexGuard, Weak}; + pub(crate) type LinkRecordAlive = + crate::instrumentation::Live; } mod table; pub(crate) use table::{new_link_table, LinkNotices, LinkOutcome, LinkTable}; diff --git a/concurrency/src/link/table.rs b/concurrency/src/link/table.rs index c7ce16e59..b54cf03e4 100644 --- a/concurrency/src/link/table.rs +++ b/concurrency/src/link/table.rs @@ -33,6 +33,8 @@ pub(crate) struct LinkEntry { pub peer_links: Weak>, /// A pending late `NoProc` notice (the peer was past D at `link`). pub late: bool, + /// `spawned_link_records_alive` (the Loom model: `()`). + _alive: super::sync::LinkRecordAlive, } /// An actor's exit table: its link records and its death point. @@ -118,6 +120,7 @@ pub(crate) fn install( peer: peer.clone(), peer_links: Arc::downgrade(peer_links), late: true, + _alive: Default::default(), }); } return LinkOutcome::NoProc; @@ -128,6 +131,7 @@ pub(crate) fn install( peer: peer.clone(), peer_links: Arc::downgrade(peer_links), late: false, + _alive: Default::default(), }); b.entries.push(LinkEntry { peer_id: own.id, @@ -135,6 +139,7 @@ pub(crate) fn install( peer: own.clone(), peer_links: Arc::downgrade(own_links), late: false, + _alive: Default::default(), }); LinkOutcome::Linked }) diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index 5c15904ec..41fe7dc2d 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -176,7 +176,7 @@ macro_rules! counters { #[derive(Debug, Clone)] #[non_exhaustive] pub struct Snapshot { $( pub $name: u64, )* pub queue_wait: HistogramSnapshot, pub handler_duration: HistogramSnapshot, pub request_latency: HistogramSnapshot, pub deferred_reply_latency: HistogramSnapshot } - pub fn snapshot() -> Snapshot { Snapshot { $( $name: COUNTERS.$name.load(Relaxed), )* queue_wait: QUEUE_WAIT.snapshot(), handler_duration: HANDLER_DURATION.snapshot(), request_latency: REQUEST_LATENCY.snapshot(), deferred_reply_latency: DEFERRED_REPLY_LATENCY.snapshot() } } + fn read_snapshot() -> Snapshot { Snapshot { $( $name: COUNTERS.$name.load(Relaxed), )* queue_wait: QUEUE_WAIT.snapshot(), handler_duration: HANDLER_DURATION.snapshot(), request_latency: REQUEST_LATENCY.snapshot(), deferred_reply_latency: DEFERRED_REPLY_LATENCY.snapshot() } } fn render_counters(out: &mut String, s: &Snapshot) { $( let name = concat!("spawned_", stringify!($name)); @@ -233,7 +233,34 @@ counters!( jobs_rejected_total, jobs_emits_total, jobs_unfinished_at_stop_total, + // Retained internal structures (`live`): each returns to its idle + // baseline once the work that created it is gone. + reply_slots_alive, + reply_tokens_alive, + reply_registries_alive, + pending_replies_entries, + deadline_heap_entries, + deadline_service_entries, + deadline_service_actors, + jobs_alive, + monitors_alive, + link_records_alive, + registered_names, + timers_alive, ); + +/// Process aggregates (see [`Snapshot`]). The deadline service's sizes are +/// read here, under its own lock; every other value is an atomic load. +pub fn snapshot() -> Snapshot { + let (entries, actors) = crate::reply::deadline::service_sizes(); + COUNTERS + .deadline_service_entries + .store(entries as u64, Relaxed); + COUNTERS + .deadline_service_actors + .store(actors as u64, Relaxed); + read_snapshot() +} static QUEUE_WAIT: Histogram = Histogram::new(); static HANDLER_DURATION: Histogram = Histogram::new(); static REQUEST_LATENCY: Histogram = Histogram::new(); @@ -367,6 +394,72 @@ pub(crate) mod hooks { } } +/// Retained-structure gauges: a zero-sized [`Live`](live::Live) field in each +/// structure counts it from construction to drop (no lock, one relaxed +/// atomic at each end, at points that already allocate or free it). +pub(crate) mod live { + use super::COUNTERS; + use crate::instrumentation::gauge::*; + use std::marker::PhantomData; + use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; + + pub(crate) trait Gauge { + fn gauge() -> &'static AtomicU64; + } + macro_rules! gauges { + ($($marker:ident => $field:ident),* $(,)?) => {$( + impl Gauge for $marker { + #[inline(always)] + fn gauge() -> &'static AtomicU64 { + &COUNTERS.$field + } + } + )*}; + } + gauges!( + ReplySlots => reply_slots_alive, + ReplyTokens => reply_tokens_alive, + ReplyRegistries => reply_registries_alive, + PendingEntries => pending_replies_entries, + DeadlineEntries => deadline_heap_entries, + Jobs => jobs_alive, + Monitors => monitors_alive, + LinkRecords => link_records_alive, + Names => registered_names, + Timers => timers_alive, + ); + + /// One live `G` structure, counted from construction to drop. + pub(crate) struct Live(PhantomData G>); + impl Default for Live { + #[inline] + fn default() -> Self { + G::gauge().fetch_add(1, Relaxed); + Self(PhantomData) + } + } + impl Drop for Live { + #[inline] + fn drop(&mut self) { + G::gauge().fetch_sub(1, Relaxed); + } + } + impl std::fmt::Debug for Live { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Live") + } + } + /// Structures not owned by one value (map entries): counted in bulk. + #[inline] + pub(crate) fn added(n: usize) { + G::gauge().fetch_add(n as u64, Relaxed); + } + #[inline] + pub(crate) fn removed(n: usize) { + G::gauge().fetch_sub(n as u64, Relaxed); + } +} + #[derive(Clone, Default)] pub(crate) struct Mailbox(Arc); impl Mailbox { diff --git a/concurrency/src/monitor.rs b/concurrency/src/monitor.rs index 2a7c5d66b..05d55d533 100644 --- a/concurrency/src/monitor.rs +++ b/concurrency/src/monitor.rs @@ -73,7 +73,17 @@ impl Message for Down { } mod sync { + use crate::instrumentation::{gauge::Monitors, live}; pub(crate) use std::sync::{Mutex, MutexGuard}; + /// `spawned_monitors_alive`: entries entering and leaving monitor tables. + #[inline(always)] + pub(crate) fn monitors_added(n: usize) { + live::added::(n); + } + #[inline(always)] + pub(crate) fn monitors_removed(n: usize) { + live::removed::(n); + } } pub(crate) mod fence; @@ -124,13 +134,14 @@ mod tests { fn dropped_registration_removes_only_its_own_entry() { // A waiter future that is rejected or aborted before it runs must not // leave its entry behind for the rest of the observer's lifetime. - let table: MonitorTable = Default::default(); + let table: MonitorTable = std::sync::Arc::new(fence::open()); let (mine, other) = (MonitorRef::next(), MonitorRef::next()); - let mut entries = std::collections::HashMap::new(); let token = spawned_rt::sync::CancellationToken::new; - entries.insert(mine, fence::Entry::Active(token())); - entries.insert(other, fence::Entry::Active(token())); - *table.lock().unwrap() = Some(entries); + for key in [mine, other] { + let registered = + fence::register(&table, key, token(), || false, || false, || {}, || {}); + assert_eq!(registered, fence::Registration::Active); + } drop(MonitorRegistration(table.clone(), mine)); let entries = table.lock().unwrap(); let entries = entries.as_ref().unwrap(); diff --git a/concurrency/src/monitor/fence.rs b/concurrency/src/monitor/fence.rs index f7bbaa27e..9b1992cfc 100644 --- a/concurrency/src/monitor/fence.rs +++ b/concurrency/src/monitor/fence.rs @@ -10,7 +10,7 @@ //! death point, record a trace event and enqueue a `Down` (which never takes //! a monitor table lock). -use super::sync::{Mutex, MutexGuard}; +use super::sync::{monitors_added, monitors_removed, Mutex, MutexGuard}; use std::collections::HashMap; use std::hash::Hash; @@ -72,10 +72,13 @@ pub(crate) fn register( if target_dead() { late(); entries.insert(key, Entry::Queued); + monitors_added(1); return Registration::Late; } entries.insert(key, Entry::Active(value)); active(); + // Keys are fresh `MonitorRef`s: an insert never replaces an entry. + monitors_added(1); Registration::Active } @@ -113,6 +116,7 @@ pub(crate) fn take_queued(table: &Entries, key: &K) -> bo }; if matches!(entries.get(key), Some(Entry::Queued)) { entries.remove(key); + monitors_removed(1); true } else { false @@ -134,6 +138,7 @@ pub(crate) fn remove( return None; } removed(); + monitors_removed(1); match entries.remove(key) { Some(Entry::Active(value)) => Some(value), _ => None, @@ -153,6 +158,7 @@ pub(crate) fn demonitor( ) -> Option { let mut guard = lock(table); let entry = guard.as_mut().and_then(|entries| entries.remove(key))?; + monitors_removed(1); removed(); match entry { Entry::Active(value) => Some(value), @@ -164,6 +170,7 @@ pub(crate) fn demonitor( /// waiters the caller wakes after the lock is released. pub(crate) fn close(table: &Entries) -> Option> { lock(table).take().map(|entries| { + monitors_removed(entries.len()); entries .into_iter() .filter_map(|(key, entry)| match entry { diff --git a/concurrency/src/registry.rs b/concurrency/src/registry.rs index c9ec1d6be..69510735c 100644 --- a/concurrency/src/registry.rs +++ b/concurrency/src/registry.rs @@ -6,6 +6,8 @@ use std::sync::{OnceLock, RwLock}; struct Entry { value: Box, owner: Option<(crate::ActorId, u64)>, + /// `spawned_registered_names`. + _alive: crate::instrumentation::Live, } type Store = RwLock>; static NEXT_REGISTRATION: AtomicU64 = AtomicU64::new(1); @@ -54,6 +56,7 @@ pub fn register( e.insert(Entry { value: Box::new(value), owner: None, + _alive: Default::default(), }); Ok(()) } @@ -114,6 +117,7 @@ pub fn register_owned( slot.insert(Entry { value: Box::new(value), owner: Some(owner), + _alive: Default::default(), }); Ok(RegistrationGuard { name: name.into(), diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 7fd354d88..bd699d4a0 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -33,6 +33,8 @@ struct Entry { at: u64, seq: u64, item: Weak, + /// `spawned_deadline_heap_entries`, stale entries included. + _alive: crate::instrumentation::Live, } impl PartialEq for Entry { fn eq(&self, other: &Self) -> bool { @@ -94,6 +96,7 @@ impl DeadlineHeap { at, seq: self.seq, item, + _alive: Default::default(), })); (earliest, upgraded) } @@ -159,6 +162,14 @@ pub(crate) fn schedule(registry: &Arc, at: u64) { thread_service::schedule(registry, at); } +/// The process-wide service's `(queue entries, actors)`: queue entries +/// include superseded ones not yet compacted. Read on a metrics scrape, +/// under the service's lock; `(0, 0)` before the service first starts. +#[cfg(feature = "metrics")] +pub(crate) fn service_sizes() -> (usize, usize) { + thread_service::sizes() +} + mod thread_service { use super::*; use std::collections::HashMap; @@ -254,6 +265,14 @@ mod thread_service { assert!(observed.lock().unwrap().upgrade().is_none()); } + #[cfg(feature = "metrics")] + pub(super) fn sizes() -> (usize, usize) { + SERVICE.get().map_or((0, 0), |service| { + let state = lock(service); + (state.queue.len(), state.actors.len()) + }) + } + pub(super) fn forget(key: u64) { let Some(service) = SERVICE.get() else { return; diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 79ee101f8..d96d2fed2 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -252,6 +252,9 @@ pub(crate) struct JobState { /// user payload, destroyed by [`JobSet::dispose_cancelled`] once the /// owner's death point answered and woke every caller. exit_released: Mutex>, + /// `spawned_jobs_alive`: from launch until the last reference (run, + /// handle, queued message, held result) is gone. + _alive: crate::instrumentation::Live, } /// Payloads an owner's exit took from a job, destroyed after its death point. @@ -278,6 +281,7 @@ impl JobState { emit_wakers: Mutex::new(Vec::new()), scheduled: Mutex::new(Vec::new()), exit_released: Mutex::new(None), + _alive: Default::default(), }) } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index db81bbb0f..b40c33331 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -63,6 +63,9 @@ mod sync { pub(crate) type SlotOwner = (); /// The owner's final reason, as the slot records it at its close. pub(crate) type Reason = crate::error::ExitReason; + /// Counts the slot in `spawned_reply_slots_alive` (the Loom model: `()`). + pub(crate) type SlotAlive = + crate::instrumentation::Live; /// Run a slot transition as one step of the lifecycle-trace /// linearization (the identity without the feature). @@ -601,6 +604,8 @@ pub struct ReplyTo { registry: Arc, /// When it was deferred (sampled; zero-sized without `metrics`). stamp: crate::instrumentation::hooks::DeferStamp, + /// `spawned_reply_tokens_alive`. + _alive: crate::instrumentation::Live, } impl std::fmt::Debug for ReplyTo { @@ -621,6 +626,7 @@ impl ReplyTo { slot: Some(slot), registry, stamp: crate::instrumentation::hooks::deferred(), + _alive: Default::default(), }; token.registry.token_added(); token diff --git a/concurrency/src/reply/pending.rs b/concurrency/src/reply/pending.rs index 1cb1c9c43..ab69d2ba1 100644 --- a/concurrency/src/reply/pending.rs +++ b/concurrency/src/reply/pending.rs @@ -75,9 +75,13 @@ impl InsertError { /// bounds it): `take` and `insert` are linear in the held entries. pub struct PendingReplies { cap: usize, - tokens: Vec<(K, ReplyTo)>, + tokens: Vec<(K, ReplyTo, Counted)>, } +/// An entry in `spawned_pending_replies_entries` (terminal, unpurged ones +/// included). +type Counted = crate::instrumentation::Live; + impl std::fmt::Debug for PendingReplies { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("PendingReplies") @@ -99,33 +103,36 @@ impl PendingReplies { /// token) or a full table hands the new token back. pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError> { self.purge(); - if self.tokens.iter().any(|(k, _)| *k == key) { + if self.tokens.iter().any(|(k, _, _)| *k == key) { return Err(InsertError::Duplicate(token)); } if self.tokens.len() >= self.cap { return Err(InsertError::Full(token)); } - self.tokens.push((key, token)); + self.tokens.push((key, token, Counted::default())); Ok(()) } /// Remove and return the live token under `key`; a terminal one is /// removed and skipped. pub fn take(&mut self, key: &K) -> Option> { - let at = self.tokens.iter().position(|(k, _)| k == key)?; - let (_, token) = self.tokens.remove(at); + let at = self.tokens.iter().position(|(k, _, _)| k == key)?; + let (_, token, _) = self.tokens.remove(at); (!token.is_closed()).then_some(token) } /// Remove every terminal token. fn purge(&mut self) { - self.tokens.retain(|(_, token)| !token.is_closed()); + self.tokens.retain(|(_, token, _)| !token.is_closed()); } /// Live (non-terminal) tokens held: a scan, so a token whose caller /// just left no longer counts. pub fn len(&self) -> usize { - self.tokens.iter().filter(|(_, t)| !t.is_closed()).count() + self.tokens + .iter() + .filter(|(_, t, _)| !t.is_closed()) + .count() } /// Whether no live token is held. @@ -138,6 +145,7 @@ impl PendingReplies { pub fn drain(&mut self) -> impl Iterator)> + '_ { self.tokens .drain(..) + .map(|(key, token, _)| (key, token)) .filter(|(_, token)| !token.is_closed()) } } diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index cf4503b1f..4017001e8 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -121,6 +121,8 @@ pub(crate) struct Registry { discharge: Mutex>>, #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService, + /// `spawned_reply_registries_alive`. + _alive: crate::instrumentation::Live, } fn lock(lists: &Mutex) -> std::sync::MutexGuard<'_, Lists> { @@ -169,6 +171,7 @@ impl Registry { } else { deadline::SimService::default() }, + _alive: Default::default(), }); keys() .lock() diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 6c5b63132..ae078ba15 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -312,6 +312,8 @@ pub(crate) struct Slot { /// The slot lock: the value and waker handoff, and the order of a /// deadline shortening against a reply's deadline read. inner: Mutex>, + /// `spawned_reply_slots_alive` (zero-sized; nothing without `metrics`). + _alive: super::sync::SlotAlive, } #[inline] @@ -379,6 +381,7 @@ impl Slot { payload: Payload::Empty, waker: None, }), + _alive: Default::default(), } } diff --git a/concurrency/src/tasks/time.rs b/concurrency/src/tasks/time.rs index ef0db0d03..de1316505 100644 --- a/concurrency/src/tasks/time.rs +++ b/concurrency/src/tasks/time.rs @@ -7,6 +7,9 @@ use super::actor::{Actor, Context, Handler}; use crate::message::Message; use core::pin::pin; +/// `spawned_timers_alive`: held by the timer task until it ends. +type TimerAlive = crate::instrumentation::Live; + /// Handle returned by [`send_after`] and [`send_interval`]. /// /// Cancel the timer by calling `timer.cancellation_token.cancel()`. @@ -25,6 +28,7 @@ where { let cancellation_token = CancellationToken::new(); let cloned_token = cancellation_token.clone(); + let alive = TimerAlive::default(); let actor_cancellation_token = ctx.cancellation_token(); let spawn_ctx = ctx.clone(); let join_handle = spawn_ctx.spawn_framework(async move { @@ -37,6 +41,7 @@ where let _ = ctx.send(msg); }); let _ = select(cancel_conditions, async_block).await; + let _held_until_here = &alive; }); TimerHandle { join_handle, @@ -57,6 +62,7 @@ where { let cancellation_token = CancellationToken::new(); let cloned_token = cancellation_token.clone(); + let alive = TimerAlive::default(); let actor_cancellation_token = ctx.cancellation_token(); let spawn_ctx = ctx.clone(); let join_handle = spawn_ctx.spawn_framework(async move { @@ -76,6 +82,7 @@ where futures::future::Either::Right(_) => (), } } + let _held_until_here = &alive; }); TimerHandle { join_handle, diff --git a/concurrency/src/threads/time.rs b/concurrency/src/threads/time.rs index 244b30b43..27975de20 100644 --- a/concurrency/src/threads/time.rs +++ b/concurrency/src/threads/time.rs @@ -6,6 +6,9 @@ use spawned_rt::threads::{self as rt, CancelRegistration, CancellationToken, Joi use super::actor::{Actor, Context, Handler}; use crate::message::Message; +/// `spawned_timers_alive`: held by the timer thread until it ends. +type TimerAlive = crate::instrumentation::Live; + /// Handle returned by [`send_after`] and [`send_interval`]. /// /// Cancel the timer by calling `timer.cancellation_token.cancel()`. @@ -48,9 +51,11 @@ where let actor_token = ctx.cancellation_token(); let (wake_rx, registrations) = wake_on_cancel(&timer_token, &actor_token); + let alive = TimerAlive::default(); let join_handle = rt::spawn(move || { let _registrations = registrations; + let _alive = alive; match wake_rx.recv_timeout(period) { Err(RecvTimeoutError::Timeout) => { if !timer_token.is_cancelled() && !actor_token.is_cancelled() { @@ -83,9 +88,11 @@ where let actor_token = ctx.cancellation_token(); let (wake_rx, registrations) = wake_on_cancel(&timer_token, &actor_token); + let alive = TimerAlive::default(); let join_handle = rt::spawn(move || { let _registrations = registrations; + let _alive = alive; while let Err(RecvTimeoutError::Timeout) = wake_rx.recv_timeout(period) { if timer_token.is_cancelled() || actor_token.is_cancelled() { break; diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 16721be4c..5e0b29cb8 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -19,6 +19,7 @@ mod sync { pub(crate) type SlotMutexGuard<'a, T> = MutexGuard<'a, T>; pub(crate) type SlotOwner = Option<()>; pub(crate) type Reason = (); + pub(crate) type SlotAlive = (); pub(crate) fn section(f: impl FnOnce() -> T) -> T { f() } diff --git a/concurrency/tests/link_fence_loom.rs b/concurrency/tests/link_fence_loom.rs index 3c450657e..605b167a4 100644 --- a/concurrency/tests/link_fence_loom.rs +++ b/concurrency/tests/link_fence_loom.rs @@ -24,6 +24,9 @@ mod sync { pub(crate) use loom::sync::{Mutex, MutexGuard}; pub(crate) use std::sync::{Arc, Weak}; + pub(crate) type LinkRecordAlive = (); + pub(crate) fn monitors_added(_: usize) {} + pub(crate) fn monitors_removed(_: usize) {} } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/concurrency/tests/metrics_retained.rs b/concurrency/tests/metrics_retained.rs new file mode 100644 index 000000000..4443a4fda --- /dev/null +++ b/concurrency/tests/metrics_retained.rs @@ -0,0 +1,521 @@ +//! Retained-structure gauges (`metrics` feature; docs/GUIDE.md, "Metrics"): +//! every gauge returns to its idle baseline once the actors that created +//! the structures are gone. This is also a leak test: a structure that +//! survives its actor (a slot, token, heap or service entry, job state, +//! monitor, link record, name or timer) keeps its gauge above the baseline. +//! +//! The churn exercises, in both backends: plain requests, raw +//! `send_request` receivers (owner deadline heap), deferred replies held in +//! `PendingReplies` (answered, and left for the death point), pipe jobs, +//! monitors and links in both stop orders, timers and registered names. +#![cfg(all(feature = "async", feature = "metrics", not(spawned_sim)))] +use spawned_concurrency::{ + message::Message, metrics, registry, tasks, threads, ChildHandle, Deadline, Down, + PendingReplies, PendingReply, +}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +/// The gauges are process-wide: the tests in this binary run one at a time. +static SERIAL: Mutex<()> = Mutex::new(()); + +const ACTORS: usize = 24; +/// Deferred requests expire quickly, so deadline-service entries of a dead +/// owner (removed when due) drain within the quiescence wait. +const PARK_DEADLINE: Duration = Duration::from_millis(200); + +type Gauge = (&'static str, fn(&metrics::Snapshot) -> u64); +const GAUGES: &[Gauge] = &[ + ("reply_slots_alive", |s| s.reply_slots_alive), + ("reply_tokens_alive", |s| s.reply_tokens_alive), + ("reply_registries_alive", |s| s.reply_registries_alive), + ("pending_replies_entries", |s| s.pending_replies_entries), + ("deadline_heap_entries", |s| s.deadline_heap_entries), + ("deadline_service_entries", |s| s.deadline_service_entries), + ("deadline_service_actors", |s| s.deadline_service_actors), + ("jobs_alive", |s| s.jobs_alive), + ("monitors_alive", |s| s.monitors_alive), + ("link_records_alive", |s| s.link_records_alive), + ("registered_names", |s| s.registered_names), + ("timers_alive", |s| s.timers_alive), + ("actors_active", |s| s.actors_active), + ("requests_active", |s| s.requests_active), + ("deferred_pending", |s| s.deferred_pending), + ("jobs_running", |s| s.jobs_running), + ("mailbox_depth", |s| s.mailbox_depth), +]; + +fn read() -> Vec { + let s = metrics::snapshot(); + GAUGES.iter().map(|(_, g)| g(&s)).collect() +} + +/// The gauges once nothing has changed for a while: the previous work's +/// asynchronous tail (timer threads ending, deadlines coming due) is over. +fn quiet() -> Vec { + let limit = Instant::now() + Duration::from_secs(10); + let mut last = read(); + let mut since = Instant::now(); + while since.elapsed() < PARK_DEADLINE * 2 && Instant::now() < limit { + std::thread::sleep(Duration::from_millis(10)); + let now = read(); + if now != last { + last = now; + since = Instant::now(); + } + } + last +} + +/// Wait until every gauge is back at `baseline`; fail naming the drifted ones. +fn settles_to(baseline: &[u64], what: &str) { + let limit = Instant::now() + Duration::from_secs(10); + loop { + let now = read(); + if now == baseline { + return; + } + if Instant::now() > limit { + let drift: Vec = GAUGES + .iter() + .zip(baseline.iter().zip(&now)) + .filter(|(_, (b, n))| b != n) + .map(|((name, _), (b, n))| format!("{name}: baseline {b}, now {n}")) + .collect(); + panic!("{what}: gauges did not return to baseline: {drift:?}"); + } + std::thread::sleep(Duration::from_millis(5)); + } +} + +/// The retained-structure gauges (the first `RETAINED` of `GAUGES`). +const RETAINED: usize = 12; + +/// Every retained-structure gauge rose during the round. +fn assert_exercised(before: &[u64], peak: &[u64]) { + for (i, (name, _)) in GAUGES.iter().enumerate().take(RETAINED) { + if *name == "deadline_service_actors" { + continue; // visited and removed quickly; seen through its entries + } + assert!( + peak[i] > before[i], + "{name} never rose during the round: the churn does not exercise it" + ); + } +} + +struct Ask; +impl Message for Ask { + type Result = u64; +} +struct Park; +impl Message for Park { + type Result = u64; +} +struct Flush; +impl Message for Flush { + type Result = (); +} +struct Watch(ChildHandle); +impl Message for Watch { + type Result = (); +} +struct StartJob; +impl Message for StartJob { + type Result = (); +} +struct JobDone; +impl Message for JobDone { + type Result = (); +} +#[derive(Clone)] +struct Tick; +impl Message for Tick { + type Result = (); +} + +// --------------------------------------------------------------------------- +// tasks +// --------------------------------------------------------------------------- + +struct AsyncWorker { + name: String, + parked: PendingReplies, + next: u64, +} +impl AsyncWorker { + fn new(name: String) -> Self { + Self { + name, + parked: PendingReplies::with_capacity(16), + next: 0, + } + } +} +impl tasks::Actor for AsyncWorker { + async fn started(&mut self, ctx: &tasks::Context) { + registry::register(&self.name, self.next).unwrap(); + tasks::send_after(Duration::from_millis(5), ctx.clone(), Tick); + tasks::send_interval(Duration::from_secs(3600), ctx.clone(), Tick); + } + async fn stopped(&mut self, _: &tasks::Context) { + registry::unregister(&self.name); + } +} +impl tasks::Handler for AsyncWorker { + async fn handle(&mut self, _: Ask, reply: PendingReply<'_, u64>, _: &tasks::Context) { + reply.reply(1); + } +} +impl tasks::Handler for AsyncWorker { + async fn handle(&mut self, _: Park, reply: PendingReply<'_, u64>, _: &tasks::Context) { + self.next += 1; + let token = reply.defer().unwrap(); + self.parked.insert(self.next, token).unwrap(); + } +} +impl tasks::Handler for AsyncWorker { + async fn handle(&mut self, _: Flush, reply: PendingReply<'_, ()>, _: &tasks::Context) { + for (key, token) in self.parked.drain() { + token.reply(key); + } + reply.reply(()); + } +} +impl tasks::Handler for AsyncWorker { + async fn handle(&mut self, w: Watch, reply: PendingReply<'_, ()>, ctx: &tasks::Context) { + ctx.monitor(&w.0); + ctx.link(&w.0); + reply.reply(()); + } +} +impl tasks::Handler for AsyncWorker { + async fn handle( + &mut self, + _: StartJob, + reply: PendingReply<'_, ()>, + ctx: &tasks::Context, + ) { + ctx.pipe( + tasks::JobOptions::new(), + |_| async { 7u64 }, + |_: tasks::JobOutcome| JobDone, + ) + .unwrap(); + ctx.pipe_blocking( + tasks::JobOptions::new(), + |_| 7u64, + |_: tasks::JobOutcome| JobDone, + ) + .unwrap(); + reply.reply(()); + } +} +impl tasks::Handler for AsyncWorker { + async fn handle(&mut self, _: JobDone, r: PendingReply<'_, ()>, _: &tasks::Context) { + r.reply(()); + } +} +impl tasks::Handler for AsyncWorker { + async fn handle(&mut self, _: Tick, r: PendingReply<'_, ()>, _: &tasks::Context) { + r.reply(()); + } +} +impl tasks::Handler for AsyncWorker { + async fn handle(&mut self, _: Down, r: PendingReply<'_, ()>, _: &tasks::Context) { + r.reply(()); + } +} + +async fn async_pair(round: usize, i: usize) { + use tasks::ActorStart; + let peer = AsyncWorker::new(format!("retained-a-{round}-{i}-peer")).start(); + let worker = AsyncWorker::new(format!("retained-a-{round}-{i}")).start(); + worker.request(Watch(peer.child_handle())).await.unwrap(); + for _ in 0..4 { + assert_eq!(worker.request(Ask).await.unwrap(), 1); + } + // A raw receiver awaited while pending puts its deadline in the + // owner's heap. + let raw = worker + .send_request(Ask, Deadline::after(PARK_DEADLINE)) + .unwrap(); + assert_eq!(raw.await.unwrap(), 1); + let parked: Vec<_> = (0..3) + .map(|_| worker.send_request(Park, PARK_DEADLINE).unwrap()) + .collect(); + worker.request(Flush).await.unwrap(); + for rx in parked { + assert!(rx.await.is_ok()); + } + worker.request(StartJob).await.unwrap(); + // Left deferred: answered at the death point. + let orphan = worker.send_request(Park, PARK_DEADLINE).unwrap(); + worker.request(Ask).await.unwrap(); + let (first, second) = if i.is_multiple_of(2) { + (&worker, &peer) + } else { + (&peer, &worker) + }; + first.context().stop(); + first.join().await; + second.context().stop(); + second.join().await; + assert!(orphan.await.is_err()); +} + +fn async_round(rt: &tokio::runtime::Runtime, round: usize) { + rt.block_on(futures::future::join_all( + (0..ACTORS).map(|i| async_pair(round, i)), + )); +} + +// --------------------------------------------------------------------------- +// threads +// --------------------------------------------------------------------------- + +struct NativeWorker { + name: String, + parked: PendingReplies, + next: u64, +} +impl NativeWorker { + fn new(name: String) -> Self { + Self { + name, + parked: PendingReplies::with_capacity(16), + next: 0, + } + } +} +impl threads::Actor for NativeWorker { + fn started(&mut self, ctx: &threads::Context) { + registry::register(&self.name, self.next).unwrap(); + threads::send_after(Duration::from_millis(5), ctx.clone(), Tick); + threads::send_interval(Duration::from_secs(3600), ctx.clone(), Tick); + } + fn stopped(&mut self, _: &threads::Context) { + registry::unregister(&self.name); + } +} +impl threads::Handler for NativeWorker { + fn handle(&mut self, _: Ask, reply: PendingReply<'_, u64>, _: &threads::Context) { + reply.reply(1); + } +} +impl threads::Handler for NativeWorker { + fn handle(&mut self, _: Park, reply: PendingReply<'_, u64>, _: &threads::Context) { + self.next += 1; + let token = reply.defer().unwrap(); + self.parked.insert(self.next, token).unwrap(); + } +} +impl threads::Handler for NativeWorker { + fn handle(&mut self, _: Flush, reply: PendingReply<'_, ()>, _: &threads::Context) { + for (key, token) in self.parked.drain() { + token.reply(key); + } + reply.reply(()); + } +} +impl threads::Handler for NativeWorker { + fn handle(&mut self, w: Watch, reply: PendingReply<'_, ()>, ctx: &threads::Context) { + ctx.monitor(&w.0); + ctx.link(&w.0); + reply.reply(()); + } +} +impl threads::Handler for NativeWorker { + fn handle(&mut self, _: StartJob, reply: PendingReply<'_, ()>, ctx: &threads::Context) { + ctx.pipe_blocking( + threads::JobOptions::new(), + |_| 7u64, + |_: threads::JobOutcome| JobDone, + ) + .unwrap(); + reply.reply(()); + } +} +impl threads::Handler for NativeWorker { + fn handle(&mut self, _: JobDone, r: PendingReply<'_, ()>, _: &threads::Context) { + r.reply(()); + } +} +impl threads::Handler for NativeWorker { + fn handle(&mut self, _: Tick, r: PendingReply<'_, ()>, _: &threads::Context) { + r.reply(()); + } +} +impl threads::Handler for NativeWorker { + fn handle(&mut self, _: Down, r: PendingReply<'_, ()>, _: &threads::Context) { + r.reply(()); + } +} + +fn native_pair(round: usize, i: usize) { + use threads::ActorStart; + let peer = NativeWorker::new(format!("retained-n-{round}-{i}-peer")).start(); + let worker = NativeWorker::new(format!("retained-n-{round}-{i}")).start(); + worker.request(Watch(peer.child_handle())).unwrap(); + for _ in 0..4 { + assert_eq!(worker.request(Ask).unwrap(), 1); + } + let parked: Vec<_> = (0..3) + .map(|_| worker.send_request(Park, PARK_DEADLINE).unwrap()) + .collect(); + worker.request(Flush).unwrap(); + for rx in parked { + assert!(rx.recv().is_ok()); + } + worker.request(StartJob).unwrap(); + let orphan = worker.send_request(Park, PARK_DEADLINE).unwrap(); + worker.request(Ask).unwrap(); + let (first, second) = if i.is_multiple_of(2) { + (&worker, &peer) + } else { + (&peer, &worker) + }; + first.context().stop(); + first.join(); + second.context().stop(); + second.join(); + assert!(orphan.recv().is_err()); +} + +fn native_round(round: usize) { + std::thread::scope(|scope| { + for i in 0..ACTORS { + scope.spawn(move || native_pair(round, i)); + } + }); +} + +// --------------------------------------------------------------------------- +// tests +// --------------------------------------------------------------------------- + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap() +} + +/// Run `round` once to warm process-wide lazies (deadline service, detached +/// registry), then measure: the round must raise every gauge, and every +/// gauge must settle back to the baseline. +fn churn_returns_to_baseline(what: &str, mut round: impl FnMut(usize)) { + round(0); + let baseline = quiet(); + // Sample the peak from a watcher while the round runs. + let peak = Mutex::new(vec![0u64; GAUGES.len()]); + let done = std::sync::atomic::AtomicBool::new(false); + std::thread::scope(|scope| { + scope.spawn(|| { + while !done.load(std::sync::atomic::Ordering::Relaxed) { + let now = read(); + let mut peak = peak.lock().unwrap(); + for (p, n) in peak.iter_mut().zip(now) { + *p = (*p).max(n); + } + drop(peak); + std::thread::sleep(Duration::from_micros(200)); + } + }); + round(1); + done.store(true, std::sync::atomic::Ordering::Relaxed); + }); + let peak = peak.into_inner().unwrap(); + assert_exercised(&baseline, &peak); + settles_to(&baseline, what); +} + +#[test] +fn tasks_churn_returns_every_gauge_to_baseline() { + let _serial = SERIAL.lock().unwrap_or_else(|p| p.into_inner()); + let rt = runtime(); + churn_returns_to_baseline("tasks churn", |round| async_round(&rt, round)); +} + +#[test] +fn threads_churn_returns_every_gauge_to_baseline() { + let _serial = SERIAL.lock().unwrap_or_else(|p| p.into_inner()); + churn_returns_to_baseline("threads churn", native_round); +} + +/// Repeated churn in both backends for a bounded time +/// (`SPAWNED_RETAINED_CHURN_SECS`, default 6): after every few rounds every +/// gauge is back at the baseline, so nothing accumulates across rounds. +#[test] +fn long_churn_does_not_drift() { + let _serial = SERIAL.lock().unwrap_or_else(|p| p.into_inner()); + let secs = std::env::var("SPAWNED_RETAINED_CHURN_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(6); + let rt = runtime(); + async_round(&rt, 0); + native_round(0); + let baseline = quiet(); + let end = Instant::now() + Duration::from_secs(secs); + let mut round = 1; + while Instant::now() < end { + for _ in 0..4 { + async_round(&rt, round); + native_round(round); + round += 1; + } + settles_to(&baseline, &format!("after {round} rounds")); + } + assert!(round > 4, "the churn ran at least one batch"); +} + +/// A long-lived threads actor arming timers: `timers_alive` returns to its +/// baseline while the actor keeps running (each timer's registration on the +/// actor's stop token used to stay until the actor stopped). +#[test] +fn threads_timers_do_not_accumulate_on_a_running_actor() { + let _serial = SERIAL.lock().unwrap_or_else(|p| p.into_inner()); + use threads::ActorStart; + // The worker arms a short `send_after` (fires) and a long + // `send_interval` (lives until it stops) itself. + let before = quiet()[11]; + let baseline = before + 1; + let actor = NativeWorker::new("retained-timer-host".into()).start(); + for _ in 0..100 { + threads::send_after(Duration::from_millis(1), actor.context(), Tick); + } + let cancelled: Vec<_> = (0..50) + .map(|_| threads::send_interval(Duration::from_secs(3600), actor.context(), Tick)) + .collect(); + for timer in &cancelled { + timer.cancellation_token.cancel(); + } + let limit = Instant::now() + Duration::from_secs(10); + while metrics::snapshot().timers_alive != baseline && Instant::now() < limit { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(metrics::snapshot().timers_alive, baseline); + actor.request(Ask).unwrap(); + actor.context().stop(); + actor.join(); + // Leave nothing behind for the next test's baseline. + let limit = Instant::now() + Duration::from_secs(10); + while metrics::snapshot().timers_alive != before && Instant::now() < limit { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(metrics::snapshot().timers_alive, before); +} + +/// The gauges are exported as Prometheus gauges. +#[test] +fn retained_gauges_are_exported() { + let text = metrics::render_prometheus(); + for (name, _) in GAUGES { + assert!( + text.contains(&format!("# TYPE spawned_{name} gauge\n")), + "spawned_{name} is not exported as a gauge" + ); + } +} diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 2681172de..d41b5a224 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -14,6 +14,7 @@ mod sync { pub(crate) type SlotMutexGuard<'a, T> = MutexGuard<'a, T>; pub(crate) type SlotOwner = Option<()>; pub(crate) type Reason = (); + pub(crate) type SlotAlive = (); pub(crate) fn section(f: impl FnOnce() -> T) -> T { f() } diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 589b3f8a2..f0e58497d 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -1589,6 +1589,40 @@ handler future destroyed without returning an outcome counts as aborted; this does not imply its business effects were rolled back. Native blocking handlers still stop cooperatively. +### Retained internal structures + +These gauges count what spawned itself keeps in memory, so an operator can +tell from one scrape whether memory growth is inside the framework: a gauge +that climbs with uptime while the actor and request counts stay flat points +at a leak (or an application that holds framework objects), with no heap +profiling build. Each one is counted where its structure is created and +destroyed (a zero-sized field in the structure: one relaxed atomic at each +end, no lock), except the deadline service's two sizes, read at scrape under +the service's own lock. All of them are compiled out without the feature. + +| Gauge (`spawned_*`) | Counts | Returns to baseline when | +|---|---|---| +| `reply_slots_alive` | Reply slots of requests (`request`, `send_request`, protocol calls): held by the envelope, the handler's `PendingReply`/`ReplyTo` and the caller's receiver. | Every holder is gone; a stored, never-awaited `ReplyReceiver` keeps its slot. | +| `reply_tokens_alive` | `ReplyTo` tokens (deferred obligations), answered or not. Unlike `deferred_pending`, a token whose caller already got an answer (expired, abandoned) still counts until it is dropped. | Tokens are answered or dropped. | +| `pending_replies_entries` | Entries in every `PendingReplies` table, terminal entries not yet purged included. | Entries are taken, drained or purged by the next insert, or the table is dropped. | +| `reply_registries_alive` | Per-instance reply registries: one per live actor instance, kept while anything references it (an `ActorRef`, a `ReplyTo`, a job). | Every handle to dead instances is dropped. | +| `deadline_heap_entries` | Entries in the per-instance deadline heaps (deferred tokens, raw receivers awaited while pending, job deadlines and `emit_at`), stale and superseded entries not yet compacted included. | Entries are due, compacted (when stale entries outnumber live ones), or the owner dies. | +| `deadline_service_entries` | The process-wide deadline service's queue, superseded entries and entries of dead owners included (dropped when due, or compacted at twice the live owners plus 64). | Their deadlines pass. | +| `deadline_service_actors` | Owners the deadline service will visit. | Their earliest deadline is serviced or the owner dies. | +| `jobs_alive` | Job states (`pipe`, `pipe_blocking`, `pipe_reply*`): running work, a held or queued result, a `JobHandle` or a queued job message keeps one. `jobs_running` counts only executing work. | The job ended and its handle and messages are gone. | +| `monitors_alive` | Entries in observers' monitor tables: active monitors and `Down` notices queued but not yet handled. | Monitors are demonitored, their `Down` is handled, or the observer exits. | +| `link_records_alive` | Link records (two per link, one per pending late `NoProc` notice), including records taken at a death point until their notices are delivered. | Links are removed or either end exits. | +| `registered_names` | Names in the global registry (`registry::register`, `register_owned`, `ctx.register_name`). | Names are unregistered (managed names: when the owner's cleanup ends). | +| `timers_alive` | `send_after` / `send_interval` timers whose task or thread is still running (with it, in `threads` mode, its callback registrations on the actor's and its own stop tokens). | Timers fire or are cancelled, or the actor stops. | + +Gauges are process-wide sums and never labelled. They have idle baselines +above zero (the detached registry, registered names an application keeps, an +interval timer): compare against the same process at rest, or alert on +growth. Process-wide one-time structures (the deadline-service thread, the +global registry map) are not counted. The disposal worker pool proposed with +the deadline-service fix (#282) will add its queue length and worker count +here when it lands. + ### Caller and raw receiver boundaries Direct `ActorRef::request`, `Context::request`, and erased-recipient `request` @@ -1632,8 +1666,7 @@ outside the library. Histogram sums saturate at `u64::MAX` nanoseconds; counters can wrap after `u64::MAX` events. Scrapes are approximate concurrent observations. There is no built-in HTTP server, OpenTelemetry exporter, causal tracing, -oldest-queued-message age, per-actor histogram or active-monitor gauge in this -initial implementation. These can be evaluated independently of the existing +oldest-queued-message age or per-actor histogram in this implementation. These can be evaluated independently of the existing metrics and without inserting exporter work into handlers. Use application measurements before choosing a sampling rate; shared atomic diff --git a/docs/TESTING.md b/docs/TESTING.md index 1bbef729d..eaee64e1b 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -2029,6 +2029,7 @@ times. | `registry` | `registry::registered().len()` | | `actors_live` | harness actor states constructed and not yet dropped (started minus fully stopped) | | `actors_active`, `mailbox_depth`, `handlers_active`, `requests_active` | spawned's own `metrics` gauges | +| `reply_slots_alive` … `timers_alive` | spawned's retained-structure gauges (GUIDE, "Retained internal structures"): reply slots, tokens, `PendingReplies` entries, reply registries, deadline heap and service entries, service owners, jobs, monitors, link records, names, timers | | `actors_started`, `scenarios`, `errors` | progress counters (not judged) | Framework-internal `ActorRef` and `Context` clones are not countable through @@ -2113,8 +2114,13 @@ gate too; they cannot inherit a passing source check without measurement. ```sh python3 benchmarks/hotpath.py gate # HEAD vs merge base with main python3 benchmarks/hotpath.py gate --base origin/main --head origin/my-branch --threshold 3 +python3 benchmarks/hotpath.py gate --features metrics # both sides with the metrics feature ``` +CI gates the default build. `--features` builds both sides with extra +features, for a change whose cost sits behind one (instrumentation under +`metrics`); run it by hand and quote it in the PR. + Instruction counts miss costs that are not instructions: lock contention, extra wake-ups and parking, cache-line transfers. A change that moves work between threads can pass the gate and still be slower, so the wall-clock diff --git a/verification/planted/patches/15-mon-down-after-demonitor.patch b/verification/planted/patches/15-mon-down-after-demonitor.patch index da8ad616f..d6e29e452 100644 --- a/verification/planted/patches/15-mon-down-after-demonitor.patch +++ b/verification/planted/patches/15-mon-down-after-demonitor.patch @@ -29,8 +29,9 @@ index f7bbaa27..d7778cf3 100644 }; if active { deliver(); -@@ -155,7 +162,8 @@ pub(crate) fn demonitor( +@@ -164,8 +171,9 @@ pub(crate) fn demonitor( let entry = guard.as_mut().and_then(|entries| entries.remove(key))?; + monitors_removed(1); removed(); match entry { - Entry::Active(value) => Some(value), diff --git a/verification/planted/patches/16-mon-dead-target-no-down.patch b/verification/planted/patches/16-mon-dead-target-no-down.patch index b12e950f3..503cdc169 100644 --- a/verification/planted/patches/16-mon-dead-target-no-down.patch +++ b/verification/planted/patches/16-mon-dead-target-no-down.patch @@ -4,12 +4,13 @@ diff --git a/concurrency/src/monitor/fence.rs b/concurrency/src/monitor/fence.rs index f7bbaa27..d40d7c7e 100644 --- a/concurrency/src/monitor/fence.rs +++ b/concurrency/src/monitor/fence.rs -@@ -70,9 +70,8 @@ pub(crate) fn register( +@@ -70,10 +70,8 @@ pub(crate) fn register( return Registration::Inert; }; if target_dead() { - late(); - entries.insert(key, Entry::Queued); +- monitors_added(1); - return Registration::Late; + let _ = late; + return Registration::Inert;