Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,15 @@
its baseline (`tests/metrics_retained.rs`), which doubles as a leak test.
`timers_alive` is how the threads-mode timer callback leak above was
found.
- `metrics`: the disposal worker pool's gauges, `spawned_disposal_queue_entries`,
`disposal_workers_alive` and `disposal_workers_idle` (read at scrape under
the pool's lock), and the counter `spawned_disposal_workers_capped_total`
(times all 64 workers were busy, once per episode, beside the existing
warning). Compiled out without the feature; no change on the hand-off path
otherwise. A test expires deadlines in both backends with blocking and
non-blocking disposal and checks the queue drains and every worker retires
after its keep-alive (`disposal.keep_alive` failpoint with `failpoints`).
The soak harness judges the three gauges.
- Fix the flaky `supervision::transient_startup_timeout_retries_after_actual_cleanup`
(a 20 ms startup deadline raced the fast starts on a loaded host, which also
made cargo-mutants report surviving mutants as caught): the replacement's
Expand Down
3 changes: 3 additions & 0 deletions concurrency/benches/soak/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,9 @@ impl Recorder {
m.link_records_alive as f64,
m.registered_names as f64,
m.timers_alive as f64,
m.disposal_queue_entries as f64,
m.disposal_workers_alive as f64,
m.disposal_workers_idle as f64,
];
debug_assert_eq!(values.len(), SERIES.len());
let sample = Sample {
Expand Down
5 changes: 4 additions & 1 deletion concurrency/benches/soak/report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ pub const SERIES: &[&str] = &[
"link_records_alive",
"registered_names",
"timers_alive",
"disposal_queue_entries",
"disposal_workers_alive",
"disposal_workers_idle",
];

/// Flat `key = number` lines (valid TOML with dotted keys); `#` comments.
Expand Down Expand Up @@ -81,7 +84,7 @@ impl Phase {
pub struct Sample {
pub elapsed: f64,
pub phase: Phase,
pub values: [f64; 23],
pub values: [f64; 26],
pub started: u64,
pub scenarios: u64,
pub errors: u64,
Expand Down
10 changes: 10 additions & 0 deletions concurrency/benches/soak/thresholds.toml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ trend.monitors_alive = 10
trend.link_records_alive = 10
trend.registered_names = 10
trend.timers_alive = 10
trend.disposal_queue_entries = 10
trend.disposal_workers_alive = 10
trend.disposal_workers_idle = 10

# Minimum growth: a trend also needs slope x steady-window hours above this
# many units (MiB for rss_mb and heap_mb) to fail. Bounded pools (Tokio's
Expand Down Expand Up @@ -70,6 +73,9 @@ growth.monitors_alive = 5
growth.link_records_alive = 5
growth.registered_names = 5
growth.timers_alive = 5
growth.disposal_queue_entries = 5
growth.disposal_workers_alive = 5
growth.disposal_workers_idle = 5

# Quiesce: after the workload stops and the long-lived ActorSystem shuts
# down, each series must return to within `slack` of its baseline (sampled
Expand Down Expand Up @@ -98,3 +104,7 @@ quiesce.monitors_alive = 0
quiesce.link_records_alive = 0
quiesce.registered_names = 0
quiesce.timers_alive = 0
# Idle disposal workers retire after their 10 s keep-alive, within the timeout.
quiesce.disposal_queue_entries = 0
quiesce.disposal_workers_alive = 0
quiesce.disposal_workers_idle = 0
2 changes: 2 additions & 0 deletions concurrency/src/instrumentation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ mod disabled {
pub(crate) fn job_emitted() {}
#[inline(always)]
pub(crate) fn jobs_unfinished(_: usize) {}
#[inline(always)]
pub(crate) fn disposal_capped() {}
}

/// Retained-structure gauges: zero-sized, no-op guards without `metrics`.
Expand Down
22 changes: 20 additions & 2 deletions concurrency/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -247,10 +247,15 @@ counters!(
link_records_alive,
registered_names,
timers_alive,
disposal_queue_entries,
disposal_workers_alive,
disposal_workers_idle,
disposal_workers_capped_total,
);

/// Process aggregates (see [`Snapshot`]). The deadline service's sizes are
/// read here, under its own lock; every other value is an atomic load.
/// Process aggregates (see [`Snapshot`]). The deadline service's and the
/// disposal pool's sizes are read here, each under its own lock; every other
/// value is an atomic load.
pub fn snapshot() -> Snapshot {
let (entries, actors) = crate::reply::deadline::service_sizes();
COUNTERS
Expand All @@ -259,6 +264,14 @@ pub fn snapshot() -> Snapshot {
COUNTERS
.deadline_service_actors
.store(actors as u64, Relaxed);
let (queued, workers, idle) = crate::reply::disposal::pool_sizes();
COUNTERS
.disposal_queue_entries
.store(queued as u64, Relaxed);
COUNTERS
.disposal_workers_alive
.store(workers as u64, Relaxed);
COUNTERS.disposal_workers_idle.store(idle as u64, Relaxed);
read_snapshot()
}
static QUEUE_WAIT: Histogram = Histogram::new();
Expand Down Expand Up @@ -392,6 +405,11 @@ pub(crate) mod hooks {
.jobs_unfinished_at_stop_total
.fetch_add(n as u64, Relaxed);
}
/// Every disposal worker was busy at the cap (counted once per episode,
/// like the `disposal_workers_capped` warning).
pub(crate) fn disposal_capped() {
COUNTERS.disposal_workers_capped_total.fetch_add(1, Relaxed);
}
}

/// Retained-structure gauges: a zero-sized [`Live`](live::Live) field in each
Expand Down
32 changes: 24 additions & 8 deletions concurrency/src/reply/disposal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -240,11 +240,24 @@ struct Pool {
ready: Condvar,
}

static POOL: OnceLock<Arc<Pool>> = OnceLock::new();

fn pool() -> &'static Arc<Pool> {
static POOL: OnceLock<Arc<Pool>> = OnceLock::new();
POOL.get_or_init(Default::default)
}

/// The pool's `(queued items, workers, idle workers)`: workers include one
/// still starting, idle ones one woken and not yet back under the lock. Read
/// on a metrics scrape, under the pool's lock; `(0, 0, 0)` before the first
/// hand-off.
#[cfg(feature = "metrics")]
pub(crate) fn pool_sizes() -> (usize, usize, usize) {
POOL.get().map_or((0, 0, 0), |pool| {
let state = lock(pool);
(state.queue.len(), state.workers, state.idle)
})
}

fn lock(pool: &Pool) -> std::sync::MutexGuard<'_, State> {
pool.state.lock().unwrap_or_else(|p| p.into_inner())
}
Expand Down Expand Up @@ -291,14 +304,17 @@ pub(crate) fn submit(work: Vec<Item>) {
fn start_worker(pool: &Arc<Pool>, start: Start) {
match start {
Start::No => {}
Start::Capped => crate::actor_cleanup::log_safely(|| {
tracing::warn!(
event = "disposal_workers_capped",
max = MAX_WORKERS,
"every deadline disposal worker is busy: later wake-ups and \
Start::Capped => {
crate::instrumentation::hooks::disposal_capped();
crate::actor_cleanup::log_safely(|| {
tracing::warn!(
event = "disposal_workers_capped",
max = MAX_WORKERS,
"every deadline disposal worker is busy: later wake-ups and \
payload drops queue until one returns"
);
}),
);
})
}
Start::Yes => {
let worker = pool.clone();
let spawned = std::thread::Builder::new()
Expand Down
Loading
Loading