Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,20 @@
`callbacks_registered`; the timer thread now holds its registrations and
drops them when it fires or is cancelled. Tasks-mode timers were not
affected (Tokio removes a cancellation waiter when its future is dropped).
- `metrics`: gauges of spawned's own retained structures, so memory growth
can be attributed to the framework (or ruled out) from one scrape, without a
heap-profiling build: `spawned_reply_slots_alive`, `reply_tokens_alive`,
`pending_replies_entries`, `reply_registries_alive`,
`deadline_heap_entries`, `deadline_service_entries`,
`deadline_service_actors`, `jobs_alive`, `monitors_alive`,
`link_records_alive`, `registered_names` and `timers_alive` (docs/GUIDE.md,
"Retained internal structures"). Counted by a zero-sized field at each
structure's creation and drop (one relaxed atomic each end, no lock); the
deadline service's sizes are read at scrape. Compiled out without the
feature. A churn test in both backends checks that every gauge returns to
its baseline (`tests/metrics_retained.rs`), which doubles as a leak test.
`timers_alive` is how the threads-mode timer callback leak above was
found.
- Fix the flaky `supervision::transient_startup_timeout_retries_after_actual_cleanup`
(a 20 ms startup deadline raced the fast starts on a loaded host, which also
made cargo-mutants report surviving mutants as caught): the replacement's
Expand Down
15 changes: 9 additions & 6 deletions benchmarks/hotpath.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
"""Hot-path regression checks: a revision against its base.

hotpath.py gate [--base REV] [--head REV] [--threshold PCT]
hotpath.py abba [--base REV] [--head REV] [--reps N] [--cores LIST]
hotpath.py gate [--base REV] [--head REV] [--threshold PCT] [--features F]
hotpath.py abba [--base REV] [--head REV] [--reps N] [--cores LIST] [--features F]

Both build `benchmarks/hotpath/driver.rs` (the head's copy, in both trees, so
both sides run the same workload) as a release example of each revision.
Expand Down Expand Up @@ -48,16 +48,17 @@ def default_base(head):
sys.exit("no --base given and no main/origin/main to take a merge base with")


def build(work, label, rev):
def build(work, label, rev, features=None):
"""Check out `rev` in a worktree, add the driver, build it; return the binary."""
tree = work / label
sh(["git", "worktree", "add", "--detach", "--force", str(tree), rev], cwd=ROOT,
capture_output=True)
shutil.copy(DRIVER, tree / "concurrency" / "examples" / "hotpath.rs")
target = work / f"target-{label}" # one per side: identical crate hashes would collide
env = dict(os.environ, CARGO_TARGET_DIR=str(target), CARGO_INCREMENTAL="0")
extra = ["--features", features] if features else []
sh(["cargo", "build", "--release", "--locked", "-q", "-p", "spawned-concurrency",
"--example", "hotpath"], cwd=tree, env=env)
"--example", "hotpath", *extra], cwd=tree, env=env)
binary = work / f"hotpath-{label}"
shutil.copy(target / "release" / "examples" / "hotpath", binary)
return binary
Expand Down Expand Up @@ -193,6 +194,8 @@ def main():
p.add_argument("--ops", type=int, default=500_000, help="abba: operations per run")
p.add_argument("--cores", help="abba: taskset core list, e.g. 12-15")
p.add_argument("--csv", help="abba: raw samples")
p.add_argument("--features", help="cargo features for both sides, e.g. metrics "
"(default: the crate's defaults, as CI gates)")
args = p.parse_args()
args.reps = args.reps or (3 if args.mode == "gate" else 20)

Expand All @@ -201,8 +204,8 @@ def main():
work = Path(args.work or tempfile.mkdtemp(prefix="hotpath-")).resolve()
work.mkdir(parents=True, exist_ok=True)
try:
base_bin = build(work, "base", base)
head_bin = build(work, "head", head)
base_bin = build(work, "base", base, args.features)
head_bin = build(work, "head", head, args.features)
run = gate if args.mode == "gate" else abba
sys.exit(run(args, base_bin, head_bin, base, head))
finally:
Expand Down
12 changes: 12 additions & 0 deletions concurrency/benches/soak/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,18 @@ impl Recorder {
m.mailbox_depth as f64,
m.handlers_active as f64,
m.requests_active as f64,
m.reply_slots_alive as f64,
m.reply_tokens_alive as f64,
m.pending_replies_entries as f64,
m.reply_registries_alive as f64,
m.deadline_heap_entries as f64,
m.deadline_service_entries as f64,
m.deadline_service_actors as f64,
m.jobs_alive as f64,
m.monitors_alive as f64,
m.link_records_alive as f64,
m.registered_names as f64,
m.timers_alive as f64,
];
debug_assert_eq!(values.len(), SERIES.len());
let sample = Sample {
Expand Down
14 changes: 13 additions & 1 deletion concurrency/benches/soak/report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,18 @@ pub const SERIES: &[&str] = &[
"mailbox_depth",
"handlers_active",
"requests_active",
"reply_slots_alive",
"reply_tokens_alive",
"pending_replies_entries",
"reply_registries_alive",
"deadline_heap_entries",
"deadline_service_entries",
"deadline_service_actors",
"jobs_alive",
"monitors_alive",
"link_records_alive",
"registered_names",
"timers_alive",
];

/// Flat `key = number` lines (valid TOML with dotted keys); `#` comments.
Expand Down Expand Up @@ -69,7 +81,7 @@ impl Phase {
pub struct Sample {
pub elapsed: f64,
pub phase: Phase,
pub values: [f64; 11],
pub values: [f64; 23],
pub started: u64,
pub scenarios: u64,
pub errors: u64,
Expand Down
38 changes: 38 additions & 0 deletions concurrency/benches/soak/thresholds.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,20 @@ trend.actors_active = 10
trend.mailbox_depth = 100
trend.handlers_active = 10
trend.requests_active = 10
# spawned's retained-structure gauges (docs/GUIDE.md, "Retained internal
# structures"): counts per hour.
trend.reply_slots_alive = 10
trend.reply_tokens_alive = 10
trend.pending_replies_entries = 10
trend.reply_registries_alive = 10
trend.deadline_heap_entries = 10
trend.deadline_service_entries = 10
trend.deadline_service_actors = 10
trend.jobs_alive = 10
trend.monitors_alive = 10
trend.link_records_alive = 10
trend.registered_names = 10
trend.timers_alive = 10

# Minimum growth: a trend also needs slope x steady-window hours above this
# many units (MiB for rss_mb and heap_mb) to fail. Bounded pools (Tokio's
Expand All @@ -44,6 +58,18 @@ growth.actors_active = 5
growth.mailbox_depth = 50
growth.handlers_active = 3
growth.requests_active = 3
growth.reply_slots_alive = 5
growth.reply_tokens_alive = 5
growth.pending_replies_entries = 5
growth.reply_registries_alive = 5
growth.deadline_heap_entries = 5
growth.deadline_service_entries = 5
growth.deadline_service_actors = 5
growth.jobs_alive = 5
growth.monitors_alive = 5
growth.link_records_alive = 5
growth.registered_names = 5
growth.timers_alive = 5

# Quiesce: after the workload stops and the long-lived ActorSystem shuts
# down, each series must return to within `slack` of its baseline (sampled
Expand All @@ -60,3 +86,15 @@ quiesce.actors_active = 0
quiesce.mailbox_depth = 0
quiesce.handlers_active = 0
quiesce.requests_active = 0
quiesce.reply_slots_alive = 0
quiesce.reply_tokens_alive = 0
quiesce.pending_replies_entries = 0
quiesce.reply_registries_alive = 0
quiesce.deadline_heap_entries = 0
quiesce.deadline_service_entries = 0
quiesce.deadline_service_actors = 0
quiesce.jobs_alive = 0
quiesce.monitors_alive = 0
quiesce.link_records_alive = 0
quiesce.registered_names = 0
quiesce.timers_alive = 0
47 changes: 46 additions & 1 deletion concurrency/src/instrumentation.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,38 @@
//! Internal instrumentation; disabled builds use zero-sized no-op guards.

/// A field counting one live `G` structure (`gauge::*`) from construction to
/// drop; zero-sized, and a no-op without `metrics`.
pub(crate) type Live<G> = live::Live<G>;
#[cfg(feature = "metrics")]
pub(crate) use crate::metrics::{
accepted, hooks, reply, ActorObservation, Mailbox, MessageObservation, RequestObservation,
accepted, hooks, live, reply, ActorObservation, Mailbox, MessageObservation, RequestObservation,
};

/// Markers of the retained-structure gauges (`metrics::live`): which
/// `spawned_*` gauge a [`Live`](live::Live) field counts.
pub(crate) mod gauge {
/// `reply_slots_alive`: request reply slots.
pub(crate) enum ReplySlots {}
/// `reply_tokens_alive`: `ReplyTo` tokens (deferred obligations).
pub(crate) enum ReplyTokens {}
/// `reply_registries_alive`: per-instance reply registries.
pub(crate) enum ReplyRegistries {}
/// `pending_replies_entries`: entries in every `PendingReplies`.
pub(crate) enum PendingEntries {}
/// `deadline_heap_entries`: per-instance deadline heap entries.
pub(crate) enum DeadlineEntries {}
/// `jobs_alive`: job states (runs, handles, held results).
pub(crate) enum Jobs {}
/// `monitors_alive`: entries in monitor tables.
pub(crate) enum Monitors {}
/// `link_records_alive`: link records.
pub(crate) enum LinkRecords {}
/// `registered_names`: names in the global registry.
pub(crate) enum Names {}
/// `timers_alive`: `send_after` / `send_interval` timers.
pub(crate) enum Timers {}
}

#[cfg(not(feature = "metrics"))]
mod disabled {
use crate::{ActorError, ExitReason};
Expand Down Expand Up @@ -56,6 +85,22 @@ mod disabled {
#[inline(always)]
pub(crate) fn jobs_unfinished(_: usize) {}
}

/// Retained-structure gauges: zero-sized, no-op guards without `metrics`.
pub(crate) mod live {
use std::marker::PhantomData;
pub(crate) struct Live<G>(PhantomData<fn() -> G>);
impl<G> Default for Live<G> {
#[inline(always)]
fn default() -> Self {
Self(PhantomData)
}
}
#[inline(always)]
pub(crate) fn added<G>(_: usize) {}
#[inline(always)]
pub(crate) fn removed<G>(_: usize) {}
}
}
#[cfg(not(feature = "metrics"))]
pub(crate) use disabled::*;
2 changes: 2 additions & 0 deletions concurrency/src/link.rs
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,8 @@ fn apply_signal(

mod sync {
pub(crate) use std::sync::{Arc, Mutex, MutexGuard, Weak};
pub(crate) type LinkRecordAlive =
crate::instrumentation::Live<crate::instrumentation::gauge::LinkRecords>;
}
mod table;
pub(crate) use table::{new_link_table, LinkNotices, LinkOutcome, LinkTable};
Expand Down
5 changes: 5 additions & 0 deletions concurrency/src/link/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ pub(crate) struct LinkEntry {
pub peer_links: Weak<Mutex<LinkState>>,
/// A pending late `NoProc` notice (the peer was past D at `link`).
pub late: bool,
/// `spawned_link_records_alive` (the Loom model: `()`).
_alive: super::sync::LinkRecordAlive,
}

/// An actor's exit table: its link records and its death point.
Expand Down Expand Up @@ -118,6 +120,7 @@ pub(crate) fn install(
peer: peer.clone(),
peer_links: Arc::downgrade(peer_links),
late: true,
_alive: Default::default(),
});
}
return LinkOutcome::NoProc;
Expand All @@ -128,13 +131,15 @@ pub(crate) fn install(
peer: peer.clone(),
peer_links: Arc::downgrade(peer_links),
late: false,
_alive: Default::default(),
});
b.entries.push(LinkEntry {
peer_id: own.id,
gen,
peer: own.clone(),
peer_links: Arc::downgrade(own_links),
late: false,
_alive: Default::default(),
});
LinkOutcome::Linked
})
Expand Down
95 changes: 94 additions & 1 deletion concurrency/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ macro_rules! counters {
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct Snapshot { $( pub $name: u64, )* pub queue_wait: HistogramSnapshot, pub handler_duration: HistogramSnapshot, pub request_latency: HistogramSnapshot, pub deferred_reply_latency: HistogramSnapshot }
pub fn snapshot() -> Snapshot { Snapshot { $( $name: COUNTERS.$name.load(Relaxed), )* queue_wait: QUEUE_WAIT.snapshot(), handler_duration: HANDLER_DURATION.snapshot(), request_latency: REQUEST_LATENCY.snapshot(), deferred_reply_latency: DEFERRED_REPLY_LATENCY.snapshot() } }
fn read_snapshot() -> Snapshot { Snapshot { $( $name: COUNTERS.$name.load(Relaxed), )* queue_wait: QUEUE_WAIT.snapshot(), handler_duration: HANDLER_DURATION.snapshot(), request_latency: REQUEST_LATENCY.snapshot(), deferred_reply_latency: DEFERRED_REPLY_LATENCY.snapshot() } }
fn render_counters(out: &mut String, s: &Snapshot) {
$(
let name = concat!("spawned_", stringify!($name));
Expand Down Expand Up @@ -233,7 +233,34 @@ counters!(
jobs_rejected_total,
jobs_emits_total,
jobs_unfinished_at_stop_total,
// Retained internal structures (`live`): each returns to its idle
// baseline once the work that created it is gone.
reply_slots_alive,
reply_tokens_alive,
reply_registries_alive,
pending_replies_entries,
deadline_heap_entries,
deadline_service_entries,
deadline_service_actors,
jobs_alive,
monitors_alive,
link_records_alive,
registered_names,
timers_alive,
);

/// Process aggregates (see [`Snapshot`]). The deadline service's sizes are
/// read here, under its own lock; every other value is an atomic load.
pub fn snapshot() -> Snapshot {
let (entries, actors) = crate::reply::deadline::service_sizes();
COUNTERS
.deadline_service_entries
.store(entries as u64, Relaxed);
COUNTERS
.deadline_service_actors
.store(actors as u64, Relaxed);
read_snapshot()
}
static QUEUE_WAIT: Histogram = Histogram::new();
static HANDLER_DURATION: Histogram = Histogram::new();
static REQUEST_LATENCY: Histogram = Histogram::new();
Expand Down Expand Up @@ -367,6 +394,72 @@ pub(crate) mod hooks {
}
}

/// Retained-structure gauges: a zero-sized [`Live`](live::Live) field in each
/// structure counts it from construction to drop (no lock, one relaxed
/// atomic at each end, at points that already allocate or free it).
pub(crate) mod live {
use super::COUNTERS;
use crate::instrumentation::gauge::*;
use std::marker::PhantomData;
use std::sync::atomic::{AtomicU64, Ordering::Relaxed};

pub(crate) trait Gauge {
fn gauge() -> &'static AtomicU64;
}
macro_rules! gauges {
($($marker:ident => $field:ident),* $(,)?) => {$(
impl Gauge for $marker {
#[inline(always)]
fn gauge() -> &'static AtomicU64 {
&COUNTERS.$field
}
}
)*};
}
gauges!(
ReplySlots => reply_slots_alive,
ReplyTokens => reply_tokens_alive,
ReplyRegistries => reply_registries_alive,
PendingEntries => pending_replies_entries,
DeadlineEntries => deadline_heap_entries,
Jobs => jobs_alive,
Monitors => monitors_alive,
LinkRecords => link_records_alive,
Names => registered_names,
Timers => timers_alive,
);

/// One live `G` structure, counted from construction to drop.
pub(crate) struct Live<G: Gauge>(PhantomData<fn() -> G>);
impl<G: Gauge> Default for Live<G> {
#[inline]
fn default() -> Self {
G::gauge().fetch_add(1, Relaxed);
Self(PhantomData)
}
}
impl<G: Gauge> Drop for Live<G> {
#[inline]
fn drop(&mut self) {
G::gauge().fetch_sub(1, Relaxed);
}
}
impl<G: Gauge> std::fmt::Debug for Live<G> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("Live")
}
}
/// Structures not owned by one value (map entries): counted in bulk.
#[inline]
pub(crate) fn added<G: Gauge>(n: usize) {
G::gauge().fetch_add(n as u64, Relaxed);
}
#[inline]
pub(crate) fn removed<G: Gauge>(n: usize) {
G::gauge().fetch_sub(n as u64, Relaxed);
}
}

#[derive(Clone, Default)]
pub(crate) struct Mailbox(Arc<AtomicU64>);
impl Mailbox {
Expand Down
Loading
Loading