Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,15 @@
instructions per `send` (+3.6% over 0.8.0 on `async_ct_send_*`, +0.4% to
+1.2% on the async request benchmarks). With the hint every gated async
benchmark is below 0.8.0 (send -3.4%, requests -0.6% to -0.9%).
- Fixed: in `threads` mode every `send_after` / `send_interval` left a
callback (and its wake-up channel) on the actor's stop token until the actor
stopped, so a long-lived threads actor that kept arming timers grew without
bound. `spawned_rt::threads::CancellationToken` gains
`register_on_cancel`, whose `CancelRegistration` removes the callback in
O(1) when dropped (callbacks still fire in registration order), and
`callbacks_registered`; the timer thread now holds its registrations and
drops them when it fires or is cancelled. Tasks-mode timers were not
affected (Tokio removes a cancellation waiter when its future is dropped).
- Fix the flaky `supervision::transient_startup_timeout_retries_after_actual_cleanup`
(a 20 ms startup deadline raced the fast starts on a loaded host, which also
made cargo-mutants report surviving mutants as caught): the replacement's
Expand Down
59 changes: 33 additions & 26 deletions concurrency/src/threads/time.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use std::sync::mpsc::{self, RecvTimeoutError};
use std::time::Duration;

use spawned_rt::threads::{self as rt, CancellationToken, JoinHandle};
use spawned_rt::threads::{self as rt, CancelRegistration, CancellationToken, JoinHandle};

use super::actor::{Actor, Context, Handler};
use crate::message::Message;
Expand All @@ -16,6 +16,27 @@ pub struct TimerHandle {
pub cancellation_token: CancellationToken,
}

/// A timer thread's wake-up: either token's cancellation sends on the
/// channel. The thread holds both registrations and drops them when it ends,
/// so the actor's long-lived token does not keep a callback per timer that
/// has already fired or been cancelled.
fn wake_on_cancel(
timer: &CancellationToken,
actor: &CancellationToken,
) -> (mpsc::Receiver<()>, [CancelRegistration; 2]) {
let (wake_tx, wake_rx) = mpsc::channel::<()>();
let timer_tx = wake_tx.clone();
let registrations = [
timer.register_on_cancel(Box::new(move || {
let _ = timer_tx.send(());
})),
actor.register_on_cancel(Box::new(move || {
let _ = wake_tx.send(());
})),
];
(wake_rx, registrations)
}

/// Send a single message to an actor after a delay.
pub fn send_after<A, M>(period: Duration, ctx: Context<A>, msg: M) -> TimerHandle
where
Expand All @@ -26,24 +47,18 @@ where
let timer_token = cancellation_token.clone();
let actor_token = ctx.cancellation_token();

let (wake_tx, wake_rx) = mpsc::channel::<()>();

let wake_tx1 = wake_tx.clone();
timer_token.on_cancel(Box::new(move || {
let _ = wake_tx1.send(());
}));
let (wake_rx, registrations) = wake_on_cancel(&timer_token, &actor_token);

actor_token.on_cancel(Box::new(move || {
let _ = wake_tx.send(());
}));

let join_handle = rt::spawn(move || match wake_rx.recv_timeout(period) {
Err(RecvTimeoutError::Timeout) => {
if !timer_token.is_cancelled() && !actor_token.is_cancelled() {
let _ = ctx.send(msg);
let join_handle = rt::spawn(move || {
let _registrations = registrations;
match wake_rx.recv_timeout(period) {
Err(RecvTimeoutError::Timeout) => {
if !timer_token.is_cancelled() && !actor_token.is_cancelled() {
let _ = ctx.send(msg);
}
}
Ok(()) | Err(RecvTimeoutError::Disconnected) => {}
}
Ok(()) | Err(RecvTimeoutError::Disconnected) => {}
});

TimerHandle {
Expand All @@ -67,18 +82,10 @@ where
let timer_token = cancellation_token.clone();
let actor_token = ctx.cancellation_token();

let (wake_tx, wake_rx) = mpsc::channel::<()>();

let wake_tx1 = wake_tx.clone();
timer_token.on_cancel(Box::new(move || {
let _ = wake_tx1.send(());
}));

actor_token.on_cancel(Box::new(move || {
let _ = wake_tx.send(());
}));
let (wake_rx, registrations) = wake_on_cancel(&timer_token, &actor_token);

let join_handle = rt::spawn(move || {
let _registrations = registrations;
while let Err(RecvTimeoutError::Timeout) = wake_rx.recv_timeout(period) {
if timer_token.is_cancelled() || actor_token.is_cancelled() {
break;
Expand Down
53 changes: 52 additions & 1 deletion concurrency/src/threads/timer_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ struct Clocked {
}
impl Actor for Clocked {}

#[derive(Debug)]
#[derive(Debug, Clone)]
struct Fired;
impl Message for Fired {
type Result = ();
Expand Down Expand Up @@ -377,3 +377,54 @@ pub fn threads_send_interval_is_fixed_delay_while_the_actor_stalls() {
},
);
}

/// A long-lived actor's stop token keeps no callback for a timer that fired
/// or was cancelled: each `send_after`/`send_interval` used to leave one on
/// it until the actor stopped, so an actor re-arming timers grew without
/// bound.
#[test]
pub fn threads_timers_release_their_stop_token_callbacks_while_the_actor_runs() {
crate::test_wait::bounded(
concat!(
module_path!(),
"::threads_timers_release_their_stop_token_callbacks_while_the_actor_runs"
),
move || {
let handled = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let actor = Clocked {
handled: handled.clone(),
stall: None,
}
.start();
let token = actor.context().cancellation_token();
let baseline = token.callbacks_registered();
let fired = 50;
for _ in 0..fired {
let _timer = send_after(Duration::from_millis(1), actor.context(), Fired);
}
let long = Duration::from_secs(3600);
let mut cancelled: Vec<_> = (0..50)
.map(|_| send_after(long, actor.context(), Fired))
.collect();
cancelled.extend((0..10).map(|_| send_interval(long, actor.context(), Fired)));
assert!(token.callbacks_registered() >= baseline + cancelled.len());
for timer in &cancelled {
timer.cancellation_token.cancel();
}
assert!(
wait_for(|| handled.lock().unwrap().len(), |n| *n >= fired) >= fired,
"timers never fired"
);
let left = wait_for(|| token.callbacks_registered(), |n| *n == baseline);
assert_eq!(
left,
baseline,
"the running actor's stop token kept {} timer callbacks",
left - baseline
);
assert!(!token.is_cancelled(), "the actor is still running");
actor.child_handle().stop();
actor.join();
},
);
}
2 changes: 1 addition & 1 deletion docs/GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -928,7 +928,7 @@ Wraps standard library primitives:
| `spawn(f)` | Spawn an OS thread |
| `sleep(duration)` | Block the current thread |
| `JoinHandle` | Handle to a spawned thread |
| `CancellationToken` | Cooperative cancellation with callback support via `on_cancel()` |
| `CancellationToken` | Cooperative cancellation with callback support: `on_cancel()` keeps a callback until cancellation; `register_on_cancel()` returns a `CancelRegistration` whose drop removes it (for waiters that finish first, such as timers) |
| `mpsc` | Multi-producer, single-consumer channel (wraps `std::sync::mpsc`) |
| `oneshot` | Single-use channel (wraps `std::sync::mpsc`) |
| `ctrl_c()` | Returns a closure that blocks until Ctrl+C. Supports multiple subscribers. |
Expand Down
3 changes: 2 additions & 1 deletion docs/TESTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -1074,7 +1074,8 @@ RUSTFLAGS="--cfg spawned_loom" cargo test -p spawned-rt --test cancellation_loom
```

The custom cfg isolates instrumentation from Tokio's own Loom configuration.
These models cover registration/cancellation races and reentrant callbacks,
These models cover registration/cancellation races, unregistration (a
`CancelRegistration` dropped) racing a cancel, and reentrant callbacks,
not the entire actor runtime. Do not run ordinary actor tests with this cfg.

The reply and job cores of deferred replies share their source with Loom
Expand Down
Loading
Loading