From 012fc404c067ec5a54f549f8c42f98e43d0c6ba4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 03:58:56 +0000 Subject: [PATCH 1/4] test: step the scheduler's database clock instead of waiting on wall time Due-time selection, lease takeover and the next-wake read now run against a task-local database clock set by the test, and assert tick counts and exact waits per cycle. One real-clock run remains, with a single due time and a generous bound, to catch a scheduler that hangs; it fails clearly when setup runs past the due time. --- .../src/store/main/refresh_clock.rs | 6 + .../tests/scheduler/ticks/helpers.rs | 70 ++-- .../tests/scheduler/ticks/test_latency.rs | 300 +++++++++++------- .../tests/scheduler/ticks/test_types.rs | 29 +- .../tests/store/clock/helpers.rs | 26 ++ .../tests/store/test_contracts.rs | 2 + 6 files changed, 294 insertions(+), 139 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/store/clock/helpers.rs diff --git a/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs index 0453de1ae3..272c39465b 100644 --- a/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs +++ b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs @@ -9,6 +9,12 @@ use crate::store::models::{Connection, ControlTransaction}; pub(crate) async fn refresh_clock( transaction: ControlTransaction, ) -> Result { + #[cfg(test)] + if let Some(now) = crate::store::tests::clock_helpers::database_clock() { + let mut transaction = transaction; + transaction.now = now; + return Ok(transaction); + } let sql = match transaction.connection { Some(Connection::Postgres(_)) => "SELECT clock_timestamp() AS now", _ => "SELECT strftime('%Y-%m-%dT%H:%M:%f000Z','now') AS now", diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 4b4134eb01..44c542c8f3 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -25,8 +25,9 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::ticks::test_types::{ - CycleRecorder, FaultAnomalies, FaultStep, RecordedCycle, TransactionFault, + ClockCycleStep, CycleRecorder, FaultAnomalies, FaultStep, RecordedCycle, TransactionFault, }; +use crate::store::tests::clock_helpers::set_database_clock; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::helpers::read_recall_events; use crate::store::tests::lease_helpers::expire_lease; @@ -761,30 +762,53 @@ pub(crate) async fn run_scheduler_until( (ticks, cycles) } -/// How late the cycle that made a decision woke: after `ready` or the previous poll, whichever is later. -pub(crate) fn decision_wake( - cycles: &[RecordedCycle], - decided_at: DateTime, - ready: DateTime, -) -> (TimeDelta, &RecordedCycle) { - let index = cycles +/// Publish an `orders` calendar trigger due `offsets` seconds after `base` and return its ID. +pub(crate) async fn publish_calendar( + store: &ControlStore, + location: Uuid, + (base, offsets): (DateTime, &[i64]), +) -> Uuid { + let times: Vec = offsets .iter() - .rposition(|cycle| cycle.timings.started_at <= decided_at) - .expect("a cycle started before the decision"); - let free = index - .checked_sub(1) - .map_or(ready, |previous| ready.max(cycles[previous].polled_at)); - (cycles[index].timings.started_at - free, &cycles[index]) + .map(|offset| { + (base + TimeDelta::seconds(*offset)).to_rfc3339_opts(SecondsFormat::Secs, true) + }) + .collect(); + let mut input = snapshot(location); + input.documents.insert( + "triggers".into(), + json!({"calendar_orders": {"description": "Calendar orders", "job_key": "orders", + "schedule": {"kind": "calendar", "times": times, "refresh_at": "2099-01-01T00:00:00Z"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id } -pub(crate) async fn queued_times( +/// Run each step's cycle at its database time and assert who leads, how many ticks it made and how long it sleeps. +pub(crate) async fn step_cycles( store: &ControlStore, - location: Uuid, -) -> BTreeMap> { - list_runs(store, (location, "development"), 100) - .await - .expect("runs") - .into_iter() - .map(|run| (run.run_id, run.queued_at)) - .collect() + (base, steps, description): (DateTime, &[ClockCycleStep], &str), +) { + let mut holders = [ + SchedulerState::new(Uuid::now_v7()), + SchedulerState::new(Uuid::now_v7()), + ]; + for step in steps { + set_database_clock(base + TimeDelta::seconds(step.at_seconds)); + let (state, cycle) = scheduler_cycle(store, holders[step.holder].clone()) + .await + .expect("scheduler cycle"); + holders[step.holder] = state; + assert_eq!( + (cycle.leader, cycle.ticks, cycle.wait_for), + (step.expected_leader, step.expected_ticks, step.expected_wait), + "{description}: holder {} at +{}s", + step.holder, + step.at_seconds + ); + } } + diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs index 6496af2506..242a9435a8 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -1,43 +1,185 @@ -//! Bounds how late the scheduler wakes for due times and standby catch-up behind a slow launcher. +//! Due-time selection steps a set database clock; one real-clock run only catches a scheduler that hangs. use std::time::Duration; -use chrono::{DurationRound, SecondsFormat, TimeDelta, Utc}; -use serde_json::json; +use chrono::{DurationRound, TimeDelta, Utc}; use crate::credentials::tests::location_helpers::create_location; -use crate::declarations::main::list_triggers::list_triggers; +use crate::scheduler::main::list_ticks::list_ticks; use crate::scheduler::tests::contracts::ticks::helpers::{ - decision_wake, queued_times, run_scheduler_until, snapshot, + publish_calendar, run_scheduler_until, step_cycles, +}; +use crate::scheduler::tests::contracts::ticks::test_types::{ + ClockCycleStep, SchedulerClockTestCase, SchedulerWakeTestCase, }; -use crate::scheduler::tests::contracts::ticks::test_types::SchedulerLatencyTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; -use crate::store::tests::lease_helpers::expire_lease; -use crate::store::tests::replica_helpers::second_store; -use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::clock_helpers::with_database_clock; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; +const ON_TIME_THEN_FAILOVER: &[ClockCycleStep] = &[ + ClockCycleStep { + holder: 0, + at_seconds: 0, + expected_leader: true, + expected_ticks: 0, + expected_wait: Duration::from_secs(2), + }, + ClockCycleStep { + holder: 0, + at_seconds: 2, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(1), + }, + ClockCycleStep { + holder: 0, + at_seconds: 3, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(1), + }, + ClockCycleStep { + holder: 0, + at_seconds: 4, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(4), + }, + // The leader stops without releasing its lease, renewed at +4 for 30 s, while +8 and +9 fall due. + ClockCycleStep { + holder: 1, + at_seconds: 10, + expected_leader: false, + expected_ticks: 0, + expected_wait: Duration::from_secs(5), + }, + ClockCycleStep { + holder: 1, + at_seconds: 33, + expected_leader: false, + expected_ticks: 0, + expected_wait: Duration::from_secs(5), + }, + // The standby takes over once the lease lapses and ticks only the latest overdue time. + ClockCycleStep { + holder: 1, + at_seconds: 35, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(5), + }, +]; + +const LATE_CYCLE: &[ClockCycleStep] = &[ + ClockCycleStep { + holder: 0, + at_seconds: 0, + expected_leader: true, + expected_ticks: 0, + expected_wait: Duration::from_secs(2), + }, + ClockCycleStep { + holder: 0, + at_seconds: 1, + expected_leader: true, + expected_ticks: 0, + expected_wait: Duration::from_secs(1), + }, + // +2 and +3 are both overdue: one tick, for +3. + ClockCycleStep { + holder: 0, + at_seconds: 3, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(1), + }, + ClockCycleStep { + holder: 0, + at_seconds: 4, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(5), + }, +]; + #[test] -fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks_queue_within_their_latency_bounds() +fn given_a_set_database_clock_when_times_fall_due_then_each_cycle_ticks_the_latest_due_time_and_sleeps_until_the_next() { let test_cases = [ - SchedulerLatencyTestCase { - description: "SQLite scheduler latency", + SchedulerClockTestCase { + description: "SQLite on-time ticks then standby catch-up", + backend: Backend::Sqlite, + calendar: &[2, 3, 4, 8, 9], + steps: ON_TIME_THEN_FAILOVER, + expected_total_ticks: 4, + }, + SchedulerClockTestCase { + description: "PostgreSQL on-time ticks then standby catch-up", + backend: Backend::Postgres, + calendar: &[2, 3, 4, 8, 9], + steps: ON_TIME_THEN_FAILOVER, + expected_total_ticks: 4, + }, + SchedulerClockTestCase { + description: "SQLite late cycle folds overdue times", backend: Backend::Sqlite, - expected_ticks: 3, - expected_ticks_after_failover: 4, - expected_max_wake_after_due: TimeDelta::milliseconds(500), - expected_max_failover_wake: TimeDelta::milliseconds(1_500), + calendar: &[2, 3, 4], + steps: LATE_CYCLE, + expected_total_ticks: 2, + }, + SchedulerClockTestCase { + description: "PostgreSQL late cycle folds overdue times", + backend: Backend::Postgres, + calendar: &[2, 3, 4], + steps: LATE_CYCLE, + expected_total_ticks: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + let base = Utc::now() + .duration_trunc(TimeDelta::seconds(1)) + .expect("second"); + let ticks = with_database_clock(base, async { + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let trigger_id = + publish_calendar(store, location, (base, test_case.calendar)).await; + step_cycles(store, (base, test_case.steps, test_case.description)).await; + list_ticks(store, trigger_id, 100).await.expect("ticks") + }) + .await; + assert_eq!( + ticks.len(), + test_case.expected_total_ticks, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_running_scheduler_when_a_time_falls_due_then_it_is_decided_without_hanging() { + let test_cases = [ + SchedulerWakeTestCase { + description: "SQLite real-clock wake", + backend: Backend::Sqlite, + window: TimeDelta::seconds(3), launcher_poll: Duration::from_millis(700), }, - SchedulerLatencyTestCase { - description: "PostgreSQL scheduler latency", + SchedulerWakeTestCase { + description: "PostgreSQL real-clock wake", backend: Backend::Postgres, - expected_ticks: 3, - expected_ticks_after_failover: 4, - expected_max_wake_after_due: TimeDelta::milliseconds(500), - expected_max_failover_wake: TimeDelta::milliseconds(1_500), + window: TimeDelta::seconds(3), launcher_poll: Duration::from_millis(700), }, ]; @@ -54,101 +196,39 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks let base = Utc::now() .duration_trunc(TimeDelta::seconds(1)) .expect("second"); - let at = |seconds| base + TimeDelta::seconds(seconds); - let text = |seconds| at(seconds).to_rfc3339_opts(SecondsFormat::Secs, true); - let mut input = snapshot(location); - input.documents.insert( - "jobs".into(), - json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), - ); - input.documents.insert( - "triggers".into(), - json!({"calendar_orders": {"description": "Calendar orders", "job_key": "orders", - "schedule": {"kind": "calendar", "times": [text(2), text(3), text(4), text(8), text(9)], - "refresh_at": "2029-01-01T00:00:00Z"}}}), - ); - publish_snapshot(store, &input).await.expect("publication"); - let trigger_id = list_triggers(store, location, "development") - .await - .expect("trigger")[0] - .trigger_id; - let (ticks, cycles) = - run_scheduler_until(store, trigger_id, (at(6), test_case.launcher_poll)).await; - let queued = queued_times(store, location).await; - let earliest = ticks - .iter() - .min_by_key(|tick| tick.scheduled_for) - .expect("ticks"); - let requested = earliest.run_id.expect("the first due time queues a run"); + let due = base + TimeDelta::seconds(3); + let trigger_id = publish_calendar(store, location, (base, &[3])).await; + let published = Utc::now(); assert!( - queued[&requested] <= earliest.evaluated_at, - "{}: the run is queued with its tick", - test_case.description - ); - assert!( - ticks - .iter() - .filter(|tick| tick.tick_id != earliest.tick_id) - .all(|tick| tick.covered_by_run_id == Some(requested)), - "{}: later times are covered by the waiting run", - test_case.description + published < due, + "{}: setup ran {:?} past the due time, so nothing could tick", + test_case.description, + published - due ); - assert_eq!( - ticks.len(), - test_case.expected_ticks, - "{}", - test_case.description + let (ticks, cycles) = run_scheduler_until( + store, + trigger_id, + (due + test_case.window + TimeDelta::seconds(1), test_case.launcher_poll), + ) + .await; + println!( + "{}: published {:?} after base, ticks {ticks:?}", + test_case.description, + published - base, ); - for tick in &ticks { - let (wake, cycle) = decision_wake(&cycles, tick.evaluated_at, tick.scheduled_for); + for cycle in &cycles { println!( - "{}: due {} woke {wake:?}, store {:?} to decision, cycle {cycle:?}", - test_case.description, - tick.scheduled_for, - tick.evaluated_at - cycle.timings.started_at, - ); - assert!( - wake <= test_case.expected_max_wake_after_due, - "{}: woke {wake:?} after the due time", - test_case.description + "{}: cycle {:?}, polled {}", + test_case.description, cycle.timings, cycle.polled_at ); } - // The first leader stopped without releasing its lease while two more times fell due; - // the standby catches up on the latest one only. - // A standby is already polling when that lease lapses, so its own retry wait counts. - tokio::time::sleep((at(10) - Utc::now()).to_std().unwrap_or_default()).await; - let second = second_store(store).await; - let lapse = async { - tokio::time::sleep((at(11) - Utc::now()).to_std().unwrap_or_default()).await; - expire_lease(store).await; - Utc::now() - }; - let ((ticks, cycles), available) = tokio::join!( - run_scheduler_until(&second, trigger_id, (at(14), Duration::ZERO)), - lapse - ); - assert_eq!( - ticks.len(), - test_case.expected_ticks_after_failover, - "{}: the standby ticks only the latest overdue time", - test_case.description - ); - let latest = ticks - .iter() - .find(|tick| tick.scheduled_for == at(9)) - .expect("the latest overdue time is decided"); - let (wake, cycle) = decision_wake(&cycles, latest.evaluated_at, available); - println!( - "{} failover: woke {wake:?} after the lapse, store {:?} to decision, cycle {cycle:?}", - test_case.description, - latest.evaluated_at - cycle.timings.started_at, - ); + assert_eq!(ticks.len(), 1, "{}", test_case.description); assert!( - wake <= test_case.expected_max_failover_wake, - "{}: woke {wake:?} after the lapse", - test_case.description + ticks[0].evaluated_at - due <= test_case.window, + "{}: decided {:?} after the due time", + test_case.description, + ticks[0].evaluated_at - due ); - close_store(second).await; close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index dafaebb699..2a1a58d83e 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -172,13 +172,30 @@ pub(crate) struct FaultStep { pub(crate) cycles: [bool; 2], } -pub(crate) struct SchedulerLatencyTestCase { +/// One scheduler cycle at a set database clock, by the leader (0) or the standby (1). +pub(crate) struct ClockCycleStep { + pub(crate) holder: usize, + pub(crate) at_seconds: i64, + pub(crate) expected_leader: bool, + pub(crate) expected_ticks: usize, + /// How long the loop sleeps after this cycle: exactly until the next due time, at most a renewal. + pub(crate) expected_wait: std::time::Duration, +} + +pub(crate) struct SchedulerClockTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, - pub(crate) expected_ticks: usize, - pub(crate) expected_ticks_after_failover: usize, - pub(crate) expected_max_wake_after_due: chrono::TimeDelta, - pub(crate) expected_max_failover_wake: chrono::TimeDelta, - /// Work after the cycle's next-due read, which must not delay the wake. + /// Calendar times, in seconds after the test's base time. + pub(crate) calendar: &'static [i64], + pub(crate) steps: &'static [ClockCycleStep], + pub(crate) expected_total_ticks: usize, +} + +pub(crate) struct SchedulerWakeTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// Only a hang misses this: a healthy scheduler decides the time within milliseconds. + pub(crate) window: chrono::TimeDelta, + /// Work after the cycle's next-due read, which must not delay the wake past the window. pub(crate) launcher_poll: std::time::Duration, } diff --git a/crates/sqlbuild-orchestration/tests/store/clock/helpers.rs b/crates/sqlbuild-orchestration/tests/store/clock/helpers.rs new file mode 100644 index 0000000000..dd28dcea42 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/clock/helpers.rs @@ -0,0 +1,26 @@ +//! A settable database clock for one test task, so scheduling contracts step time instead of waiting. + +use std::cell::Cell; +use std::future::Future; + +use chrono::{DateTime, Utc}; + +tokio::task_local! { + static DATABASE_NOW: Cell>; +} + +/// Every transaction `future` begins reads the database clock from this task instead of the store. +pub(crate) async fn with_database_clock( + start: DateTime, + future: impl Future, +) -> T { + DATABASE_NOW.scope(Cell::new(start), future).await +} + +pub(crate) fn set_database_clock(now: DateTime) { + DATABASE_NOW.with(|clock| clock.set(now)); +} + +pub(crate) fn database_clock() -> Option> { + DATABASE_NOW.try_with(Cell::get).ok() +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs index b2b30bae97..2133cbcebe 100644 --- a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -1,5 +1,7 @@ //! Recall storage integration contracts for SQLite and PostgreSQL. +#[path = "clock/helpers.rs"] +pub(crate) mod clock_helpers; #[path = "faults/helpers.rs"] pub(crate) mod fault_helpers; #[path = "faults/test_types.rs"] From d3cd391dee9bc0b78d3d2640dfd0204a679113eb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 04:09:43 +0000 Subject: [PATCH 2/4] test: assert which due times ticked under the stepped clock --- .../tests/scheduler/ticks/test_latency.rs | 20 +++++++++---------- .../tests/scheduler/ticks/test_types.rs | 5 +++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs index 242a9435a8..ba2b2f59f7 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -112,28 +112,28 @@ fn given_a_set_database_clock_when_times_fall_due_then_each_cycle_ticks_the_late backend: Backend::Sqlite, calendar: &[2, 3, 4, 8, 9], steps: ON_TIME_THEN_FAILOVER, - expected_total_ticks: 4, + expected_ticked: &[2, 3, 4, 9], }, SchedulerClockTestCase { description: "PostgreSQL on-time ticks then standby catch-up", backend: Backend::Postgres, calendar: &[2, 3, 4, 8, 9], steps: ON_TIME_THEN_FAILOVER, - expected_total_ticks: 4, + expected_ticked: &[2, 3, 4, 9], }, SchedulerClockTestCase { description: "SQLite late cycle folds overdue times", backend: Backend::Sqlite, calendar: &[2, 3, 4], steps: LATE_CYCLE, - expected_total_ticks: 2, + expected_ticked: &[3, 4], }, SchedulerClockTestCase { description: "PostgreSQL late cycle folds overdue times", backend: Backend::Postgres, calendar: &[2, 3, 4], steps: LATE_CYCLE, - expected_total_ticks: 2, + expected_ticked: &[3, 4], }, ]; for test_case in test_cases { @@ -156,12 +156,12 @@ fn given_a_set_database_clock_when_times_fall_due_then_each_cycle_ticks_the_late list_ticks(store, trigger_id, 100).await.expect("ticks") }) .await; - assert_eq!( - ticks.len(), - test_case.expected_total_ticks, - "{}", - test_case.description - ); + let mut ticked: Vec = ticks + .iter() + .map(|tick| (tick.scheduled_for - base).num_seconds()) + .collect(); + ticked.sort_unstable(); + assert_eq!(ticked, test_case.expected_ticked, "{}", test_case.description); close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 2a1a58d83e..f9bf4d8972 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -178,7 +178,7 @@ pub(crate) struct ClockCycleStep { pub(crate) at_seconds: i64, pub(crate) expected_leader: bool, pub(crate) expected_ticks: usize, - /// How long the loop sleeps after this cycle: exactly until the next due time, at most a renewal. + /// The cycle's wait to the next due time, capped at one renewal interval. pub(crate) expected_wait: std::time::Duration, } @@ -188,7 +188,8 @@ pub(crate) struct SchedulerClockTestCase { /// Calendar times, in seconds after the test's base time. pub(crate) calendar: &'static [i64], pub(crate) steps: &'static [ClockCycleStep], - pub(crate) expected_total_ticks: usize, + /// Due times that ticked, in seconds after the base time. + pub(crate) expected_ticked: &'static [i64], } pub(crate) struct SchedulerWakeTestCase { From 0b21f50ab8a852dac1fc51eb431d501abd5d27a7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 04:17:37 +0000 Subject: [PATCH 3/4] style: format the stepped-clock scheduler test --- .../tests/scheduler/ticks/helpers.rs | 7 +++++-- .../tests/scheduler/ticks/test_latency.rs | 11 +++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 44c542c8f3..51a15ed7ee 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -804,11 +804,14 @@ pub(crate) async fn step_cycles( holders[step.holder] = state; assert_eq!( (cycle.leader, cycle.ticks, cycle.wait_for), - (step.expected_leader, step.expected_ticks, step.expected_wait), + ( + step.expected_leader, + step.expected_ticks, + step.expected_wait + ), "{description}: holder {} at +{}s", step.holder, step.at_seconds ); } } - diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs index ba2b2f59f7..7b90edd3e6 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -161,7 +161,11 @@ fn given_a_set_database_clock_when_times_fall_due_then_each_cycle_ticks_the_late .map(|tick| (tick.scheduled_for - base).num_seconds()) .collect(); ticked.sort_unstable(); - assert_eq!(ticked, test_case.expected_ticked, "{}", test_case.description); + assert_eq!( + ticked, test_case.expected_ticked, + "{}", + test_case.description + ); close_test_store(fixture).await; }); } @@ -208,7 +212,10 @@ fn given_a_running_scheduler_when_a_time_falls_due_then_it_is_decided_without_ha let (ticks, cycles) = run_scheduler_until( store, trigger_id, - (due + test_case.window + TimeDelta::seconds(1), test_case.launcher_poll), + ( + due + test_case.window + TimeDelta::seconds(1), + test_case.launcher_poll, + ), ) .await; println!( From 4f1f602a2e0b1976afb39f162f516eaa25136267 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 04:24:51 +0000 Subject: [PATCH 4/4] test: name the real-clock scheduler test's expected outcomes --- .../tests/scheduler/ticks/test_latency.rs | 17 ++++++++++++----- .../tests/scheduler/ticks/test_types.rs | 5 +++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs index 7b90edd3e6..d76711f008 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -177,13 +177,15 @@ fn given_a_running_scheduler_when_a_time_falls_due_then_it_is_decided_without_ha SchedulerWakeTestCase { description: "SQLite real-clock wake", backend: Backend::Sqlite, - window: TimeDelta::seconds(3), + expected_ticks: 1, + expected_max_decision_delay: TimeDelta::seconds(3), launcher_poll: Duration::from_millis(700), }, SchedulerWakeTestCase { description: "PostgreSQL real-clock wake", backend: Backend::Postgres, - window: TimeDelta::seconds(3), + expected_ticks: 1, + expected_max_decision_delay: TimeDelta::seconds(3), launcher_poll: Duration::from_millis(700), }, ]; @@ -213,7 +215,7 @@ fn given_a_running_scheduler_when_a_time_falls_due_then_it_is_decided_without_ha store, trigger_id, ( - due + test_case.window + TimeDelta::seconds(1), + due + test_case.expected_max_decision_delay + TimeDelta::seconds(1), test_case.launcher_poll, ), ) @@ -229,9 +231,14 @@ fn given_a_running_scheduler_when_a_time_falls_due_then_it_is_decided_without_ha test_case.description, cycle.timings, cycle.polled_at ); } - assert_eq!(ticks.len(), 1, "{}", test_case.description); + assert_eq!( + ticks.len(), + test_case.expected_ticks, + "{}", + test_case.description + ); assert!( - ticks[0].evaluated_at - due <= test_case.window, + ticks[0].evaluated_at - due <= test_case.expected_max_decision_delay, "{}: decided {:?} after the due time", test_case.description, ticks[0].evaluated_at - due diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index f9bf4d8972..92ce72ba69 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -195,8 +195,9 @@ pub(crate) struct SchedulerClockTestCase { pub(crate) struct SchedulerWakeTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, + pub(crate) expected_ticks: usize, /// Only a hang misses this: a healthy scheduler decides the time within milliseconds. - pub(crate) window: chrono::TimeDelta, - /// Work after the cycle's next-due read, which must not delay the wake past the window. + pub(crate) expected_max_decision_delay: chrono::TimeDelta, + /// Work after the cycle's next-due read, which must not delay the decision past its bound. pub(crate) launcher_poll: std::time::Duration, }