diff --git a/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs index 0453de1ae3..272c39465b 100644 --- a/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs +++ b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs @@ -9,6 +9,12 @@ use crate::store::models::{Connection, ControlTransaction}; pub(crate) async fn refresh_clock( transaction: ControlTransaction, ) -> Result { + #[cfg(test)] + if let Some(now) = crate::store::tests::clock_helpers::database_clock() { + let mut transaction = transaction; + transaction.now = now; + return Ok(transaction); + } let sql = match transaction.connection { Some(Connection::Postgres(_)) => "SELECT clock_timestamp() AS now", _ => "SELECT strftime('%Y-%m-%dT%H:%M:%f000Z','now') AS now", diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 4b4134eb01..51a15ed7ee 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -25,8 +25,9 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::ticks::test_types::{ - CycleRecorder, FaultAnomalies, FaultStep, RecordedCycle, TransactionFault, + ClockCycleStep, CycleRecorder, FaultAnomalies, FaultStep, RecordedCycle, TransactionFault, }; +use crate::store::tests::clock_helpers::set_database_clock; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::helpers::read_recall_events; use crate::store::tests::lease_helpers::expire_lease; @@ -761,30 +762,56 @@ pub(crate) async fn run_scheduler_until( (ticks, cycles) } -/// How late the cycle that made a decision woke: after `ready` or the previous poll, whichever is later. -pub(crate) fn decision_wake( - cycles: &[RecordedCycle], - decided_at: DateTime, - ready: DateTime, -) -> (TimeDelta, &RecordedCycle) { - let index = cycles +/// Publish an `orders` calendar trigger due `offsets` seconds after `base` and return its ID. +pub(crate) async fn publish_calendar( + store: &ControlStore, + location: Uuid, + (base, offsets): (DateTime, &[i64]), +) -> Uuid { + let times: Vec = offsets .iter() - .rposition(|cycle| cycle.timings.started_at <= decided_at) - .expect("a cycle started before the decision"); - let free = index - .checked_sub(1) - .map_or(ready, |previous| ready.max(cycles[previous].polled_at)); - (cycles[index].timings.started_at - free, &cycles[index]) + .map(|offset| { + (base + TimeDelta::seconds(*offset)).to_rfc3339_opts(SecondsFormat::Secs, true) + }) + .collect(); + let mut input = snapshot(location); + input.documents.insert( + "triggers".into(), + json!({"calendar_orders": {"description": "Calendar orders", "job_key": "orders", + "schedule": {"kind": "calendar", "times": times, "refresh_at": "2099-01-01T00:00:00Z"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id } -pub(crate) async fn queued_times( +/// Run each step's cycle at its database time and assert who leads, how many ticks it made and how long it sleeps. +pub(crate) async fn step_cycles( store: &ControlStore, - location: Uuid, -) -> BTreeMap> { - list_runs(store, (location, "development"), 100) - .await - .expect("runs") - .into_iter() - .map(|run| (run.run_id, run.queued_at)) - .collect() + (base, steps, description): (DateTime, &[ClockCycleStep], &str), +) { + let mut holders = [ + SchedulerState::new(Uuid::now_v7()), + SchedulerState::new(Uuid::now_v7()), + ]; + for step in steps { + set_database_clock(base + TimeDelta::seconds(step.at_seconds)); + let (state, cycle) = scheduler_cycle(store, holders[step.holder].clone()) + .await + .expect("scheduler cycle"); + holders[step.holder] = state; + assert_eq!( + (cycle.leader, cycle.ticks, cycle.wait_for), + ( + step.expected_leader, + step.expected_ticks, + step.expected_wait + ), + "{description}: holder {} at +{}s", + step.holder, + step.at_seconds + ); + } } diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs index 6496af2506..d76711f008 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -1,43 +1,191 @@ -//! Bounds how late the scheduler wakes for due times and standby catch-up behind a slow launcher. +//! Due-time selection steps a set database clock; one real-clock run only catches a scheduler that hangs. use std::time::Duration; -use chrono::{DurationRound, SecondsFormat, TimeDelta, Utc}; -use serde_json::json; +use chrono::{DurationRound, TimeDelta, Utc}; use crate::credentials::tests::location_helpers::create_location; -use crate::declarations::main::list_triggers::list_triggers; +use crate::scheduler::main::list_ticks::list_ticks; use crate::scheduler::tests::contracts::ticks::helpers::{ - decision_wake, queued_times, run_scheduler_until, snapshot, + publish_calendar, run_scheduler_until, step_cycles, +}; +use crate::scheduler::tests::contracts::ticks::test_types::{ + ClockCycleStep, SchedulerClockTestCase, SchedulerWakeTestCase, }; -use crate::scheduler::tests::contracts::ticks::test_types::SchedulerLatencyTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; -use crate::store::tests::lease_helpers::expire_lease; -use crate::store::tests::replica_helpers::second_store; -use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::clock_helpers::with_database_clock; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; +const ON_TIME_THEN_FAILOVER: &[ClockCycleStep] = &[ + ClockCycleStep { + holder: 0, + at_seconds: 0, + expected_leader: true, + expected_ticks: 0, + expected_wait: Duration::from_secs(2), + }, + ClockCycleStep { + holder: 0, + at_seconds: 2, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(1), + }, + ClockCycleStep { + holder: 0, + at_seconds: 3, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(1), + }, + ClockCycleStep { + holder: 0, + at_seconds: 4, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(4), + }, + // The leader stops without releasing its lease, renewed at +4 for 30 s, while +8 and +9 fall due. + ClockCycleStep { + holder: 1, + at_seconds: 10, + expected_leader: false, + expected_ticks: 0, + expected_wait: Duration::from_secs(5), + }, + ClockCycleStep { + holder: 1, + at_seconds: 33, + expected_leader: false, + expected_ticks: 0, + expected_wait: Duration::from_secs(5), + }, + // The standby takes over once the lease lapses and ticks only the latest overdue time. + ClockCycleStep { + holder: 1, + at_seconds: 35, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(5), + }, +]; + +const LATE_CYCLE: &[ClockCycleStep] = &[ + ClockCycleStep { + holder: 0, + at_seconds: 0, + expected_leader: true, + expected_ticks: 0, + expected_wait: Duration::from_secs(2), + }, + ClockCycleStep { + holder: 0, + at_seconds: 1, + expected_leader: true, + expected_ticks: 0, + expected_wait: Duration::from_secs(1), + }, + // +2 and +3 are both overdue: one tick, for +3. + ClockCycleStep { + holder: 0, + at_seconds: 3, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(1), + }, + ClockCycleStep { + holder: 0, + at_seconds: 4, + expected_leader: true, + expected_ticks: 1, + expected_wait: Duration::from_secs(5), + }, +]; + #[test] -fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks_queue_within_their_latency_bounds() +fn given_a_set_database_clock_when_times_fall_due_then_each_cycle_ticks_the_latest_due_time_and_sleeps_until_the_next() { let test_cases = [ - SchedulerLatencyTestCase { - description: "SQLite scheduler latency", + SchedulerClockTestCase { + description: "SQLite on-time ticks then standby catch-up", + backend: Backend::Sqlite, + calendar: &[2, 3, 4, 8, 9], + steps: ON_TIME_THEN_FAILOVER, + expected_ticked: &[2, 3, 4, 9], + }, + SchedulerClockTestCase { + description: "PostgreSQL on-time ticks then standby catch-up", + backend: Backend::Postgres, + calendar: &[2, 3, 4, 8, 9], + steps: ON_TIME_THEN_FAILOVER, + expected_ticked: &[2, 3, 4, 9], + }, + SchedulerClockTestCase { + description: "SQLite late cycle folds overdue times", + backend: Backend::Sqlite, + calendar: &[2, 3, 4], + steps: LATE_CYCLE, + expected_ticked: &[3, 4], + }, + SchedulerClockTestCase { + description: "PostgreSQL late cycle folds overdue times", + backend: Backend::Postgres, + calendar: &[2, 3, 4], + steps: LATE_CYCLE, + expected_ticked: &[3, 4], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + let base = Utc::now() + .duration_trunc(TimeDelta::seconds(1)) + .expect("second"); + let ticks = with_database_clock(base, async { + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let trigger_id = + publish_calendar(store, location, (base, test_case.calendar)).await; + step_cycles(store, (base, test_case.steps, test_case.description)).await; + list_ticks(store, trigger_id, 100).await.expect("ticks") + }) + .await; + let mut ticked: Vec = ticks + .iter() + .map(|tick| (tick.scheduled_for - base).num_seconds()) + .collect(); + ticked.sort_unstable(); + assert_eq!( + ticked, test_case.expected_ticked, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_running_scheduler_when_a_time_falls_due_then_it_is_decided_without_hanging() { + let test_cases = [ + SchedulerWakeTestCase { + description: "SQLite real-clock wake", backend: Backend::Sqlite, - expected_ticks: 3, - expected_ticks_after_failover: 4, - expected_max_wake_after_due: TimeDelta::milliseconds(500), - expected_max_failover_wake: TimeDelta::milliseconds(1_500), + expected_ticks: 1, + expected_max_decision_delay: TimeDelta::seconds(3), launcher_poll: Duration::from_millis(700), }, - SchedulerLatencyTestCase { - description: "PostgreSQL scheduler latency", + SchedulerWakeTestCase { + description: "PostgreSQL real-clock wake", backend: Backend::Postgres, - expected_ticks: 3, - expected_ticks_after_failover: 4, - expected_max_wake_after_due: TimeDelta::milliseconds(500), - expected_max_failover_wake: TimeDelta::milliseconds(1_500), + expected_ticks: 1, + expected_max_decision_delay: TimeDelta::seconds(3), launcher_poll: Duration::from_millis(700), }, ]; @@ -54,101 +202,47 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks let base = Utc::now() .duration_trunc(TimeDelta::seconds(1)) .expect("second"); - let at = |seconds| base + TimeDelta::seconds(seconds); - let text = |seconds| at(seconds).to_rfc3339_opts(SecondsFormat::Secs, true); - let mut input = snapshot(location); - input.documents.insert( - "jobs".into(), - json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), - ); - input.documents.insert( - "triggers".into(), - json!({"calendar_orders": {"description": "Calendar orders", "job_key": "orders", - "schedule": {"kind": "calendar", "times": [text(2), text(3), text(4), text(8), text(9)], - "refresh_at": "2029-01-01T00:00:00Z"}}}), - ); - publish_snapshot(store, &input).await.expect("publication"); - let trigger_id = list_triggers(store, location, "development") - .await - .expect("trigger")[0] - .trigger_id; - let (ticks, cycles) = - run_scheduler_until(store, trigger_id, (at(6), test_case.launcher_poll)).await; - let queued = queued_times(store, location).await; - let earliest = ticks - .iter() - .min_by_key(|tick| tick.scheduled_for) - .expect("ticks"); - let requested = earliest.run_id.expect("the first due time queues a run"); + let due = base + TimeDelta::seconds(3); + let trigger_id = publish_calendar(store, location, (base, &[3])).await; + let published = Utc::now(); assert!( - queued[&requested] <= earliest.evaluated_at, - "{}: the run is queued with its tick", - test_case.description - ); - assert!( - ticks - .iter() - .filter(|tick| tick.tick_id != earliest.tick_id) - .all(|tick| tick.covered_by_run_id == Some(requested)), - "{}: later times are covered by the waiting run", - test_case.description + published < due, + "{}: setup ran {:?} past the due time, so nothing could tick", + test_case.description, + published - due ); - assert_eq!( - ticks.len(), - test_case.expected_ticks, - "{}", - test_case.description + let (ticks, cycles) = run_scheduler_until( + store, + trigger_id, + ( + due + test_case.expected_max_decision_delay + TimeDelta::seconds(1), + test_case.launcher_poll, + ), + ) + .await; + println!( + "{}: published {:?} after base, ticks {ticks:?}", + test_case.description, + published - base, ); - for tick in &ticks { - let (wake, cycle) = decision_wake(&cycles, tick.evaluated_at, tick.scheduled_for); + for cycle in &cycles { println!( - "{}: due {} woke {wake:?}, store {:?} to decision, cycle {cycle:?}", - test_case.description, - tick.scheduled_for, - tick.evaluated_at - cycle.timings.started_at, - ); - assert!( - wake <= test_case.expected_max_wake_after_due, - "{}: woke {wake:?} after the due time", - test_case.description + "{}: cycle {:?}, polled {}", + test_case.description, cycle.timings, cycle.polled_at ); } - // The first leader stopped without releasing its lease while two more times fell due; - // the standby catches up on the latest one only. - // A standby is already polling when that lease lapses, so its own retry wait counts. - tokio::time::sleep((at(10) - Utc::now()).to_std().unwrap_or_default()).await; - let second = second_store(store).await; - let lapse = async { - tokio::time::sleep((at(11) - Utc::now()).to_std().unwrap_or_default()).await; - expire_lease(store).await; - Utc::now() - }; - let ((ticks, cycles), available) = tokio::join!( - run_scheduler_until(&second, trigger_id, (at(14), Duration::ZERO)), - lapse - ); assert_eq!( ticks.len(), - test_case.expected_ticks_after_failover, - "{}: the standby ticks only the latest overdue time", + test_case.expected_ticks, + "{}", test_case.description ); - let latest = ticks - .iter() - .find(|tick| tick.scheduled_for == at(9)) - .expect("the latest overdue time is decided"); - let (wake, cycle) = decision_wake(&cycles, latest.evaluated_at, available); - println!( - "{} failover: woke {wake:?} after the lapse, store {:?} to decision, cycle {cycle:?}", - test_case.description, - latest.evaluated_at - cycle.timings.started_at, - ); assert!( - wake <= test_case.expected_max_failover_wake, - "{}: woke {wake:?} after the lapse", - test_case.description + ticks[0].evaluated_at - due <= test_case.expected_max_decision_delay, + "{}: decided {:?} after the due time", + test_case.description, + ticks[0].evaluated_at - due ); - close_store(second).await; close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index dafaebb699..92ce72ba69 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -172,13 +172,32 @@ pub(crate) struct FaultStep { pub(crate) cycles: [bool; 2], } -pub(crate) struct SchedulerLatencyTestCase { +/// One scheduler cycle at a set database clock, by the leader (0) or the standby (1). +pub(crate) struct ClockCycleStep { + pub(crate) holder: usize, + pub(crate) at_seconds: i64, + pub(crate) expected_leader: bool, + pub(crate) expected_ticks: usize, + /// The cycle's wait to the next due time, capped at one renewal interval. + pub(crate) expected_wait: std::time::Duration, +} + +pub(crate) struct SchedulerClockTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// Calendar times, in seconds after the test's base time. + pub(crate) calendar: &'static [i64], + pub(crate) steps: &'static [ClockCycleStep], + /// Due times that ticked, in seconds after the base time. + pub(crate) expected_ticked: &'static [i64], +} + +pub(crate) struct SchedulerWakeTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, pub(crate) expected_ticks: usize, - pub(crate) expected_ticks_after_failover: usize, - pub(crate) expected_max_wake_after_due: chrono::TimeDelta, - pub(crate) expected_max_failover_wake: chrono::TimeDelta, - /// Work after the cycle's next-due read, which must not delay the wake. + /// Only a hang misses this: a healthy scheduler decides the time within milliseconds. + pub(crate) expected_max_decision_delay: chrono::TimeDelta, + /// Work after the cycle's next-due read, which must not delay the decision past its bound. pub(crate) launcher_poll: std::time::Duration, } diff --git a/crates/sqlbuild-orchestration/tests/store/clock/helpers.rs b/crates/sqlbuild-orchestration/tests/store/clock/helpers.rs new file mode 100644 index 0000000000..dd28dcea42 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/clock/helpers.rs @@ -0,0 +1,26 @@ +//! A settable database clock for one test task, so scheduling contracts step time instead of waiting. + +use std::cell::Cell; +use std::future::Future; + +use chrono::{DateTime, Utc}; + +tokio::task_local! { + static DATABASE_NOW: Cell>; +} + +/// Every transaction `future` begins reads the database clock from this task instead of the store. +pub(crate) async fn with_database_clock( + start: DateTime, + future: impl Future, +) -> T { + DATABASE_NOW.scope(Cell::new(start), future).await +} + +pub(crate) fn set_database_clock(now: DateTime) { + DATABASE_NOW.with(|clock| clock.set(now)); +} + +pub(crate) fn database_clock() -> Option> { + DATABASE_NOW.try_with(Cell::get).ok() +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs index b2b30bae97..2133cbcebe 100644 --- a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -1,5 +1,7 @@ //! Recall storage integration contracts for SQLite and PostgreSQL. +#[path = "clock/helpers.rs"] +pub(crate) mod clock_helpers; #[path = "faults/helpers.rs"] pub(crate) mod fault_helpers; #[path = "faults/test_types.rs"]