From 77f5b2f9e8e152e917e30ab53ac8d4859a7c5811 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 12:12:56 +0000 Subject: [PATCH 1/6] fix(server): commit SQLite control-store writes durably without a checkpoint on every close Each control-store transaction opens its own connection, and closing the last one checkpointed the WAL and synced the database file on every commit, which stalled writes and scheduler ticks on slow disks. Connections now keep the WAL when they close, writes commit with synchronous=FULL, platform maintenance truncates the WAL every 30 seconds so constant readers cannot grow it without bound, and a commit that leaves a WAL over 64 MiB resets it at once. The docs say to back SQLite up with .backup or with the -wal file, never the database file alone. --- .../src/platform/_helpers/retention.rs | 20 +++- .../src/platform/constants.rs | 3 + .../src/platform/main/run_maintenance.rs | 9 +- .../src/store/_helpers/open_sqlite.rs | 2 +- .../src/store/constants.rs | 4 + .../src/store/main/begin.rs | 4 +- .../src/store/main/commit.rs | 18 ++++ .../sqlbuild-orchestration/src/store/mod.rs | 1 + .../tests/store/schema/helpers.rs | 90 +++++++++++++++- .../tests/store/schema/test_contracts.rs | 2 + .../tests/store/schema/test_crash_recovery.rs | 101 ++++++++++++++++++ .../tests/store/schema/test_types.rs | 11 ++ deploy/README.md | 5 +- website/src/content/docs/docs/cli/server.mdx | 20 ++++ 14 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/store/constants.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs index c486dd4af0..ef9e9a6329 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs @@ -1,14 +1,16 @@ //! Every retention policy runs in one bounded pass; each part is safe to repeat. use std::collections::BTreeSet; +use std::time::Duration; use chrono::{DateTime, TimeDelta, Utc}; +use rusqlite::config::DbConfig; use sqlbuild_recall::schema::constants::INVOCATION_TABLES; use sqlbuild_recall::schema::main::table_spec::table_spec; use crate::credentials::main::audit::prune_audit::prune_audit; use crate::platform::constants::{ - HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS, + CHECKPOINT_READER_WAIT_MS, HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS, PLATFORM_ALERT_RETENTION_DAYS, SENSOR_EVALUATION_RETENTION_DAYS, }; use crate::platform::models::RetentionPolicies; @@ -212,3 +214,19 @@ async fn delete_logs( } Ok(transaction) } + +/// Copy the WAL into the database and truncate it, or leave it for the next pass if readers stay. +pub(crate) async fn wal_checkpoint(store: &ControlStore) -> Result<(), ControlError> { + let ControlStore::Sqlite(path) = store else { + return Ok(()); + }; + let path = path.clone(); + tokio::task::spawn_blocking(move || { + let connection = rusqlite::Connection::open(&path)?; + connection.set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true)?; + connection.busy_timeout(Duration::from_millis(CHECKPOINT_READER_WAIT_MS))?; + connection.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))?; + Ok(()) + }) + .await? +} diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs index f8833001d1..230602ca65 100644 --- a/crates/sqlbuild-orchestration/src/platform/constants.rs +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -37,3 +37,6 @@ pub(crate) const PLAN_QUEUE_LIMIT: i64 = 4; /// A location instance unseen for longer than this cannot take a plan; locations report every 30 seconds. pub(crate) const LOCATION_SILENCE_SECONDS: i64 = 90; + +/// A SQLite WAL reset waits this long for readers while holding the write lock, then retries later. +pub(crate) const CHECKPOINT_READER_WAIT_MS: u64 = 200; diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs index 083b21e08a..db612e669e 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs @@ -1,11 +1,11 @@ -//! One maintenance role per server: health alerts every cycle, bounded retention every minute. +//! One maintenance role per server: health alerts and WAL resets every cycle, retention every minute. use std::time::{Duration, Instant}; use tokio::sync::watch; use uuid::Uuid; -use crate::platform::_helpers::retention::retain; +use crate::platform::_helpers::retention::{retain, wal_checkpoint}; use crate::platform::_helpers::watchdog::scan_health; use crate::platform::models::RetentionPolicies; use crate::store::errors::ControlError; @@ -44,6 +44,11 @@ pub async fn run_maintenance( } retained_at = Some(Instant::now()); } + match wal_checkpoint(store).await { + Ok(()) => {} + Err(error) if error.transient() => {} + Err(error) => return Err(error), + } tokio::select! { _ = tokio::time::sleep(Duration::from_secs(30)) => {}, result = shutdown.changed() => { if result.is_err() { break; } }, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs index b4ca936711..6d87f2bb2b 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs @@ -46,6 +46,6 @@ pub(crate) fn open_sqlite(path: PathBuf) -> Result { | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; connection.busy_timeout(Duration::from_secs(30))?; - connection.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL")?; + connection.execute_batch("PRAGMA journal_mode=WAL")?; Ok(ControlStore::Sqlite(path)) } diff --git a/crates/sqlbuild-orchestration/src/store/constants.rs b/crates/sqlbuild-orchestration/src/store/constants.rs new file mode 100644 index 0000000000..61d1f56a56 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/constants.rs @@ -0,0 +1,4 @@ +//! Control-store limits. + +/// A SQLite WAL past this size is reset right after the commit that grew it. +pub(crate) const LARGE_WAL_BYTES: u64 = 64 * 1024 * 1024; diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs index 94af142705..7deced8adc 100644 --- a/crates/sqlbuild-orchestration/src/store/main/begin.rs +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -4,6 +4,7 @@ use std::time::Duration; use chrono::Utc; use rusqlite::OpenFlags; +use rusqlite::config::DbConfig; use crate::store::_helpers::changes::sqlite_changes; use crate::store::errors::ControlError; @@ -25,8 +26,9 @@ pub(crate) async fn begin( | OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; connection.busy_timeout(Duration::from_secs(30))?; + connection.set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true)?; connection.execute_batch(if mode == TransactionMode::Write { - "PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" + "PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" } else { "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" })?; diff --git a/crates/sqlbuild-orchestration/src/store/main/commit.rs b/crates/sqlbuild-orchestration/src/store/main/commit.rs index cc72f730ed..e110c12a7c 100644 --- a/crates/sqlbuild-orchestration/src/store/main/commit.rs +++ b/crates/sqlbuild-orchestration/src/store/main/commit.rs @@ -1,6 +1,7 @@ //! Commit facts and their projections together. use crate::store::_helpers::changes::resolve_changes; +use crate::store::constants::LARGE_WAL_BYTES; use crate::store::errors::ControlError; use crate::store::models::{Connection, ControlTransaction}; @@ -14,6 +15,9 @@ pub(crate) async fn commit(transaction: ControlTransaction) -> Result<(), Contro Connection::Sqlite(connection) => { tokio::task::spawn_blocking(move || { connection.execute_batch("COMMIT")?; + if transaction.notifications.is_some() { + drop(truncate_large_wal(&connection)); + } if let Some(sender) = transaction.notifications { for message in messages { let _ = sender.send(message); @@ -39,3 +43,17 @@ pub(crate) async fn commit(transaction: ControlTransaction) -> Result<(), Contro } Ok(()) } + +/// Reset a WAL a large transaction left, which slows reads, unless readers hold it; best effort. +fn truncate_large_wal(connection: &rusqlite::Connection) -> Result<(), rusqlite::Error> { + let Some(database) = connection.path() else { + return Ok(()); + }; + let large = std::fs::metadata(format!("{database}-wal")) + .is_ok_and(|metadata| metadata.len() > LARGE_WAL_BYTES); + if large { + connection.busy_timeout(std::time::Duration::ZERO)?; + connection.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/store/mod.rs b/crates/sqlbuild-orchestration/src/store/mod.rs index 1228df7ae6..aca6885a73 100644 --- a/crates/sqlbuild-orchestration/src/store/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod errors; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs index bc192f33ba..8776c6a0b0 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs @@ -1,7 +1,95 @@ -//! Row readers shared by the control-store schema contracts. +//! Row readers and a killable writer process shared by the control-store schema contracts. +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::{Command, Stdio}; + +use crate::credentials::tests::location_helpers::create_location; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::open_store::open_store; +use crate::store::main::query::query; +use crate::store::models::TransactionMode; +use crate::store::tests::store_helpers::run; use crate::store::types::Row; +const CHILD_STORE_VARIABLE: &str = "SQLBUILD_TEST_CRASH_STORE"; +const ACKNOWLEDGED: &str = "committed "; + pub(crate) fn text(row: &Row, key: &str) -> String { row[key].as_str().expect("text column").to_owned() } + +/// In a `kill_writer_after` process, commit one location at a time and acknowledge each, forever. +pub(crate) fn write_until_killed_when_spawned() { + let Ok(path) = std::env::var(CHILD_STORE_VARIABLE) else { + return; + }; + run(async { + let store = open_store(&path).await.expect("open store"); + migrate_store(&store).await.expect("migrate store"); + let mut index: u64 = 0; + loop { + create_location(&store, &format!("orders-{index}"), "test-admin") + .await + .expect("create location"); + println!("{ACKNOWLEDGED}{index}"); + index += 1; + } + }); +} + +/// Kill a writer running `test` after `commits` acknowledgements; return them and the WAL bytes. +pub(crate) fn kill_writer_after(test: &str, path: &Path, commits: usize) -> (usize, u64) { + let name = format!( + "{}::{test}", + module_path!() + .split_once("::") + .expect("crate path") + .1 + .rsplit_once("::") + .expect("schema module") + .0 + ); + let mut child = Command::new(std::env::current_exe().expect("test binary")) + .args([name.as_str(), "--exact", "--nocapture", "--test-threads=1"]) + .env(CHILD_STORE_VARIABLE, path) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("start writer process"); + let acknowledged = BufReader::new(child.stdout.take().expect("writer output")) + .lines() + .map(|line| line.expect("writer line")) + .filter(|line| line.starts_with(ACKNOWLEDGED)) + .take(commits) + .count(); + child.kill().expect("kill writer"); + child.wait().expect("reap writer"); + let mut wal = path.as_os_str().to_owned(); + wal.push("-wal"); + let wal_bytes = std::fs::metadata(wal).map_or(0, |metadata| metadata.len()); + (acknowledged, wal_bytes) +} + +/// The number of locations a reopened store holds. +pub(crate) fn surviving_locations(path: &Path) -> usize { + run(async { + let store = open_store(path.to_str().expect("UTF-8 path")) + .await + .expect("reopen store"); + let transaction = begin(&store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, rows) = query( + transaction, + "SELECT COUNT(*) AS locations FROM sqb_locations", + vec![], + ) + .await + .expect("count locations"); + commit(transaction).await.expect("finish read"); + usize::try_from(rows[0]["locations"].as_i64().expect("count")).expect("count") + }) +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs index dd9741305f..6a1a10b378 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs @@ -2,6 +2,8 @@ #[path = "helpers.rs"] pub(crate) mod helpers; +#[path = "test_crash_recovery.rs"] +mod test_crash_recovery; #[path = "test_foreign_key_indexes.rs"] mod test_foreign_key_indexes; #[path = "test_store_upgrade.rs"] diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs new file mode 100644 index 0000000000..534cf830d8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs @@ -0,0 +1,101 @@ +//! Commits a killed process acknowledged survive, though closing a connection never checkpoints. + +use crate::platform::_helpers::retention::wal_checkpoint; +use crate::store::main::open_store::open_store; +use crate::store::tests::schema::helpers::{ + kill_writer_after, surviving_locations, write_until_killed_when_spawned, +}; +use crate::store::tests::schema::test_types::{CrashRecoveryTestCase, WalCheckpointTestCase}; +use crate::store::tests::store_helpers::run; + +#[test] +fn given_a_process_killed_between_commits_when_the_store_reopens_then_every_acknowledged_commit_survives() + { + write_until_killed_when_spawned(); + let test_cases = [ + CrashRecoveryTestCase { + description: "killed after the first commits", + expected_acknowledged: 5, + }, + CrashRecoveryTestCase { + description: "killed after many commits", + expected_acknowledged: 200, + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("crash store directory"); + let path = directory.path().join("control.sqlite3"); + + let (acknowledged, wal_bytes) = kill_writer_after( + "test_crash_recovery::given_a_process_killed_between_commits_when_the_store_reopens_then_every_acknowledged_commit_survives", + &path, + test_case.expected_acknowledged, + ); + let survived = surviving_locations(&path); + + assert_eq!( + acknowledged, test_case.expected_acknowledged, + "{}: the writer was killed, not finished", + test_case.description + ); + assert!( + wal_bytes > 0, + "{}: the killed writer left its commits in the WAL", + test_case.description + ); + assert!( + (test_case.expected_acknowledged..=test_case.expected_acknowledged + 1) + .contains(&survived), + "{}: {acknowledged} acknowledged, {survived} survived", + test_case.description + ); + } +} + +#[test] +fn given_commits_left_in_the_wal_when_maintenance_checkpoints_then_the_wal_is_emptied_and_rows_remain() + { + write_until_killed_when_spawned(); + let test_cases = [WalCheckpointTestCase { + description: "a killed writer's WAL", + expected_acknowledged: 20, + expected_wal_bytes: 0, + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("checkpoint store directory"); + let path = directory.path().join("control.sqlite3"); + let (acknowledged, left) = kill_writer_after( + "test_crash_recovery::given_commits_left_in_the_wal_when_maintenance_checkpoints_then_the_wal_is_emptied_and_rows_remain", + &path, + test_case.expected_acknowledged, + ); + + let wal_bytes = run(async { + let store = open_store(path.to_str().expect("UTF-8 path")) + .await + .expect("open store"); + wal_checkpoint(&store).await.expect("checkpoint"); + std::fs::metadata(directory.path().join("control.sqlite3-wal")) + .expect("WAL file") + .len() + }); + let survived = surviving_locations(&path); + + assert_eq!( + acknowledged, test_case.expected_acknowledged, + "{}", + test_case.description + ); + assert!(left > 0, "{}: the WAL held commits", test_case.description); + assert_eq!( + wal_bytes, test_case.expected_wal_bytes, + "{}", + test_case.description + ); + assert!( + survived >= test_case.expected_acknowledged, + "{}: {survived} survived", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs index 748d56da48..4792aabc20 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs @@ -4,6 +4,17 @@ use std::collections::BTreeSet; use crate::store::tests::store_test_types::Backend; +pub(crate) struct CrashRecoveryTestCase { + pub(crate) description: &'static str, + pub(crate) expected_acknowledged: usize, +} + +pub(crate) struct WalCheckpointTestCase { + pub(crate) description: &'static str, + pub(crate) expected_acknowledged: usize, + pub(crate) expected_wal_bytes: u64, +} + pub(crate) struct ForeignKeyIndexTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/deploy/README.md b/deploy/README.md index e743652bea..402ed32e28 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -354,7 +354,10 @@ outside this stack too: a stopped server cannot report its own outage. ## Back up and upgrade -Back up PostgreSQL with `pg_dump` using your normal protected database backup tooling. Also +Back up PostgreSQL with `pg_dump` using your normal protected database backup tooling. Back up +a SQLite control store with `sqlite3 ".backup ''"` while the server runs, or stop the +server and copy the database file together with its `-wal` file; the database file alone can miss +recent commits. Also back up warehouse data, the restricted `.env`, the `server` config, and referenced project images. Keep backup credentials and output outside the repository. Test restoration before an upgrade. Ordinary `docker compose down` preserves named volumes; `down --volumes` destroys them. diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 0cb5dc46d5..1daf214f5a 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -137,6 +137,26 @@ characters, and `null` when the worker reported none. It can quote warehouse dat webhook like other run evidence. `run_failed` events appear only in the webhook, not in the server's platform history. +## Back up the control database + +With no `SQLBUILD_CONTROL_DATABASE_URL`, the server keeps its control database in one SQLite file. +Back it up with SQLite's online backup while the server runs: + +```bash +sqlite3 control.sqlite3 ".backup '/backups/control.sqlite3'" +``` + +| Store | Back up with | While the server runs | +| --- | --- | --- | +| SQLite | `sqlite3 ".backup ''"` or the SQLite backup API | Yes | +| SQLite | A copy of the database file together with its `-wal` file | No; stop the server first | +| PostgreSQL | `pg_dump` | Yes | + +- Never copy the SQLite `.db` file alone. Recent commits can still be in the `-wal` file next to + it, so the copy misses them. +- Restore a backup while the server is stopped, and delete any `-wal` and `-shm` files left beside + the database you replace. + ## Upgrade Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the From c7b854daa1ce9f3225febf68a68096667a0cc43f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 12:12:57 +0000 Subject: [PATCH 2/6] fix(scheduler): keep scheduling after the lease is lost mid-cycle A renewal that found the lease expired between trigger evaluations or in the launcher's poll ended the scheduler loop, which stopped the whole server. The loop now retries, and the next cycle rejoins the election. The tick retention test writes its backlog in one transaction, and its background lease holder reports why it stopped renewing. --- .../src/workers/_helpers/supervisor.rs | 7 ++- .../src/scheduler/main/run_scheduler.rs | 4 +- .../src/scheduler/types.rs | 2 +- .../tests/scheduler/helpers.rs | 8 +++ .../tests/scheduler/ticks/helpers.rs | 38 ++++++------- .../tests/scheduler/ticks/test_loop.rs | 54 ++++++++++++++++++- .../tests/scheduler/ticks/test_retention.rs | 29 +++++----- .../tests/scheduler/ticks/test_types.rs | 39 ++++++++++++++ 8 files changed, 146 insertions(+), 35 deletions(-) diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index 8a683b2b9c..e59286585c 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -171,7 +171,12 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { self.recovered_epoch = None; drop(writeln!( std::io::stderr().lock(), - "Control store unavailable; the scheduler retries: {error}" + "{}; the scheduler retries: {error}", + if matches!(error, ControlError::Fenced) { + "Scheduler lease lost" + } else { + "Control store unavailable" + } )); } } diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index 42b2fb2ea4..220a4d506e 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -11,7 +11,7 @@ use crate::scheduler::types::RunLauncher; use crate::store::errors::ControlError; use crate::store::models::ControlStore; -/// Durable database state drives work; a transient store failure retries the next cycle. +/// Durable database state drives work; a transient failure or a lost lease retries the next cycle. pub async fn run_scheduler( store: &ControlStore, mut state: SchedulerState, @@ -36,7 +36,7 @@ pub async fn run_scheduler( let retry = Instant::now() + Duration::from_secs(1); let due = match polled { Ok(due) => due.min(retry), - Err(error) if error.transient() => { + Err(error) if error.transient() || matches!(error, ControlError::Fenced) => { launcher.retrying(&error); retry } diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs index 1515b87c9d..ec0167dcd8 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -21,6 +21,6 @@ pub trait RunLauncher { /// Hears each completed cycle, with its phase timings, before the launcher polls. fn cycled(&mut self, _cycle: &SchedulerCycle) {} - /// Hears each transient store failure the scheduler retries instead of stopping. + /// Hears each transient store failure or lost lease the scheduler retries instead of stopping. fn retrying(&mut self, _error: &ControlError) {} } diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index 34c98d16ef..b892b9daaa 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -26,9 +26,17 @@ pub(crate) fn hold_lease( let store = store.clone(); async move { tokio::time::sleep(std::time::Duration::from_secs(5)).await; + let started = std::time::Instant::now(); renew_lease(&store, &lease) .await .map(|renewed| Some(((), renewed))) + .inspect_err(|error| { + println!( + "lease holder stopped: renewal failed after {:?} with {error}; the lease was due to expire at {}", + started.elapsed(), + lease.expires_at + ); + }) } }); tokio::spawn(renewals.for_each(|_| async {})) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 2d684232f3..4b4134eb01 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -406,28 +406,30 @@ pub(crate) async fn shift_schedule(store: &ControlStore, trigger_id: Uuid, days: } /// A later covered tick, as if another due time arrived while the requested run still waited. -pub(crate) async fn append_covered_tick( +pub(crate) async fn append_covered_ticks( store: &ControlStore, trigger_id: Uuid, - scheduled_for: DateTime, + scheduled_for: &[DateTime], ) { - let transaction = begin(store, TransactionMode::Write) + let mut transaction = begin(store, TransactionMode::Write) .await - .expect("covered tick"); - let (transaction, _) = query( - transaction, - "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, \ - run_id, covered_by_run_id) SELECT $1, trigger_id, $2, evaluated_at, epoch, \ - 'covered', NULL, run_id FROM sqb_ticks WHERE trigger_id=$3 AND outcome='requested'", - vec![ - Parameter::Id(Uuid::now_v7()), - Parameter::Time(scheduled_for), - Parameter::Id(trigger_id), - ], - ) - .await - .expect("append covered tick"); - commit(transaction).await.expect("commit covered tick"); + .expect("covered ticks"); + for time in scheduled_for { + (transaction, _) = query( + transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, \ + run_id, covered_by_run_id) SELECT $1, trigger_id, $2, evaluated_at, epoch, \ + 'covered', NULL, run_id FROM sqb_ticks WHERE trigger_id=$3 AND outcome='requested'", + vec![ + Parameter::Id(Uuid::now_v7()), + Parameter::Time(*time), + Parameter::Id(trigger_id), + ], + ) + .await + .expect("append covered tick"); + } + commit(transaction).await.expect("commit covered ticks"); } /// Rewrite one trigger-state column the way an earlier build or a past start would have left it. diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index 69106153cc..b3334f4651 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -12,7 +12,7 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::ticks::helpers::snapshot; use crate::scheduler::tests::contracts::ticks::test_types::{ - IdleLauncher, TickTransactionTestCase, + FencedLoopTestCase, FencingLauncher, IdleLauncher, TickTransactionTestCase, }; use crate::store::tests::lease_helpers::expire_lease; use crate::store::tests::replica_helpers::second_store; @@ -99,3 +99,55 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain }); } } + +#[test] +fn given_a_leader_fenced_mid_cycle_when_the_loop_continues_then_it_rejoins_the_election_instead_of_stopping() + { + let test_cases = [ + FencedLoopTestCase { + description: "SQLite scheduler loop", + backend: Backend::Sqlite, + expected_rejoined: true, + }, + FencedLoopTestCase { + description: "PostgreSQL scheduler loop", + backend: Backend::Postgres, + expected_rejoined: true, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate loop store"); + let mut epochs = Vec::new(); + let (stop, shutdown) = tokio::sync::watch::channel(false); + + let (finished, ()) = tokio::join!( + run_scheduler( + store, + SchedulerState::new(Uuid::now_v7()), + shutdown, + FencingLauncher { + epochs: &mut epochs + } + ), + async { + tokio::time::sleep(Duration::from_millis(2_500)).await; + stop.send_replace(true); + } + ); + + finished.expect("a fenced leader keeps the scheduler running"); + assert_eq!( + epochs.windows(2).any(|pair| pair[1] > pair[0]), + test_case.expected_rejoined, + "{}: leads again under a newer epoch, polled with {epochs:?}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs index ffcd7d1ac2..71f3ed3af2 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs @@ -17,7 +17,7 @@ use crate::scheduler::main::{ use crate::scheduler::models::TickOutcome; use crate::scheduler::tests::contracts::helpers::hold_lease; use crate::scheduler::tests::contracts::ticks::helpers::{ - append_covered_tick, calendar_time, shift_schedule, snapshot, + append_covered_ticks, calendar_time, shift_schedule, snapshot, }; use crate::scheduler::tests::contracts::ticks::test_types::{ LatestCatchUpTestCase, TickRetentionTestCase, @@ -87,17 +87,22 @@ fn given_expired_covered_ticks_when_pruned_and_cursor_reset_then_run_linked_and_ ); let later = calendar_time(2) + TimeDelta::hours(1); let backlog = case["backlog_ticks"].as_i64().expect("backlog"); - for second in 1..=backlog { - append_covered_tick( - store, - trigger_id, - calendar_time(2) + TimeDelta::seconds(second), - ) - .await; - } - append_covered_tick(store, trigger_id, later).await; - // Inserting the backlog one tick at a time can outlast the lease on a slow - // machine; the scheduler renews between passes, so the test does too. + let started = std::time::Instant::now(); + append_covered_ticks( + store, + trigger_id, + &(1..=backlog) + .map(|second| calendar_time(2) + TimeDelta::seconds(second)) + .chain([later]) + .collect::>(), + ) + .await; + println!( + "{}: backlog written in {:?}", + test_case.description, + started.elapsed() + ); + // The scheduler renews between passes, so the test does too. let lease = renew_lease(store, &lease).await.expect("renew"); assert_eq!(prune_ticks(store, &lease).await.expect("recent ticks"), 0); shift_schedule(store, trigger_id, shift).await; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 146b81d23b..cff649e51f 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -82,6 +82,45 @@ impl crate::scheduler::types::RunLauncher for CycleRecorder<'_> { } } +/// Expires its lease and renews it on the first leader poll, so a real fence reaches the loop. +pub(crate) struct FencingLauncher<'a> { + pub(crate) epochs: &'a mut Vec, +} + +impl crate::scheduler::types::RunLauncher for FencingLauncher<'_> { + async fn poll( + &mut self, + store: &crate::store::models::ControlStore, + lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + let Some(lease) = lease else { + return Ok(()); + }; + self.epochs.push(lease.epoch); + if self.epochs.len() > 1 { + return Ok(()); + } + crate::store::tests::lease_helpers::expire_lease(store).await; + crate::scheduler::main::renew_lease::renew_lease(store, lease) + .await + .map(drop) + } + + async fn shutdown( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + Ok(()) + } +} + +pub(crate) struct FencedLoopTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_rejoined: bool, +} + pub(crate) struct TickRetentionTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, From e63ec7bc31ca9b37041df38d6e39dfffc9e2782f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:44:32 +0000 Subject: [PATCH 3/6] fix(server): keep the SQLite WAL index between transactions WIP: review fixes. Hold the store's setup connection for its lifetime so transactions that never overlap don't rebuild the WAL index and restart checkpointing; stop the fenced-loop test once it rejoins; neutral wording for a fenced cycle; bound the crash test from below only. --- .../src/workers/_helpers/supervisor.rs | 2 +- .../src/store/_helpers/open_sqlite.rs | 9 +++-- .../src/store/models.rs | 29 ++++++++++++++-- .../tests/scheduler/ticks/test_loop.rs | 19 ++++++----- .../tests/scheduler/ticks/test_types.rs | 5 +++ .../tests/store/schema/helpers.rs | 30 +++++++++++++++- .../tests/store/schema/test_crash_recovery.rs | 34 ++++++++++++++++--- .../tests/store/schema/test_types.rs | 7 ++++ website/src/content/docs/docs/cli/server.mdx | 2 +- 9 files changed, 117 insertions(+), 20 deletions(-) diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index e59286585c..875765c9ea 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -173,7 +173,7 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { std::io::stderr().lock(), "{}; the scheduler retries: {error}", if matches!(error, ControlError::Fenced) { - "Scheduler lease lost" + "Scheduler fenced" } else { "Control store unavailable" } diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs index 6d87f2bb2b..4257371c1f 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs @@ -1,10 +1,11 @@ //! Configure a file-backed SQLite control store without interpreting filenames as URIs. use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use crate::store::errors::ControlError; -use crate::store::models::ControlStore; +use crate::store::models::{ControlStore, SqliteStore}; pub(crate) fn sqlite_path(url: &str) -> Result { let path = Path::new(url.strip_prefix("sqlite://").unwrap_or(url)).to_path_buf(); @@ -47,5 +48,9 @@ pub(crate) fn open_sqlite(path: PathBuf) -> Result { )?; connection.busy_timeout(Duration::from_secs(30))?; connection.execute_batch("PRAGMA journal_mode=WAL")?; - Ok(ControlStore::Sqlite(path)) + connection.query_row("SELECT count(*) FROM sqlite_schema", [], |_| Ok(()))?; + Ok(ControlStore::Sqlite(SqliteStore { + path, + _keeper: Arc::new(Mutex::new(connection)), + })) } diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index 0e93f7ed72..e9799e4aa3 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -1,7 +1,8 @@ //! Native database handles and bound values shared by control-plane operations. use std::collections::BTreeSet; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -13,10 +14,34 @@ use uuid::Uuid; /// An installation's control store. Workers and locations never receive this handle. #[derive(Clone, Debug)] pub enum ControlStore { - Sqlite(PathBuf), + Sqlite(SqliteStore), Postgres(sqlx_postgres::PgPool), } +/// A SQLite store's file, and an idle connection held while any handle lives. +/// +/// Without it the next transaction's connection would be the file's first and rebuild the WAL +/// index, which restarts checkpointing from the start of the WAL, so the WAL would never reset. +#[derive(Clone, Debug)] +pub struct SqliteStore { + pub(crate) path: PathBuf, + pub(crate) _keeper: Arc>, +} + +impl std::ops::Deref for SqliteStore { + type Target = Path; + + fn deref(&self) -> &Path { + &self.path + } +} + +impl AsRef for SqliteStore { + fn as_ref(&self) -> &Path { + &self.path + } +} + /// The first copy wins; duplicate events retain their original storage cursor. #[derive(Clone, Debug, PartialEq, Eq)] pub struct RecallAppendOutcome { diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index b3334f4651..b8f4747081 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -125,22 +125,23 @@ fn given_a_leader_fenced_mid_cycle_when_the_loop_continues_then_it_rejoins_the_e let mut epochs = Vec::new(); let (stop, shutdown) = tokio::sync::watch::channel(false); - let (finished, ()) = tokio::join!( + let finished = tokio::time::timeout( + Duration::from_secs(30), run_scheduler( store, SchedulerState::new(Uuid::now_v7()), shutdown, FencingLauncher { - epochs: &mut epochs - } + epochs: &mut epochs, + stop: &stop, + }, ), - async { - tokio::time::sleep(Duration::from_millis(2_500)).await; - stop.send_replace(true); - } - ); + ) + .await; - finished.expect("a fenced leader keeps the scheduler running"); + finished + .unwrap_or_else(|_| Ok(())) + .expect("a fenced leader keeps the scheduler running"); assert_eq!( epochs.windows(2).any(|pair| pair[1] > pair[0]), test_case.expected_rejoined, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index cff649e51f..81733697f4 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -83,8 +83,10 @@ impl crate::scheduler::types::RunLauncher for CycleRecorder<'_> { } /// Expires its lease and renews it on the first leader poll, so a real fence reaches the loop. +/// Fences the first lease it polls with, and stops the loop once it polls under a newer epoch. pub(crate) struct FencingLauncher<'a> { pub(crate) epochs: &'a mut Vec, + pub(crate) stop: &'a tokio::sync::watch::Sender, } impl crate::scheduler::types::RunLauncher for FencingLauncher<'_> { @@ -98,6 +100,9 @@ impl crate::scheduler::types::RunLauncher for FencingLauncher<'_> { }; self.epochs.push(lease.epoch); if self.epochs.len() > 1 { + if self.epochs.windows(2).any(|pair| pair[1] > pair[0]) { + self.stop.send_replace(true); + } return Ok(()); } crate::store::tests::lease_helpers::expire_lease(store).await; diff --git a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs index 8776c6a0b0..933c52523e 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs @@ -10,7 +10,7 @@ use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; use crate::store::main::open_store::open_store; use crate::store::main::query::query; -use crate::store::models::TransactionMode; +use crate::store::models::{Parameter, TransactionMode}; use crate::store::tests::store_helpers::run; use crate::store::types::Row; @@ -93,3 +93,31 @@ pub(crate) fn surviving_locations(path: &Path) -> usize { usize::try_from(rows[0]["locations"].as_i64().expect("count")).expect("count") }) } + +/// Commit `commits` rows of `bytes` each, one transaction at a time with none overlapping; return the WAL bytes. +pub(crate) fn wal_bytes_after_sequential_writes(path: &Path, commits: usize, bytes: usize) -> u64 { + run(async { + let store = open_store(path.to_str().expect("UTF-8 path")) + .await + .expect("open store"); + let transaction = begin(&store, TransactionMode::Write).await.expect("begin"); + let (transaction, _) = query(transaction, "CREATE TABLE wal_probe (payload TEXT)", vec![]) + .await + .expect("create probe table"); + commit(transaction).await.expect("commit table"); + for index in 0..commits { + let transaction = begin(&store, TransactionMode::Write).await.expect("begin"); + let (transaction, _) = query( + transaction, + "INSERT INTO wal_probe (payload) VALUES ($1)", + vec![Parameter::Text(format!("{index:0>bytes$}"))], + ) + .await + .expect("write probe row"); + commit(transaction).await.expect("commit probe row"); + } + let mut wal = path.as_os_str().to_owned(); + wal.push("-wal"); + std::fs::metadata(wal).map_or(0, |metadata| metadata.len()) + }) +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs index 534cf830d8..b9a6af783c 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs @@ -3,9 +3,12 @@ use crate::platform::_helpers::retention::wal_checkpoint; use crate::store::main::open_store::open_store; use crate::store::tests::schema::helpers::{ - kill_writer_after, surviving_locations, write_until_killed_when_spawned, + kill_writer_after, surviving_locations, wal_bytes_after_sequential_writes, + write_until_killed_when_spawned, +}; +use crate::store::tests::schema::test_types::{ + CrashRecoveryTestCase, WalCheckpointTestCase, WalGrowthTestCase, }; -use crate::store::tests::schema::test_types::{CrashRecoveryTestCase, WalCheckpointTestCase}; use crate::store::tests::store_helpers::run; #[test] @@ -44,8 +47,7 @@ fn given_a_process_killed_between_commits_when_the_store_reopens_then_every_ackn test_case.description ); assert!( - (test_case.expected_acknowledged..=test_case.expected_acknowledged + 1) - .contains(&survived), + survived >= test_case.expected_acknowledged, "{}: {acknowledged} acknowledged, {survived} survived", test_case.description ); @@ -99,3 +101,27 @@ fn given_commits_left_in_the_wal_when_maintenance_checkpoints_then_the_wal_is_em ); } } + +#[test] +fn given_transactions_that_never_overlap_when_many_commit_then_the_wal_restarts_instead_of_growing() +{ + let test_cases = [WalGrowthTestCase { + description: "600 commits of 16 KiB, about 2,700 WAL pages", + commits: 600, + row_bytes: 16 * 1024, + expected_max_wal_bytes: 8 * 1024 * 1024, + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("WAL store directory"); + let path = directory.path().join("control.sqlite3"); + + let wal_bytes = + wal_bytes_after_sequential_writes(&path, test_case.commits, test_case.row_bytes); + + assert!( + wal_bytes <= test_case.expected_max_wal_bytes, + "{}: the WAL grew to {wal_bytes} bytes", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs index 4792aabc20..65f75293d7 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs @@ -15,6 +15,13 @@ pub(crate) struct WalCheckpointTestCase { pub(crate) expected_wal_bytes: u64, } +pub(crate) struct WalGrowthTestCase { + pub(crate) description: &'static str, + pub(crate) commits: usize, + pub(crate) row_bytes: usize, + pub(crate) expected_max_wal_bytes: u64, +} + pub(crate) struct ForeignKeyIndexTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 1daf214f5a..11832d7a93 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -152,7 +152,7 @@ sqlite3 control.sqlite3 ".backup '/backups/control.sqlite3'" | SQLite | A copy of the database file together with its `-wal` file | No; stop the server first | | PostgreSQL | `pg_dump` | Yes | -- Never copy the SQLite `.db` file alone. Recent commits can still be in the `-wal` file next to +- Never copy the SQLite database file alone. Recent commits can still be in the `-wal` file next to it, so the copy misses them. - Restore a backup while the server is stopped, and delete any `-wal` and `-shm` files left beside the database you replace. From 012201ddcd14bd2be0be06074a951d926dbb4610 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:06:15 +0000 Subject: [PATCH 4/6] test(scheduler): fail the fenced-loop test if the scheduler ignores the stop --- .../sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index b8f4747081..dd01a3869f 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -140,7 +140,7 @@ fn given_a_leader_fenced_mid_cycle_when_the_loop_continues_then_it_rejoins_the_e .await; finished - .unwrap_or_else(|_| Ok(())) + .expect("the scheduler stops once it leads again") .expect("a fenced leader keeps the scheduler running"); assert_eq!( epochs.windows(2).any(|pair| pair[1] > pair[0]), From ff1aec21eef1557b8cc051ab74a7a9aa8efd3ff1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:24:00 +0000 Subject: [PATCH 5/6] docs: regenerate the server docs skill reference --- .../sqlbuild/references/docs/cli/server.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index e0beb2e323..79f3e5074c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/cli/server/ - Configure the server - Serve over HTTPS - Receive platform events +- Back up the control database - Upgrade - Set retention - Grant roles to OIDC groups (beta) @@ -154,6 +155,26 @@ characters, and `null` when the worker reported none. It can quote warehouse dat webhook like other run evidence. `run_failed` events appear only in the webhook, not in the server's platform history. +## Back up the control database + +With no `SQLBUILD_CONTROL_DATABASE_URL`, the server keeps its control database in one SQLite file. +Back it up with SQLite's online backup while the server runs: + +```bash +sqlite3 control.sqlite3 ".backup '/backups/control.sqlite3'" +``` + +| Store | Back up with | While the server runs | +| --- | --- | --- | +| SQLite | `sqlite3 ".backup ''"` or the SQLite backup API | Yes | +| SQLite | A copy of the database file together with its `-wal` file | No; stop the server first | +| PostgreSQL | `pg_dump` | Yes | + +- Never copy the SQLite database file alone. Recent commits can still be in the `-wal` file next to + it, so the copy misses them. +- Restore a backup while the server is stopped, and delete any `-wal` and `-shm` files left beside + the database you replace. + ## Upgrade Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the From bcb37ede0693b7548dfde7ce26cac08753d314fb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:31:12 +0000 Subject: [PATCH 6/6] style: one-line doc comments and a loop-free crash-test writer --- crates/sqlbuild-orchestration/src/store/models.rs | 5 +---- .../tests/scheduler/ticks/test_types.rs | 1 - crates/sqlbuild-orchestration/tests/store/schema/helpers.rs | 4 +--- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index e9799e4aa3..fc9a7371b2 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -18,10 +18,7 @@ pub enum ControlStore { Postgres(sqlx_postgres::PgPool), } -/// A SQLite store's file, and an idle connection held while any handle lives. -/// -/// Without it the next transaction's connection would be the file's first and rebuild the WAL -/// index, which restarts checkpointing from the start of the WAL, so the WAL would never reset. +/// A SQLite store's file, and an idle connection that keeps its WAL index between transactions. #[derive(Clone, Debug)] pub struct SqliteStore { pub(crate) path: PathBuf, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 81733697f4..dafaebb699 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -82,7 +82,6 @@ impl crate::scheduler::types::RunLauncher for CycleRecorder<'_> { } } -/// Expires its lease and renews it on the first leader poll, so a real fence reaches the loop. /// Fences the first lease it polls with, and stops the loop once it polls under a newer epoch. pub(crate) struct FencingLauncher<'a> { pub(crate) epochs: &'a mut Vec, diff --git a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs index 933c52523e..fa10659ee3 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs @@ -29,13 +29,11 @@ pub(crate) fn write_until_killed_when_spawned() { run(async { let store = open_store(&path).await.expect("open store"); migrate_store(&store).await.expect("migrate store"); - let mut index: u64 = 0; - loop { + for index in 0..u64::MAX { create_location(&store, &format!("orders-{index}"), "test-admin") .await .expect("create location"); println!("{ACKNOWLEDGED}{index}"); - index += 1; } }); }