diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index 8a683b2b9c..875765c9ea 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -171,7 +171,12 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { self.recovered_epoch = None; drop(writeln!( std::io::stderr().lock(), - "Control store unavailable; the scheduler retries: {error}" + "{}; the scheduler retries: {error}", + if matches!(error, ControlError::Fenced) { + "Scheduler fenced" + } else { + "Control store unavailable" + } )); } } diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs index c486dd4af0..ef9e9a6329 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs @@ -1,14 +1,16 @@ //! Every retention policy runs in one bounded pass; each part is safe to repeat. use std::collections::BTreeSet; +use std::time::Duration; use chrono::{DateTime, TimeDelta, Utc}; +use rusqlite::config::DbConfig; use sqlbuild_recall::schema::constants::INVOCATION_TABLES; use sqlbuild_recall::schema::main::table_spec::table_spec; use crate::credentials::main::audit::prune_audit::prune_audit; use crate::platform::constants::{ - HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS, + CHECKPOINT_READER_WAIT_MS, HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS, PLATFORM_ALERT_RETENTION_DAYS, SENSOR_EVALUATION_RETENTION_DAYS, }; use crate::platform::models::RetentionPolicies; @@ -212,3 +214,19 @@ async fn delete_logs( } Ok(transaction) } + +/// Copy the WAL into the database and truncate it, or leave it for the next pass if readers stay. +pub(crate) async fn wal_checkpoint(store: &ControlStore) -> Result<(), ControlError> { + let ControlStore::Sqlite(path) = store else { + return Ok(()); + }; + let path = path.clone(); + tokio::task::spawn_blocking(move || { + let connection = rusqlite::Connection::open(&path)?; + connection.set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true)?; + connection.busy_timeout(Duration::from_millis(CHECKPOINT_READER_WAIT_MS))?; + connection.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))?; + Ok(()) + }) + .await? +} diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs index f8833001d1..230602ca65 100644 --- a/crates/sqlbuild-orchestration/src/platform/constants.rs +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -37,3 +37,6 @@ pub(crate) const PLAN_QUEUE_LIMIT: i64 = 4; /// A location instance unseen for longer than this cannot take a plan; locations report every 30 seconds. pub(crate) const LOCATION_SILENCE_SECONDS: i64 = 90; + +/// A SQLite WAL reset waits this long for readers while holding the write lock, then retries later. +pub(crate) const CHECKPOINT_READER_WAIT_MS: u64 = 200; diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs index 083b21e08a..db612e669e 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs @@ -1,11 +1,11 @@ -//! One maintenance role per server: health alerts every cycle, bounded retention every minute. +//! One maintenance role per server: health alerts and WAL resets every cycle, retention every minute. use std::time::{Duration, Instant}; use tokio::sync::watch; use uuid::Uuid; -use crate::platform::_helpers::retention::retain; +use crate::platform::_helpers::retention::{retain, wal_checkpoint}; use crate::platform::_helpers::watchdog::scan_health; use crate::platform::models::RetentionPolicies; use crate::store::errors::ControlError; @@ -44,6 +44,11 @@ pub async fn run_maintenance( } retained_at = Some(Instant::now()); } + match wal_checkpoint(store).await { + Ok(()) => {} + Err(error) if error.transient() => {} + Err(error) => return Err(error), + } tokio::select! { _ = tokio::time::sleep(Duration::from_secs(30)) => {}, result = shutdown.changed() => { if result.is_err() { break; } }, diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index 42b2fb2ea4..220a4d506e 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -11,7 +11,7 @@ use crate::scheduler::types::RunLauncher; use crate::store::errors::ControlError; use crate::store::models::ControlStore; -/// Durable database state drives work; a transient store failure retries the next cycle. +/// Durable database state drives work; a transient failure or a lost lease retries the next cycle. pub async fn run_scheduler( store: &ControlStore, mut state: SchedulerState, @@ -36,7 +36,7 @@ pub async fn run_scheduler( let retry = Instant::now() + Duration::from_secs(1); let due = match polled { Ok(due) => due.min(retry), - Err(error) if error.transient() => { + Err(error) if error.transient() || matches!(error, ControlError::Fenced) => { launcher.retrying(&error); retry } diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs index 1515b87c9d..ec0167dcd8 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -21,6 +21,6 @@ pub trait RunLauncher { /// Hears each completed cycle, with its phase timings, before the launcher polls. fn cycled(&mut self, _cycle: &SchedulerCycle) {} - /// Hears each transient store failure the scheduler retries instead of stopping. + /// Hears each transient store failure or lost lease the scheduler retries instead of stopping. fn retrying(&mut self, _error: &ControlError) {} } diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs index b4ca936711..4257371c1f 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs @@ -1,10 +1,11 @@ //! Configure a file-backed SQLite control store without interpreting filenames as URIs. use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use crate::store::errors::ControlError; -use crate::store::models::ControlStore; +use crate::store::models::{ControlStore, SqliteStore}; pub(crate) fn sqlite_path(url: &str) -> Result { let path = Path::new(url.strip_prefix("sqlite://").unwrap_or(url)).to_path_buf(); @@ -46,6 +47,10 @@ pub(crate) fn open_sqlite(path: PathBuf) -> Result { | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; connection.busy_timeout(Duration::from_secs(30))?; - connection.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL")?; - Ok(ControlStore::Sqlite(path)) + connection.execute_batch("PRAGMA journal_mode=WAL")?; + connection.query_row("SELECT count(*) FROM sqlite_schema", [], |_| Ok(()))?; + Ok(ControlStore::Sqlite(SqliteStore { + path, + _keeper: Arc::new(Mutex::new(connection)), + })) } diff --git a/crates/sqlbuild-orchestration/src/store/constants.rs b/crates/sqlbuild-orchestration/src/store/constants.rs new file mode 100644 index 0000000000..61d1f56a56 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/constants.rs @@ -0,0 +1,4 @@ +//! Control-store limits. + +/// A SQLite WAL past this size is reset right after the commit that grew it. +pub(crate) const LARGE_WAL_BYTES: u64 = 64 * 1024 * 1024; diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs index 94af142705..7deced8adc 100644 --- a/crates/sqlbuild-orchestration/src/store/main/begin.rs +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -4,6 +4,7 @@ use std::time::Duration; use chrono::Utc; use rusqlite::OpenFlags; +use rusqlite::config::DbConfig; use crate::store::_helpers::changes::sqlite_changes; use crate::store::errors::ControlError; @@ -25,8 +26,9 @@ pub(crate) async fn begin( | OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; connection.busy_timeout(Duration::from_secs(30))?; + connection.set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true)?; connection.execute_batch(if mode == TransactionMode::Write { - "PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" + "PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" } else { "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" })?; diff --git a/crates/sqlbuild-orchestration/src/store/main/commit.rs b/crates/sqlbuild-orchestration/src/store/main/commit.rs index cc72f730ed..e110c12a7c 100644 --- a/crates/sqlbuild-orchestration/src/store/main/commit.rs +++ b/crates/sqlbuild-orchestration/src/store/main/commit.rs @@ -1,6 +1,7 @@ //! Commit facts and their projections together. use crate::store::_helpers::changes::resolve_changes; +use crate::store::constants::LARGE_WAL_BYTES; use crate::store::errors::ControlError; use crate::store::models::{Connection, ControlTransaction}; @@ -14,6 +15,9 @@ pub(crate) async fn commit(transaction: ControlTransaction) -> Result<(), Contro Connection::Sqlite(connection) => { tokio::task::spawn_blocking(move || { connection.execute_batch("COMMIT")?; + if transaction.notifications.is_some() { + drop(truncate_large_wal(&connection)); + } if let Some(sender) = transaction.notifications { for message in messages { let _ = sender.send(message); @@ -39,3 +43,17 @@ pub(crate) async fn commit(transaction: ControlTransaction) -> Result<(), Contro } Ok(()) } + +/// Reset a WAL a large transaction left, which slows reads, unless readers hold it; best effort. +fn truncate_large_wal(connection: &rusqlite::Connection) -> Result<(), rusqlite::Error> { + let Some(database) = connection.path() else { + return Ok(()); + }; + let large = std::fs::metadata(format!("{database}-wal")) + .is_ok_and(|metadata| metadata.len() > LARGE_WAL_BYTES); + if large { + connection.busy_timeout(std::time::Duration::ZERO)?; + connection.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/store/mod.rs b/crates/sqlbuild-orchestration/src/store/mod.rs index 1228df7ae6..aca6885a73 100644 --- a/crates/sqlbuild-orchestration/src/store/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod errors; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index 0e93f7ed72..fc9a7371b2 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -1,7 +1,8 @@ //! Native database handles and bound values shared by control-plane operations. use std::collections::BTreeSet; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -13,10 +14,31 @@ use uuid::Uuid; /// An installation's control store. Workers and locations never receive this handle. #[derive(Clone, Debug)] pub enum ControlStore { - Sqlite(PathBuf), + Sqlite(SqliteStore), Postgres(sqlx_postgres::PgPool), } +/// A SQLite store's file, and an idle connection that keeps its WAL index between transactions. +#[derive(Clone, Debug)] +pub struct SqliteStore { + pub(crate) path: PathBuf, + pub(crate) _keeper: Arc>, +} + +impl std::ops::Deref for SqliteStore { + type Target = Path; + + fn deref(&self) -> &Path { + &self.path + } +} + +impl AsRef for SqliteStore { + fn as_ref(&self) -> &Path { + &self.path + } +} + /// The first copy wins; duplicate events retain their original storage cursor. #[derive(Clone, Debug, PartialEq, Eq)] pub struct RecallAppendOutcome { diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index 34c98d16ef..b892b9daaa 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -26,9 +26,17 @@ pub(crate) fn hold_lease( let store = store.clone(); async move { tokio::time::sleep(std::time::Duration::from_secs(5)).await; + let started = std::time::Instant::now(); renew_lease(&store, &lease) .await .map(|renewed| Some(((), renewed))) + .inspect_err(|error| { + println!( + "lease holder stopped: renewal failed after {:?} with {error}; the lease was due to expire at {}", + started.elapsed(), + lease.expires_at + ); + }) } }); tokio::spawn(renewals.for_each(|_| async {})) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 2d684232f3..4b4134eb01 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -406,28 +406,30 @@ pub(crate) async fn shift_schedule(store: &ControlStore, trigger_id: Uuid, days: } /// A later covered tick, as if another due time arrived while the requested run still waited. -pub(crate) async fn append_covered_tick( +pub(crate) async fn append_covered_ticks( store: &ControlStore, trigger_id: Uuid, - scheduled_for: DateTime, + scheduled_for: &[DateTime], ) { - let transaction = begin(store, TransactionMode::Write) + let mut transaction = begin(store, TransactionMode::Write) .await - .expect("covered tick"); - let (transaction, _) = query( - transaction, - "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, \ - run_id, covered_by_run_id) SELECT $1, trigger_id, $2, evaluated_at, epoch, \ - 'covered', NULL, run_id FROM sqb_ticks WHERE trigger_id=$3 AND outcome='requested'", - vec![ - Parameter::Id(Uuid::now_v7()), - Parameter::Time(scheduled_for), - Parameter::Id(trigger_id), - ], - ) - .await - .expect("append covered tick"); - commit(transaction).await.expect("commit covered tick"); + .expect("covered ticks"); + for time in scheduled_for { + (transaction, _) = query( + transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, \ + run_id, covered_by_run_id) SELECT $1, trigger_id, $2, evaluated_at, epoch, \ + 'covered', NULL, run_id FROM sqb_ticks WHERE trigger_id=$3 AND outcome='requested'", + vec![ + Parameter::Id(Uuid::now_v7()), + Parameter::Time(*time), + Parameter::Id(trigger_id), + ], + ) + .await + .expect("append covered tick"); + } + commit(transaction).await.expect("commit covered ticks"); } /// Rewrite one trigger-state column the way an earlier build or a past start would have left it. diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index 69106153cc..dd01a3869f 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -12,7 +12,7 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::ticks::helpers::snapshot; use crate::scheduler::tests::contracts::ticks::test_types::{ - IdleLauncher, TickTransactionTestCase, + FencedLoopTestCase, FencingLauncher, IdleLauncher, TickTransactionTestCase, }; use crate::store::tests::lease_helpers::expire_lease; use crate::store::tests::replica_helpers::second_store; @@ -99,3 +99,56 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain }); } } + +#[test] +fn given_a_leader_fenced_mid_cycle_when_the_loop_continues_then_it_rejoins_the_election_instead_of_stopping() + { + let test_cases = [ + FencedLoopTestCase { + description: "SQLite scheduler loop", + backend: Backend::Sqlite, + expected_rejoined: true, + }, + FencedLoopTestCase { + description: "PostgreSQL scheduler loop", + backend: Backend::Postgres, + expected_rejoined: true, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate loop store"); + let mut epochs = Vec::new(); + let (stop, shutdown) = tokio::sync::watch::channel(false); + + let finished = tokio::time::timeout( + Duration::from_secs(30), + run_scheduler( + store, + SchedulerState::new(Uuid::now_v7()), + shutdown, + FencingLauncher { + epochs: &mut epochs, + stop: &stop, + }, + ), + ) + .await; + + finished + .expect("the scheduler stops once it leads again") + .expect("a fenced leader keeps the scheduler running"); + assert_eq!( + epochs.windows(2).any(|pair| pair[1] > pair[0]), + test_case.expected_rejoined, + "{}: leads again under a newer epoch, polled with {epochs:?}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs index ffcd7d1ac2..71f3ed3af2 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs @@ -17,7 +17,7 @@ use crate::scheduler::main::{ use crate::scheduler::models::TickOutcome; use crate::scheduler::tests::contracts::helpers::hold_lease; use crate::scheduler::tests::contracts::ticks::helpers::{ - append_covered_tick, calendar_time, shift_schedule, snapshot, + append_covered_ticks, calendar_time, shift_schedule, snapshot, }; use crate::scheduler::tests::contracts::ticks::test_types::{ LatestCatchUpTestCase, TickRetentionTestCase, @@ -87,17 +87,22 @@ fn given_expired_covered_ticks_when_pruned_and_cursor_reset_then_run_linked_and_ ); let later = calendar_time(2) + TimeDelta::hours(1); let backlog = case["backlog_ticks"].as_i64().expect("backlog"); - for second in 1..=backlog { - append_covered_tick( - store, - trigger_id, - calendar_time(2) + TimeDelta::seconds(second), - ) - .await; - } - append_covered_tick(store, trigger_id, later).await; - // Inserting the backlog one tick at a time can outlast the lease on a slow - // machine; the scheduler renews between passes, so the test does too. + let started = std::time::Instant::now(); + append_covered_ticks( + store, + trigger_id, + &(1..=backlog) + .map(|second| calendar_time(2) + TimeDelta::seconds(second)) + .chain([later]) + .collect::>(), + ) + .await; + println!( + "{}: backlog written in {:?}", + test_case.description, + started.elapsed() + ); + // The scheduler renews between passes, so the test does too. let lease = renew_lease(store, &lease).await.expect("renew"); assert_eq!(prune_ticks(store, &lease).await.expect("recent ticks"), 0); shift_schedule(store, trigger_id, shift).await; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 146b81d23b..dafaebb699 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -82,6 +82,49 @@ impl crate::scheduler::types::RunLauncher for CycleRecorder<'_> { } } +/// Fences the first lease it polls with, and stops the loop once it polls under a newer epoch. +pub(crate) struct FencingLauncher<'a> { + pub(crate) epochs: &'a mut Vec, + pub(crate) stop: &'a tokio::sync::watch::Sender, +} + +impl crate::scheduler::types::RunLauncher for FencingLauncher<'_> { + async fn poll( + &mut self, + store: &crate::store::models::ControlStore, + lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + let Some(lease) = lease else { + return Ok(()); + }; + self.epochs.push(lease.epoch); + if self.epochs.len() > 1 { + if self.epochs.windows(2).any(|pair| pair[1] > pair[0]) { + self.stop.send_replace(true); + } + return Ok(()); + } + crate::store::tests::lease_helpers::expire_lease(store).await; + crate::scheduler::main::renew_lease::renew_lease(store, lease) + .await + .map(drop) + } + + async fn shutdown( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + Ok(()) + } +} + +pub(crate) struct FencedLoopTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_rejoined: bool, +} + pub(crate) struct TickRetentionTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs index bc192f33ba..fa10659ee3 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs @@ -1,7 +1,121 @@ -//! Row readers shared by the control-store schema contracts. +//! Row readers and a killable writer process shared by the control-store schema contracts. +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::{Command, Stdio}; + +use crate::credentials::tests::location_helpers::create_location; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::open_store::open_store; +use crate::store::main::query::query; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::store_helpers::run; use crate::store::types::Row; +const CHILD_STORE_VARIABLE: &str = "SQLBUILD_TEST_CRASH_STORE"; +const ACKNOWLEDGED: &str = "committed "; + pub(crate) fn text(row: &Row, key: &str) -> String { row[key].as_str().expect("text column").to_owned() } + +/// In a `kill_writer_after` process, commit one location at a time and acknowledge each, forever. +pub(crate) fn write_until_killed_when_spawned() { + let Ok(path) = std::env::var(CHILD_STORE_VARIABLE) else { + return; + }; + run(async { + let store = open_store(&path).await.expect("open store"); + migrate_store(&store).await.expect("migrate store"); + for index in 0..u64::MAX { + create_location(&store, &format!("orders-{index}"), "test-admin") + .await + .expect("create location"); + println!("{ACKNOWLEDGED}{index}"); + } + }); +} + +/// Kill a writer running `test` after `commits` acknowledgements; return them and the WAL bytes. +pub(crate) fn kill_writer_after(test: &str, path: &Path, commits: usize) -> (usize, u64) { + let name = format!( + "{}::{test}", + module_path!() + .split_once("::") + .expect("crate path") + .1 + .rsplit_once("::") + .expect("schema module") + .0 + ); + let mut child = Command::new(std::env::current_exe().expect("test binary")) + .args([name.as_str(), "--exact", "--nocapture", "--test-threads=1"]) + .env(CHILD_STORE_VARIABLE, path) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("start writer process"); + let acknowledged = BufReader::new(child.stdout.take().expect("writer output")) + .lines() + .map(|line| line.expect("writer line")) + .filter(|line| line.starts_with(ACKNOWLEDGED)) + .take(commits) + .count(); + child.kill().expect("kill writer"); + child.wait().expect("reap writer"); + let mut wal = path.as_os_str().to_owned(); + wal.push("-wal"); + let wal_bytes = std::fs::metadata(wal).map_or(0, |metadata| metadata.len()); + (acknowledged, wal_bytes) +} + +/// The number of locations a reopened store holds. +pub(crate) fn surviving_locations(path: &Path) -> usize { + run(async { + let store = open_store(path.to_str().expect("UTF-8 path")) + .await + .expect("reopen store"); + let transaction = begin(&store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, rows) = query( + transaction, + "SELECT COUNT(*) AS locations FROM sqb_locations", + vec![], + ) + .await + .expect("count locations"); + commit(transaction).await.expect("finish read"); + usize::try_from(rows[0]["locations"].as_i64().expect("count")).expect("count") + }) +} + +/// Commit `commits` rows of `bytes` each, one transaction at a time with none overlapping; return the WAL bytes. +pub(crate) fn wal_bytes_after_sequential_writes(path: &Path, commits: usize, bytes: usize) -> u64 { + run(async { + let store = open_store(path.to_str().expect("UTF-8 path")) + .await + .expect("open store"); + let transaction = begin(&store, TransactionMode::Write).await.expect("begin"); + let (transaction, _) = query(transaction, "CREATE TABLE wal_probe (payload TEXT)", vec![]) + .await + .expect("create probe table"); + commit(transaction).await.expect("commit table"); + for index in 0..commits { + let transaction = begin(&store, TransactionMode::Write).await.expect("begin"); + let (transaction, _) = query( + transaction, + "INSERT INTO wal_probe (payload) VALUES ($1)", + vec![Parameter::Text(format!("{index:0>bytes$}"))], + ) + .await + .expect("write probe row"); + commit(transaction).await.expect("commit probe row"); + } + let mut wal = path.as_os_str().to_owned(); + wal.push("-wal"); + std::fs::metadata(wal).map_or(0, |metadata| metadata.len()) + }) +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs index dd9741305f..6a1a10b378 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs @@ -2,6 +2,8 @@ #[path = "helpers.rs"] pub(crate) mod helpers; +#[path = "test_crash_recovery.rs"] +mod test_crash_recovery; #[path = "test_foreign_key_indexes.rs"] mod test_foreign_key_indexes; #[path = "test_store_upgrade.rs"] diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs new file mode 100644 index 0000000000..b9a6af783c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs @@ -0,0 +1,127 @@ +//! Commits a killed process acknowledged survive, though closing a connection never checkpoints. + +use crate::platform::_helpers::retention::wal_checkpoint; +use crate::store::main::open_store::open_store; +use crate::store::tests::schema::helpers::{ + kill_writer_after, surviving_locations, wal_bytes_after_sequential_writes, + write_until_killed_when_spawned, +}; +use crate::store::tests::schema::test_types::{ + CrashRecoveryTestCase, WalCheckpointTestCase, WalGrowthTestCase, +}; +use crate::store::tests::store_helpers::run; + +#[test] +fn given_a_process_killed_between_commits_when_the_store_reopens_then_every_acknowledged_commit_survives() + { + write_until_killed_when_spawned(); + let test_cases = [ + CrashRecoveryTestCase { + description: "killed after the first commits", + expected_acknowledged: 5, + }, + CrashRecoveryTestCase { + description: "killed after many commits", + expected_acknowledged: 200, + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("crash store directory"); + let path = directory.path().join("control.sqlite3"); + + let (acknowledged, wal_bytes) = kill_writer_after( + "test_crash_recovery::given_a_process_killed_between_commits_when_the_store_reopens_then_every_acknowledged_commit_survives", + &path, + test_case.expected_acknowledged, + ); + let survived = surviving_locations(&path); + + assert_eq!( + acknowledged, test_case.expected_acknowledged, + "{}: the writer was killed, not finished", + test_case.description + ); + assert!( + wal_bytes > 0, + "{}: the killed writer left its commits in the WAL", + test_case.description + ); + assert!( + survived >= test_case.expected_acknowledged, + "{}: {acknowledged} acknowledged, {survived} survived", + test_case.description + ); + } +} + +#[test] +fn given_commits_left_in_the_wal_when_maintenance_checkpoints_then_the_wal_is_emptied_and_rows_remain() + { + write_until_killed_when_spawned(); + let test_cases = [WalCheckpointTestCase { + description: "a killed writer's WAL", + expected_acknowledged: 20, + expected_wal_bytes: 0, + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("checkpoint store directory"); + let path = directory.path().join("control.sqlite3"); + let (acknowledged, left) = kill_writer_after( + "test_crash_recovery::given_commits_left_in_the_wal_when_maintenance_checkpoints_then_the_wal_is_emptied_and_rows_remain", + &path, + test_case.expected_acknowledged, + ); + + let wal_bytes = run(async { + let store = open_store(path.to_str().expect("UTF-8 path")) + .await + .expect("open store"); + wal_checkpoint(&store).await.expect("checkpoint"); + std::fs::metadata(directory.path().join("control.sqlite3-wal")) + .expect("WAL file") + .len() + }); + let survived = surviving_locations(&path); + + assert_eq!( + acknowledged, test_case.expected_acknowledged, + "{}", + test_case.description + ); + assert!(left > 0, "{}: the WAL held commits", test_case.description); + assert_eq!( + wal_bytes, test_case.expected_wal_bytes, + "{}", + test_case.description + ); + assert!( + survived >= test_case.expected_acknowledged, + "{}: {survived} survived", + test_case.description + ); + } +} + +#[test] +fn given_transactions_that_never_overlap_when_many_commit_then_the_wal_restarts_instead_of_growing() +{ + let test_cases = [WalGrowthTestCase { + description: "600 commits of 16 KiB, about 2,700 WAL pages", + commits: 600, + row_bytes: 16 * 1024, + expected_max_wal_bytes: 8 * 1024 * 1024, + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("WAL store directory"); + let path = directory.path().join("control.sqlite3"); + + let wal_bytes = + wal_bytes_after_sequential_writes(&path, test_case.commits, test_case.row_bytes); + + assert!( + wal_bytes <= test_case.expected_max_wal_bytes, + "{}: the WAL grew to {wal_bytes} bytes", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs index 748d56da48..65f75293d7 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs @@ -4,6 +4,24 @@ use std::collections::BTreeSet; use crate::store::tests::store_test_types::Backend; +pub(crate) struct CrashRecoveryTestCase { + pub(crate) description: &'static str, + pub(crate) expected_acknowledged: usize, +} + +pub(crate) struct WalCheckpointTestCase { + pub(crate) description: &'static str, + pub(crate) expected_acknowledged: usize, + pub(crate) expected_wal_bytes: u64, +} + +pub(crate) struct WalGrowthTestCase { + pub(crate) description: &'static str, + pub(crate) commits: usize, + pub(crate) row_bytes: usize, + pub(crate) expected_max_wal_bytes: u64, +} + pub(crate) struct ForeignKeyIndexTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/deploy/README.md b/deploy/README.md index e743652bea..402ed32e28 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -354,7 +354,10 @@ outside this stack too: a stopped server cannot report its own outage. ## Back up and upgrade -Back up PostgreSQL with `pg_dump` using your normal protected database backup tooling. Also +Back up PostgreSQL with `pg_dump` using your normal protected database backup tooling. Back up +a SQLite control store with `sqlite3 ".backup ''"` while the server runs, or stop the +server and copy the database file together with its `-wal` file; the database file alone can miss +recent commits. Also back up warehouse data, the restricted `.env`, the `server` config, and referenced project images. Keep backup credentials and output outside the repository. Test restoration before an upgrade. Ordinary `docker compose down` preserves named volumes; `down --volumes` destroys them. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index e0beb2e323..79f3e5074c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/cli/server/ - Configure the server - Serve over HTTPS - Receive platform events +- Back up the control database - Upgrade - Set retention - Grant roles to OIDC groups (beta) @@ -154,6 +155,26 @@ characters, and `null` when the worker reported none. It can quote warehouse dat webhook like other run evidence. `run_failed` events appear only in the webhook, not in the server's platform history. +## Back up the control database + +With no `SQLBUILD_CONTROL_DATABASE_URL`, the server keeps its control database in one SQLite file. +Back it up with SQLite's online backup while the server runs: + +```bash +sqlite3 control.sqlite3 ".backup '/backups/control.sqlite3'" +``` + +| Store | Back up with | While the server runs | +| --- | --- | --- | +| SQLite | `sqlite3 ".backup ''"` or the SQLite backup API | Yes | +| SQLite | A copy of the database file together with its `-wal` file | No; stop the server first | +| PostgreSQL | `pg_dump` | Yes | + +- Never copy the SQLite database file alone. Recent commits can still be in the `-wal` file next to + it, so the copy misses them. +- Restore a backup while the server is stopped, and delete any `-wal` and `-shm` files left beside + the database you replace. + ## Upgrade Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 0cb5dc46d5..11832d7a93 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -137,6 +137,26 @@ characters, and `null` when the worker reported none. It can quote warehouse dat webhook like other run evidence. `run_failed` events appear only in the webhook, not in the server's platform history. +## Back up the control database + +With no `SQLBUILD_CONTROL_DATABASE_URL`, the server keeps its control database in one SQLite file. +Back it up with SQLite's online backup while the server runs: + +```bash +sqlite3 control.sqlite3 ".backup '/backups/control.sqlite3'" +``` + +| Store | Back up with | While the server runs | +| --- | --- | --- | +| SQLite | `sqlite3 ".backup ''"` or the SQLite backup API | Yes | +| SQLite | A copy of the database file together with its `-wal` file | No; stop the server first | +| PostgreSQL | `pg_dump` | Yes | + +- Never copy the SQLite database file alone. Recent commits can still be in the `-wal` file next to + it, so the copy misses them. +- Restore a backup while the server is stopped, and delete any `-wal` and `-shm` files left beside + the database you replace. + ## Upgrade Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the