Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,12 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers {
self.recovered_epoch = None;
drop(writeln!(
std::io::stderr().lock(),
"Control store unavailable; the scheduler retries: {error}"
"{}; the scheduler retries: {error}",
if matches!(error, ControlError::Fenced) {
"Scheduler fenced"
} else {
"Control store unavailable"
}
));
}
}
Expand Down
20 changes: 19 additions & 1 deletion crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
//! Every retention policy runs in one bounded pass; each part is safe to repeat.

use std::collections::BTreeSet;
use std::time::Duration;

use chrono::{DateTime, TimeDelta, Utc};
use rusqlite::config::DbConfig;
use sqlbuild_recall::schema::constants::INVOCATION_TABLES;
use sqlbuild_recall::schema::main::table_spec::table_spec;

use crate::credentials::main::audit::prune_audit::prune_audit;
use crate::platform::constants::{
HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS,
CHECKPOINT_READER_WAIT_MS, HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS,
PLATFORM_ALERT_RETENTION_DAYS, SENSOR_EVALUATION_RETENTION_DAYS,
};
use crate::platform::models::RetentionPolicies;
Expand Down Expand Up @@ -212,3 +214,19 @@ async fn delete_logs(
}
Ok(transaction)
}

/// Copy the WAL into the database and truncate it, or leave it for the next pass if readers stay.
pub(crate) async fn wal_checkpoint(store: &ControlStore) -> Result<(), ControlError> {
let ControlStore::Sqlite(path) = store else {
return Ok(());
};
let path = path.clone();
tokio::task::spawn_blocking(move || {
let connection = rusqlite::Connection::open(&path)?;
connection.set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true)?;
connection.busy_timeout(Duration::from_millis(CHECKPOINT_READER_WAIT_MS))?;
connection.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))?;
Ok(())
})
.await?
}
3 changes: 3 additions & 0 deletions crates/sqlbuild-orchestration/src/platform/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,6 @@ pub(crate) const PLAN_QUEUE_LIMIT: i64 = 4;

/// A location instance unseen for longer than this cannot take a plan; locations report every 30 seconds.
pub(crate) const LOCATION_SILENCE_SECONDS: i64 = 90;

/// A SQLite WAL reset waits this long for readers while holding the write lock, then retries later.
pub(crate) const CHECKPOINT_READER_WAIT_MS: u64 = 200;
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
//! One maintenance role per server: health alerts every cycle, bounded retention every minute.
//! One maintenance role per server: health alerts and WAL resets every cycle, retention every minute.

use std::time::{Duration, Instant};

use tokio::sync::watch;
use uuid::Uuid;

use crate::platform::_helpers::retention::retain;
use crate::platform::_helpers::retention::{retain, wal_checkpoint};
use crate::platform::_helpers::watchdog::scan_health;
use crate::platform::models::RetentionPolicies;
use crate::store::errors::ControlError;
Expand Down Expand Up @@ -44,6 +44,11 @@ pub async fn run_maintenance(
}
retained_at = Some(Instant::now());
}
match wal_checkpoint(store).await {
Ok(()) => {}
Err(error) if error.transient() => {}
Err(error) => return Err(error),
}
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(30)) => {},
result = shutdown.changed() => { if result.is_err() { break; } },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use crate::scheduler::types::RunLauncher;
use crate::store::errors::ControlError;
use crate::store::models::ControlStore;

/// Durable database state drives work; a transient store failure retries the next cycle.
/// Durable database state drives work; a transient failure or a lost lease retries the next cycle.
pub async fn run_scheduler(
store: &ControlStore,
mut state: SchedulerState,
Expand All @@ -36,7 +36,7 @@ pub async fn run_scheduler(
let retry = Instant::now() + Duration::from_secs(1);
let due = match polled {
Ok(due) => due.min(retry),
Err(error) if error.transient() => {
Err(error) if error.transient() || matches!(error, ControlError::Fenced) => {
launcher.retrying(&error);
retry
}
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlbuild-orchestration/src/scheduler/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,6 @@ pub trait RunLauncher {
/// Hears each completed cycle, with its phase timings, before the launcher polls.
fn cycled(&mut self, _cycle: &SchedulerCycle) {}

/// Hears each transient store failure the scheduler retries instead of stopping.
/// Hears each transient store failure or lost lease the scheduler retries instead of stopping.
fn retrying(&mut self, _error: &ControlError) {}
}
11 changes: 8 additions & 3 deletions crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
//! Configure a file-backed SQLite control store without interpreting filenames as URIs.

use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::Duration;

use crate::store::errors::ControlError;
use crate::store::models::ControlStore;
use crate::store::models::{ControlStore, SqliteStore};

pub(crate) fn sqlite_path(url: &str) -> Result<PathBuf, ControlError> {
let path = Path::new(url.strip_prefix("sqlite://").unwrap_or(url)).to_path_buf();
Expand Down Expand Up @@ -46,6 +47,10 @@ pub(crate) fn open_sqlite(path: PathBuf) -> Result<ControlStore, ControlError> {
| rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
connection.busy_timeout(Duration::from_secs(30))?;
connection.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL")?;
Ok(ControlStore::Sqlite(path))
connection.execute_batch("PRAGMA journal_mode=WAL")?;
connection.query_row("SELECT count(*) FROM sqlite_schema", [], |_| Ok(()))?;
Ok(ControlStore::Sqlite(SqliteStore {
path,
_keeper: Arc::new(Mutex::new(connection)),
}))
}
4 changes: 4 additions & 0 deletions crates/sqlbuild-orchestration/src/store/constants.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
//! Control-store limits.

/// A SQLite WAL past this size is reset right after the commit that grew it.
pub(crate) const LARGE_WAL_BYTES: u64 = 64 * 1024 * 1024;
4 changes: 3 additions & 1 deletion crates/sqlbuild-orchestration/src/store/main/begin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use std::time::Duration;

use chrono::Utc;
use rusqlite::OpenFlags;
use rusqlite::config::DbConfig;

use crate::store::_helpers::changes::sqlite_changes;
use crate::store::errors::ControlError;
Expand All @@ -25,8 +26,9 @@ pub(crate) async fn begin(
| OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
connection.busy_timeout(Duration::from_secs(30))?;
connection.set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true)?;
connection.execute_batch(if mode == TransactionMode::Write {
"PRAGMA foreign_keys=ON; BEGIN IMMEDIATE"
"PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE"
} else {
"PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN"
})?;
Expand Down
18 changes: 18 additions & 0 deletions crates/sqlbuild-orchestration/src/store/main/commit.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! Commit facts and their projections together.

use crate::store::_helpers::changes::resolve_changes;
use crate::store::constants::LARGE_WAL_BYTES;
use crate::store::errors::ControlError;
use crate::store::models::{Connection, ControlTransaction};

Expand All @@ -14,6 +15,9 @@ pub(crate) async fn commit(transaction: ControlTransaction) -> Result<(), Contro
Connection::Sqlite(connection) => {
tokio::task::spawn_blocking(move || {
connection.execute_batch("COMMIT")?;
if transaction.notifications.is_some() {
drop(truncate_large_wal(&connection));
}
if let Some(sender) = transaction.notifications {
for message in messages {
let _ = sender.send(message);
Expand All @@ -39,3 +43,17 @@ pub(crate) async fn commit(transaction: ControlTransaction) -> Result<(), Contro
}
Ok(())
}

/// Reset a WAL a large transaction left, which slows reads, unless readers hold it; best effort.
fn truncate_large_wal(connection: &rusqlite::Connection) -> Result<(), rusqlite::Error> {
let Some(database) = connection.path() else {
return Ok(());
};
let large = std::fs::metadata(format!("{database}-wal"))
.is_ok_and(|metadata| metadata.len() > LARGE_WAL_BYTES);
if large {
connection.busy_timeout(std::time::Duration::ZERO)?;
connection.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))?;
}
Ok(())
}
1 change: 1 addition & 0 deletions crates/sqlbuild-orchestration/src/store/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
pub(crate) mod _helpers;
pub(crate) mod constants;
pub mod errors;
pub mod main;
pub mod models;
Expand Down
26 changes: 24 additions & 2 deletions crates/sqlbuild-orchestration/src/store/models.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
//! Native database handles and bound values shared by control-plane operations.

use std::collections::BTreeSet;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
Expand All @@ -13,10 +14,31 @@ use uuid::Uuid;
/// An installation's control store. Workers and locations never receive this handle.
#[derive(Clone, Debug)]
pub enum ControlStore {
Sqlite(PathBuf),
Sqlite(SqliteStore),
Postgres(sqlx_postgres::PgPool),
}

/// A SQLite store's file, and an idle connection that keeps its WAL index between transactions.
#[derive(Clone, Debug)]
pub struct SqliteStore {
pub(crate) path: PathBuf,
pub(crate) _keeper: Arc<Mutex<rusqlite::Connection>>,
}

impl std::ops::Deref for SqliteStore {
type Target = Path;

fn deref(&self) -> &Path {
&self.path
}
}

impl AsRef<Path> for SqliteStore {
fn as_ref(&self) -> &Path {
&self.path
}
}

/// The first copy wins; duplicate events retain their original storage cursor.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RecallAppendOutcome {
Expand Down
8 changes: 8 additions & 0 deletions crates/sqlbuild-orchestration/tests/scheduler/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,17 @@ pub(crate) fn hold_lease(
let store = store.clone();
async move {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
let started = std::time::Instant::now();
renew_lease(&store, &lease)
.await
.map(|renewed| Some(((), renewed)))
.inspect_err(|error| {
println!(
"lease holder stopped: renewal failed after {:?} with {error}; the lease was due to expire at {}",
started.elapsed(),
lease.expires_at
);
})
}
});
tokio::spawn(renewals.for_each(|_| async {}))
Expand Down
38 changes: 20 additions & 18 deletions crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -406,28 +406,30 @@ pub(crate) async fn shift_schedule(store: &ControlStore, trigger_id: Uuid, days:
}

/// A later covered tick, as if another due time arrived while the requested run still waited.
pub(crate) async fn append_covered_tick(
pub(crate) async fn append_covered_ticks(
store: &ControlStore,
trigger_id: Uuid,
scheduled_for: DateTime<Utc>,
scheduled_for: &[DateTime<Utc>],
) {
let transaction = begin(store, TransactionMode::Write)
let mut transaction = begin(store, TransactionMode::Write)
.await
.expect("covered tick");
let (transaction, _) = query(
transaction,
"INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, \
run_id, covered_by_run_id) SELECT $1, trigger_id, $2, evaluated_at, epoch, \
'covered', NULL, run_id FROM sqb_ticks WHERE trigger_id=$3 AND outcome='requested'",
vec![
Parameter::Id(Uuid::now_v7()),
Parameter::Time(scheduled_for),
Parameter::Id(trigger_id),
],
)
.await
.expect("append covered tick");
commit(transaction).await.expect("commit covered tick");
.expect("covered ticks");
for time in scheduled_for {
(transaction, _) = query(
transaction,
"INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, \
run_id, covered_by_run_id) SELECT $1, trigger_id, $2, evaluated_at, epoch, \
'covered', NULL, run_id FROM sqb_ticks WHERE trigger_id=$3 AND outcome='requested'",
vec![
Parameter::Id(Uuid::now_v7()),
Parameter::Time(*time),
Parameter::Id(trigger_id),
],
)
.await
.expect("append covered tick");
}
commit(transaction).await.expect("commit covered ticks");
}

/// Rewrite one trigger-state column the way an earlier build or a past start would have left it.
Expand Down
55 changes: 54 additions & 1 deletion crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use uuid::Uuid;
use crate::credentials::tests::location_helpers::create_location;
use crate::scheduler::tests::contracts::ticks::helpers::snapshot;
use crate::scheduler::tests::contracts::ticks::test_types::{
IdleLauncher, TickTransactionTestCase,
FencedLoopTestCase, FencingLauncher, IdleLauncher, TickTransactionTestCase,
};
use crate::store::tests::lease_helpers::expire_lease;
use crate::store::tests::replica_helpers::second_store;
Expand Down Expand Up @@ -99,3 +99,56 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain
});
}
}

#[test]
fn given_a_leader_fenced_mid_cycle_when_the_loop_continues_then_it_rejoins_the_election_instead_of_stopping()
{
let test_cases = [
FencedLoopTestCase {
description: "SQLite scheduler loop",
backend: Backend::Sqlite,
expected_rejoined: true,
},
FencedLoopTestCase {
description: "PostgreSQL scheduler loop",
backend: Backend::Postgres,
expected_rejoined: true,
},
];
for test_case in test_cases {
run(async {
let Some(fixture) = open_test_store(test_case.backend).await else {
return;
};
let store = &fixture.store;
migrate_store(store).await.expect("migrate loop store");
let mut epochs = Vec::new();
let (stop, shutdown) = tokio::sync::watch::channel(false);

let finished = tokio::time::timeout(
Duration::from_secs(30),
run_scheduler(
store,
SchedulerState::new(Uuid::now_v7()),
shutdown,
FencingLauncher {
epochs: &mut epochs,
stop: &stop,
},
),
)
.await;

finished
.expect("the scheduler stops once it leads again")
.expect("a fenced leader keeps the scheduler running");
assert_eq!(
epochs.windows(2).any(|pair| pair[1] > pair[0]),
test_case.expected_rejoined,
"{}: leads again under a newer epoch, polled with {epochs:?}",
test_case.description
);
close_test_store(fixture).await;
});
}
}
Loading
Loading