Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
14a9c18
fix(server): index every control-store foreign key
kvlonge Nov 14, 2026
5665a37
fix: write keyed delete_insert atomically, add store indexes in place…
kvlonge Nov 14, 2026
329044b
style: shorten the Databricks keyed MERGE docstring
kvlonge Nov 14, 2026
03f1605
fix: stop a loader from writing to a schema named preserve
kvlonge Nov 14, 2026
268e615
fix: say a connection schema counts only for a project's only top-lev…
kvlonge Nov 14, 2026
5d14d67
fix: show the S104 hint's connection schema with command-line vars ap…
kvlonge Nov 14, 2026
c3a2bbe
fix(debug): report no database for DuckDB targets
kvlonge Nov 14, 2026
9fd9c73
fix: check only intermediate loaders under a preserved target schema
kvlonge Nov 14, 2026
b01defa
refactor: read the loader node flag without a Python call
kvlonge Nov 14, 2026
d61540a
fix(load): load a managed source into the target's loader_schema
kvlonge Nov 14, 2026
c63997b
refactor(load): pick a managed source's schema without comparing fiel…
kvlonge Nov 14, 2026
0be3958
fix(load): stop a managed source that would load into a schema named …
kvlonge Nov 14, 2026
519b010
fix(load): expand templated target locations for managed sources
kvlonge Nov 14, 2026
4d958c7
fix(load): expand only the target location a managed source falls bac…
kvlonge Nov 14, 2026
6fa7272
fix(load): read an expanded target location as text, as build does
kvlonge Nov 14, 2026
411f16f
docs(deploy): run the server and a location as Podman Quadlet units
kvlonge Nov 14, 2026
dc9e1c7
fix(scheduler): wake on a due time however long the store took, and l…
kvlonge Nov 14, 2026
0b570ab
refactor: share target lookup and load-selection request reading
kvlonge Nov 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,6 @@ pub enum TargetError {
ManagedWriteSchema(String),
/// A named target without an explicit schema.
NamedTargetSchema(String),
/// An intermediate loader, by name, whose schema resolves to the target's `preserve`.
PreservedLoaderSchema(String),
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,18 @@ pub fn target_error_message(error: &TargetError) -> String {
),
TargetError::ManagedWriteSchema(label) => format!(
"{label} has no explicitly resolved physical write schema. Set schema on the \
resource or its defaults, selected target, loader_schema, or connection."
resource or its defaults, the selected target or its loader_schema, or the \
project's only top-level [connection]."
),
TargetError::NamedTargetSchema(target) => format!(
"Named target '{target}' must explicitly set schema to a nonblank literal or \
'{PRESERVE_TARGET_VALUE}'. Resource schemas, defaults, and connection schemas do \
not select a named target schema strategy."
),
TargetError::PreservedLoaderSchema(loader) => format!(
"Intermediate loader '{loader}' would write to a schema named \
'{PRESERVE_TARGET_VALUE}'. The target's schema '{PRESERVE_TARGET_VALUE}' keeps each \
model's own schema, and a loader has none, so its destination must name a schema."
),
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,18 @@ fn given_target_error_when_formatting_then_matches_python_message() {
MessageTestCase {
description: "a managed write without a schema",
error: TargetError::ManagedWriteSchema("Model 'orders'".to_owned()),
expected_message: "Model 'orders' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, selected target, loader_schema, or connection.",
expected_message: "Model 'orders' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, the selected target or its loader_schema, or the project's only top-level [connection].",
},
MessageTestCase {
description: "a named target without a schema",
error: TargetError::NamedTargetSchema("ci".to_owned()),
expected_message: "Named target 'ci' must explicitly set schema to a nonblank literal or 'preserve'. Resource schemas, defaults, and connection schemas do not select a named target schema strategy.",
},
MessageTestCase {
description: "an intermediate loader under a preserved target schema",
error: TargetError::PreservedLoaderSchema("raw_orders".to_owned()),
expected_message: "Intermediate loader 'raw_orders' would write to a schema named 'preserve'. The target's schema 'preserve' keeps each model's own schema, and a loader has none, so its destination must name a schema.",
},
];

for test_case in test_cases {
Expand Down
19 changes: 18 additions & 1 deletion crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use sqlbuild_orchestration::runs::models::{
};
use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS;
use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease;
use sqlbuild_orchestration::scheduler::models::SchedulerLease;
use sqlbuild_orchestration::scheduler::models::{SchedulerCycle, SchedulerLease};
use sqlbuild_orchestration::store::errors::ControlError;
use sqlbuild_orchestration::store::models::ControlStore;
use sqlbuild_process_ownership::ownership::main::{
Expand All @@ -26,6 +26,7 @@ use tokio::process::Command;
use uuid::Uuid;

use crate::workers::_helpers::invocation::{arguments, environment, excluded};
use crate::workers::constants::SLOW_SCHEDULER_CYCLE;
use crate::workers::models::{
ClaimedWorker, DockerWorker, Launcher, LocalLauncher, LocalSpawn, WorkerChild, WorkerExit,
Workers,
Expand Down Expand Up @@ -149,6 +150,22 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers {
self.shutdown_workers(store, lease).await
}

fn cycled(&mut self, cycle: &SchedulerCycle) {
let timings = &cycle.timings;
if timings.total() >= SLOW_SCHEDULER_CYCLE {
drop(writeln!(
std::io::stderr().lock(),
"Slow scheduler cycle: {} ms (lease {} ms, triggers {} ms, run monitor {} ms, tick cleanup {} ms, next due {} ms)",
timings.total().as_millis(),
timings.lease.as_millis(),
timings.triggers.as_millis(),
timings.monitor.as_millis(),
timings.prune.as_millis(),
timings.next_due.as_millis(),
));
}
}

/// A claim whose commit outcome is unknown is adopted by the next recovery pass.
fn retrying(&mut self, error: &ControlError) {
self.recovered_epoch = None;
Expand Down
6 changes: 5 additions & 1 deletion crates/sqlbuild-cli/src/workers/constants.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
//! Immutable Docker image identities use the full SHA-256 hexadecimal digest.
//! Worker supervision limits.

/// Immutable Docker image identities use the full SHA-256 hexadecimal digest.
pub(super) const IMAGE_ID_HEX_LENGTH: usize = 64;

/// Healthy scheduler cycles take tens of milliseconds; slower ones are logged with phase timings.
pub(super) const SLOW_SCHEDULER_CYCLE: std::time::Duration = std::time::Duration::from_millis(500);
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Drain a bounded due-trigger page, renewing the lease between persisted evaluations.

use std::time::Duration;
use std::time::{Duration, Instant};

use chrono::{DateTime, Utc};
use uuid::Uuid;
Expand Down Expand Up @@ -42,24 +42,26 @@ pub(crate) async fn drain_due_triggers(
Ok((lease, cycle))
}

/// Shorten the cycle wait to the next running trigger's due time, never below zero.
/// Shorten the wait to the next due time, never below zero, measured from the returned instant.
pub(crate) async fn wait_for_next_due(
store: &ControlStore,
wait_for: Duration,
) -> Result<Duration, ControlError> {
) -> Result<(Duration, Instant), ControlError> {
let transaction = begin(store, TransactionMode::Read).await?;
let now = transaction.now;
let read_at = Instant::now();
let (transaction, rows) = query(transaction,
"SELECT MIN(s.next_due_at) AS next_due_at FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \
JOIN sqb_locations l ON l.location_id=t.location_id \
WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", vec![]).await?;
let next: Option<DateTime<Utc>> = serde_json::from_value(rows[0]["next_due_at"].clone())
.map_err(|error| ControlError::Invalid(error.to_string()))?;
commit(transaction).await?;
Ok(next.map_or(wait_for, |next| {
let wait_for = next.map_or(wait_for, |next| {
(next - now)
.to_std()
.unwrap_or(Duration::ZERO)
.min(wait_for)
}))
});
Ok((wait_for, read_at))
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
//! A cycle renews leadership and drains a bounded set of persisted due triggers.

use std::time::Duration;
use std::time::{Duration, Instant};

use chrono::Utc;

use crate::runs::constants::MONITOR_BATCH_SIZE;
use crate::runs::main::monitor_runs::monitor_runs;
Expand All @@ -9,14 +11,23 @@ use crate::scheduler::_helpers::tick_retention::prune_ticks_when_due;
use crate::scheduler::constants::RENEW_INTERVAL_SECONDS;
use crate::scheduler::main::_acquire_lease::acquire_lease;
use crate::scheduler::main::renew_lease::renew_lease;
use crate::scheduler::models::{SchedulerCycle, SchedulerState};
use crate::scheduler::models::{CycleTimings, SchedulerCycle, SchedulerState};
use crate::store::errors::ControlError;
use crate::store::models::ControlStore;

pub(crate) async fn scheduler_cycle(
store: &ControlStore,
mut state: SchedulerState,
) -> Result<(SchedulerState, SchedulerCycle), ControlError> {
let mut timings = CycleTimings {
started_at: Utc::now(),
lease: Duration::ZERO,
triggers: Duration::ZERO,
monitor: Duration::ZERO,
prune: Duration::ZERO,
next_due: Duration::ZERO,
};
let phase = Instant::now();
let renewal = match state.lease.as_ref() {
Some(lease) => renew_lease(store, lease).await.map(Some),
None => acquire_lease(store, state.holder).await,
Expand All @@ -26,23 +37,34 @@ pub(crate) async fn scheduler_cycle(
Err(ControlError::Fenced) => None,
Err(error) => return Err(error),
};
timings.lease = phase.elapsed();
let cycle = SchedulerCycle {
leader: state.lease.is_some(),
evaluated: 0,
ticks: 0,
unresponsive: 0,
wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS),
wait_from: Instant::now(),
timings,
};
let Some(lease) = state.lease.clone() else {
return Ok((state, cycle));
};
let phase = Instant::now();
let (lease, mut cycle) = drain_due_triggers(store, lease, cycle).await?;
cycle.timings.triggers = phase.elapsed();
let phase = Instant::now();
cycle.unresponsive = monitor_runs(store, &lease).await?;
if cycle.unresponsive == MONITOR_BATCH_SIZE {
cycle.wait_for = Duration::ZERO;
}
cycle.timings.monitor = phase.elapsed();
let phase = Instant::now();
state.pruned_at = prune_ticks_when_due(store, &lease, state.pruned_at).await?;
state.lease = Some(lease);
cycle.wait_for = wait_for_next_due(store, cycle.wait_for).await?;
cycle.timings.prune = phase.elapsed();
let phase = Instant::now();
(cycle.wait_for, cycle.wait_from) = wait_for_next_due(store, cycle.wait_for).await?;
cycle.timings.next_due = phase.elapsed();
Ok((state, cycle))
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
use std::time::Duration;

use tokio::sync::watch;
use tokio::time::Instant;

use crate::scheduler::main::_scheduler_cycle::scheduler_cycle;
use crate::scheduler::models::SchedulerState;
Expand All @@ -24,23 +25,25 @@ pub async fn run_scheduler(
Ok((next_state, cycle)) => {
state = next_state;
last_lease = state.lease.clone();
launcher.cycled(&cycle);
launcher
.poll(store, state.lease.as_ref())
.await
.map(|()| cycle.wait_for)
.map(|()| Instant::from_std(cycle.wait_from + cycle.wait_for))
}
Err(error) => Err(error),
};
let wait_for = match polled {
Ok(wait_for) => wait_for,
let retry = Instant::now() + Duration::from_secs(1);
let due = match polled {
Ok(due) => due.min(retry),
Err(error) if error.transient() => {
launcher.retrying(&error);
Duration::from_secs(1)
retry
}
Err(error) => return Err(error),
};
tokio::select! {
_ = tokio::time::sleep(wait_for.min(Duration::from_secs(1))) => {},
_ = tokio::time::sleep_until(due) => {},
result = shutdown.changed() => { if result.is_err() { break; } },
}
}
Expand Down
20 changes: 20 additions & 0 deletions crates/sqlbuild-orchestration/src/scheduler/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,26 @@ pub struct SchedulerCycle {
pub ticks: usize,
pub unresponsive: usize,
pub wait_for: std::time::Duration,
/// When `wait_for` was read, so the loop wakes on time however long the store takes afterwards.
pub wait_from: std::time::Instant,
pub timings: CycleTimings,
}

/// Wall time per cycle phase, which separates a late wake from slow store writes.
#[derive(Clone, Debug)]
pub struct CycleTimings {
pub started_at: DateTime<Utc>,
pub lease: std::time::Duration,
pub triggers: std::time::Duration,
pub monitor: std::time::Duration,
pub prune: std::time::Duration,
pub next_due: std::time::Duration,
}

impl CycleTimings {
pub fn total(&self) -> std::time::Duration {
self.lease + self.triggers + self.monitor + self.prune + self.next_due
}
}

/// Validated at the native boundary and evaluated as wall-clock times with SQLBuild's DST policy.
Expand Down
5 changes: 4 additions & 1 deletion crates/sqlbuild-orchestration/src/scheduler/types.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Compound scheduler return values.

use crate::scheduler::models::SchedulerLease;
use crate::scheduler::models::{SchedulerCycle, SchedulerLease};
use crate::store::errors::ControlError;
use crate::store::models::ControlStore;

Expand All @@ -18,6 +18,9 @@ pub trait RunLauncher {
lease: Option<&SchedulerLease>,
) -> impl Future<Output = Result<(), ControlError>> + Send;

/// Hears each completed cycle, with its phase timings, before the launcher polls.
fn cycled(&mut self, _cycle: &SchedulerCycle) {}

/// Hears each transient store failure the scheduler retries instead of stopping.
fn retrying(&mut self, _error: &ControlError) {}
}
9 changes: 9 additions & 0 deletions crates/sqlbuild-orchestration/src/store/_helpers/migration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,15 @@ use crate::store::types::Row;

pub(crate) const SCHEMA: &str = include_str!("../schema.sql");

/// Additive indexes, created on every open and kept out of the digest so old stores gain them.
pub(crate) const INDEXES: &str = include_str!("../indexes.sql");

pub(crate) fn statements(sql: &str) -> impl Iterator<Item = &str> {
sql.split(';')
.map(str::trim)
.filter(|statement| !statement.is_empty())
}

pub(crate) fn schema_digest(store: &ControlStore) -> String {
sha256_hex(format!("{}\n{SCHEMA}", recall_schema(store).join(";\n")).as_bytes())
}
Expand Down
10 changes: 10 additions & 0 deletions crates/sqlbuild-orchestration/src/store/indexes.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
CREATE INDEX IF NOT EXISTS sqb_jobs_snapshot ON sqb_jobs(snapshot_id);
CREATE INDEX IF NOT EXISTS sqb_triggers_snapshot ON sqb_triggers(snapshot_id);
CREATE INDEX IF NOT EXISTS sqb_triggers_removed ON sqb_triggers(removed_in_snapshot_id) WHERE removed_in_snapshot_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS sqb_runs_trigger ON sqb_runs(trigger_id) WHERE trigger_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS sqb_runs_rerun_of ON sqb_runs(rerun_of_run_id) WHERE rerun_of_run_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS sqb_location_work_session ON sqb_location_work(session_id) WHERE session_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS sqb_location_work_snapshot ON sqb_location_work(snapshot_id);
CREATE INDEX IF NOT EXISTS sqb_run_requests_run ON sqb_run_requests(run_id);
CREATE INDEX IF NOT EXISTS sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash);
CREATE INDEX IF NOT EXISTS sqb_role_assignments_location ON sqb_role_assignments(location_id);
15 changes: 8 additions & 7 deletions crates/sqlbuild-orchestration/src/store/main/migrate_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,17 @@

use uuid::Uuid;

use crate::store::_helpers::migration::{SCHEMA, render, schema_digest, verify_schema};
use crate::store::_helpers::migration::{
INDEXES, SCHEMA, render, schema_digest, statements, verify_schema,
};
use crate::store::_helpers::recall_schema::recall_schema;
use crate::store::errors::ControlError;
use crate::store::main::begin::begin;
use crate::store::main::commit::commit;
use crate::store::main::query::query;
use crate::store::models::{ControlStore, Parameter, TransactionMode};

/// Create the schema once, or reject a schema this server did not create.
/// Create or verify the schema, then add any missing additive indexes.
pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> {
let mut transaction = begin(store, TransactionMode::Write).await?;
(transaction, _) = query(transaction, &render(store,
Expand All @@ -28,11 +30,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> {
for statement in recall_schema(store) {
(transaction, _) = query(transaction, &statement, vec![]).await?;
}
for statement in SCHEMA
.split(';')
.map(str::trim)
.filter(|statement| !statement.is_empty())
{
for statement in statements(SCHEMA) {
(transaction, _) = query(transaction, &render(store, statement), vec![]).await?;
}
let now = transaction.now;
Expand All @@ -46,5 +44,8 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> {
)
.await?;
}
for statement in statements(INDEXES) {
(transaction, _) = query(transaction, statement, vec![]).await?;
}
commit(transaction).await
}
Loading
Loading