diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs index 9f066da682..9c1dcf2dc0 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs @@ -13,4 +13,6 @@ pub enum TargetError { ManagedWriteSchema(String), /// A named target without an explicit schema. NamedTargetSchema(String), + /// An intermediate loader, by name, whose schema resolves to the target's `preserve`. + PreservedLoaderSchema(String), } diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs index 1adc077120..ba4b45c4a6 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs @@ -35,12 +35,18 @@ pub fn target_error_message(error: &TargetError) -> String { ), TargetError::ManagedWriteSchema(label) => format!( "{label} has no explicitly resolved physical write schema. Set schema on the \ - resource or its defaults, selected target, loader_schema, or connection." + resource or its defaults, the selected target or its loader_schema, or the \ + project's only top-level [connection]." ), TargetError::NamedTargetSchema(target) => format!( "Named target '{target}' must explicitly set schema to a nonblank literal or \ '{PRESERVE_TARGET_VALUE}'. Resource schemas, defaults, and connection schemas do \ not select a named target schema strategy." ), + TargetError::PreservedLoaderSchema(loader) => format!( + "Intermediate loader '{loader}' would write to a schema named \ + '{PRESERVE_TARGET_VALUE}'. The target's schema '{PRESERVE_TARGET_VALUE}' keeps each \ + model's own schema, and a loader has none, so its destination must name a schema." + ), } } diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs index e7a26a098f..b1fd264dac 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs @@ -46,13 +46,18 @@ fn given_target_error_when_formatting_then_matches_python_message() { MessageTestCase { description: "a managed write without a schema", error: TargetError::ManagedWriteSchema("Model 'orders'".to_owned()), - expected_message: "Model 'orders' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, selected target, loader_schema, or connection.", + expected_message: "Model 'orders' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, the selected target or its loader_schema, or the project's only top-level [connection].", }, MessageTestCase { description: "a named target without a schema", error: TargetError::NamedTargetSchema("ci".to_owned()), expected_message: "Named target 'ci' must explicitly set schema to a nonblank literal or 'preserve'. Resource schemas, defaults, and connection schemas do not select a named target schema strategy.", }, + MessageTestCase { + description: "an intermediate loader under a preserved target schema", + error: TargetError::PreservedLoaderSchema("raw_orders".to_owned()), + expected_message: "Intermediate loader 'raw_orders' would write to a schema named 'preserve'. The target's schema 'preserve' keeps each model's own schema, and a loader has none, so its destination must name a schema.", + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index 80d4e295dc..8a683b2b9c 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -16,7 +16,7 @@ use sqlbuild_orchestration::runs::models::{ }; use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; -use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::scheduler::models::{SchedulerCycle, SchedulerLease}; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::models::ControlStore; use sqlbuild_process_ownership::ownership::main::{ @@ -26,6 +26,7 @@ use tokio::process::Command; use uuid::Uuid; use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::constants::SLOW_SCHEDULER_CYCLE; use crate::workers::models::{ ClaimedWorker, DockerWorker, Launcher, LocalLauncher, LocalSpawn, WorkerChild, WorkerExit, Workers, @@ -149,6 +150,22 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { self.shutdown_workers(store, lease).await } + fn cycled(&mut self, cycle: &SchedulerCycle) { + let timings = &cycle.timings; + if timings.total() >= SLOW_SCHEDULER_CYCLE { + drop(writeln!( + std::io::stderr().lock(), + "Slow scheduler cycle: {} ms (lease {} ms, triggers {} ms, run monitor {} ms, tick cleanup {} ms, next due {} ms)", + timings.total().as_millis(), + timings.lease.as_millis(), + timings.triggers.as_millis(), + timings.monitor.as_millis(), + timings.prune.as_millis(), + timings.next_due.as_millis(), + )); + } + } + /// A claim whose commit outcome is unknown is adopted by the next recovery pass. fn retrying(&mut self, error: &ControlError) { self.recovered_epoch = None; diff --git a/crates/sqlbuild-cli/src/workers/constants.rs b/crates/sqlbuild-cli/src/workers/constants.rs index 881c48bea1..6db1d72d80 100644 --- a/crates/sqlbuild-cli/src/workers/constants.rs +++ b/crates/sqlbuild-cli/src/workers/constants.rs @@ -1,3 +1,7 @@ -//! Immutable Docker image identities use the full SHA-256 hexadecimal digest. +//! Worker supervision limits. +/// Immutable Docker image identities use the full SHA-256 hexadecimal digest. pub(super) const IMAGE_ID_HEX_LENGTH: usize = 64; + +/// Healthy scheduler cycles take tens of milliseconds; slower ones are logged with phase timings. +pub(super) const SLOW_SCHEDULER_CYCLE: std::time::Duration = std::time::Duration::from_millis(500); diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs index 7161888c49..30ed2819c1 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -1,6 +1,6 @@ //! Drain a bounded due-trigger page, renewing the lease between persisted evaluations. -use std::time::Duration; +use std::time::{Duration, Instant}; use chrono::{DateTime, Utc}; use uuid::Uuid; @@ -42,13 +42,14 @@ pub(crate) async fn drain_due_triggers( Ok((lease, cycle)) } -/// Shorten the cycle wait to the next running trigger's due time, never below zero. +/// Shorten the wait to the next due time, never below zero, measured from the returned instant. pub(crate) async fn wait_for_next_due( store: &ControlStore, wait_for: Duration, -) -> Result { +) -> Result<(Duration, Instant), ControlError> { let transaction = begin(store, TransactionMode::Read).await?; let now = transaction.now; + let read_at = Instant::now(); let (transaction, rows) = query(transaction, "SELECT MIN(s.next_due_at) AS next_due_at FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ JOIN sqb_locations l ON l.location_id=t.location_id \ @@ -56,10 +57,11 @@ pub(crate) async fn wait_for_next_due( let next: Option> = serde_json::from_value(rows[0]["next_due_at"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; commit(transaction).await?; - Ok(next.map_or(wait_for, |next| { + let wait_for = next.map_or(wait_for, |next| { (next - now) .to_std() .unwrap_or(Duration::ZERO) .min(wait_for) - })) + }); + Ok((wait_for, read_at)) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs index c6267cfc84..f27642c52d 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs @@ -1,6 +1,8 @@ //! A cycle renews leadership and drains a bounded set of persisted due triggers. -use std::time::Duration; +use std::time::{Duration, Instant}; + +use chrono::Utc; use crate::runs::constants::MONITOR_BATCH_SIZE; use crate::runs::main::monitor_runs::monitor_runs; @@ -9,7 +11,7 @@ use crate::scheduler::_helpers::tick_retention::prune_ticks_when_due; use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; use crate::scheduler::main::_acquire_lease::acquire_lease; use crate::scheduler::main::renew_lease::renew_lease; -use crate::scheduler::models::{SchedulerCycle, SchedulerState}; +use crate::scheduler::models::{CycleTimings, SchedulerCycle, SchedulerState}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -17,6 +19,15 @@ pub(crate) async fn scheduler_cycle( store: &ControlStore, mut state: SchedulerState, ) -> Result<(SchedulerState, SchedulerCycle), ControlError> { + let mut timings = CycleTimings { + started_at: Utc::now(), + lease: Duration::ZERO, + triggers: Duration::ZERO, + monitor: Duration::ZERO, + prune: Duration::ZERO, + next_due: Duration::ZERO, + }; + let phase = Instant::now(); let renewal = match state.lease.as_ref() { Some(lease) => renew_lease(store, lease).await.map(Some), None => acquire_lease(store, state.holder).await, @@ -26,23 +37,34 @@ pub(crate) async fn scheduler_cycle( Err(ControlError::Fenced) => None, Err(error) => return Err(error), }; + timings.lease = phase.elapsed(); let cycle = SchedulerCycle { leader: state.lease.is_some(), evaluated: 0, ticks: 0, unresponsive: 0, wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), + wait_from: Instant::now(), + timings, }; let Some(lease) = state.lease.clone() else { return Ok((state, cycle)); }; + let phase = Instant::now(); let (lease, mut cycle) = drain_due_triggers(store, lease, cycle).await?; + cycle.timings.triggers = phase.elapsed(); + let phase = Instant::now(); cycle.unresponsive = monitor_runs(store, &lease).await?; if cycle.unresponsive == MONITOR_BATCH_SIZE { cycle.wait_for = Duration::ZERO; } + cycle.timings.monitor = phase.elapsed(); + let phase = Instant::now(); state.pruned_at = prune_ticks_when_due(store, &lease, state.pruned_at).await?; state.lease = Some(lease); - cycle.wait_for = wait_for_next_due(store, cycle.wait_for).await?; + cycle.timings.prune = phase.elapsed(); + let phase = Instant::now(); + (cycle.wait_for, cycle.wait_from) = wait_for_next_due(store, cycle.wait_for).await?; + cycle.timings.next_due = phase.elapsed(); Ok((state, cycle)) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index 2917ec1447..42b2fb2ea4 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -3,6 +3,7 @@ use std::time::Duration; use tokio::sync::watch; +use tokio::time::Instant; use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; use crate::scheduler::models::SchedulerState; @@ -24,23 +25,25 @@ pub async fn run_scheduler( Ok((next_state, cycle)) => { state = next_state; last_lease = state.lease.clone(); + launcher.cycled(&cycle); launcher .poll(store, state.lease.as_ref()) .await - .map(|()| cycle.wait_for) + .map(|()| Instant::from_std(cycle.wait_from + cycle.wait_for)) } Err(error) => Err(error), }; - let wait_for = match polled { - Ok(wait_for) => wait_for, + let retry = Instant::now() + Duration::from_secs(1); + let due = match polled { + Ok(due) => due.min(retry), Err(error) if error.transient() => { launcher.retrying(&error); - Duration::from_secs(1) + retry } Err(error) => return Err(error), }; tokio::select! { - _ = tokio::time::sleep(wait_for.min(Duration::from_secs(1))) => {}, + _ = tokio::time::sleep_until(due) => {}, result = shutdown.changed() => { if result.is_err() { break; } }, } } diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index c448b4138e..c861f59ed6 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -40,6 +40,26 @@ pub struct SchedulerCycle { pub ticks: usize, pub unresponsive: usize, pub wait_for: std::time::Duration, + /// When `wait_for` was read, so the loop wakes on time however long the store takes afterwards. + pub wait_from: std::time::Instant, + pub timings: CycleTimings, +} + +/// Wall time per cycle phase, which separates a late wake from slow store writes. +#[derive(Clone, Debug)] +pub struct CycleTimings { + pub started_at: DateTime, + pub lease: std::time::Duration, + pub triggers: std::time::Duration, + pub monitor: std::time::Duration, + pub prune: std::time::Duration, + pub next_due: std::time::Duration, +} + +impl CycleTimings { + pub fn total(&self) -> std::time::Duration { + self.lease + self.triggers + self.monitor + self.prune + self.next_due + } } /// Validated at the native boundary and evaluated as wall-clock times with SQLBuild's DST policy. diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs index 18c32c7ac9..1515b87c9d 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -1,6 +1,6 @@ //! Compound scheduler return values. -use crate::scheduler::models::SchedulerLease; +use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -18,6 +18,9 @@ pub trait RunLauncher { lease: Option<&SchedulerLease>, ) -> impl Future> + Send; + /// Hears each completed cycle, with its phase timings, before the launcher polls. + fn cycled(&mut self, _cycle: &SchedulerCycle) {} + /// Hears each transient store failure the scheduler retries instead of stopping. fn retrying(&mut self, _error: &ControlError) {} } diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 980caf2878..8a2ed7baa2 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -9,6 +9,15 @@ use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); +/// Additive indexes, created on every open and kept out of the digest so old stores gain them. +pub(crate) const INDEXES: &str = include_str!("../indexes.sql"); + +pub(crate) fn statements(sql: &str) -> impl Iterator { + sql.split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) +} + pub(crate) fn schema_digest(store: &ControlStore) -> String { sha256_hex(format!("{}\n{SCHEMA}", recall_schema(store).join(";\n")).as_bytes()) } diff --git a/crates/sqlbuild-orchestration/src/store/indexes.sql b/crates/sqlbuild-orchestration/src/store/indexes.sql new file mode 100644 index 0000000000..4f141c93ee --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/indexes.sql @@ -0,0 +1,10 @@ +CREATE INDEX IF NOT EXISTS sqb_jobs_snapshot ON sqb_jobs(snapshot_id); +CREATE INDEX IF NOT EXISTS sqb_triggers_snapshot ON sqb_triggers(snapshot_id); +CREATE INDEX IF NOT EXISTS sqb_triggers_removed ON sqb_triggers(removed_in_snapshot_id) WHERE removed_in_snapshot_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sqb_runs_trigger ON sqb_runs(trigger_id) WHERE trigger_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sqb_runs_rerun_of ON sqb_runs(rerun_of_run_id) WHERE rerun_of_run_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sqb_location_work_session ON sqb_location_work(session_id) WHERE session_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sqb_location_work_snapshot ON sqb_location_work(snapshot_id); +CREATE INDEX IF NOT EXISTS sqb_run_requests_run ON sqb_run_requests(run_id); +CREATE INDEX IF NOT EXISTS sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash); +CREATE INDEX IF NOT EXISTS sqb_role_assignments_location ON sqb_role_assignments(location_id); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index ed230c65a9..e82180367e 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -2,7 +2,9 @@ use uuid::Uuid; -use crate::store::_helpers::migration::{SCHEMA, render, schema_digest, verify_schema}; +use crate::store::_helpers::migration::{ + INDEXES, SCHEMA, render, schema_digest, statements, verify_schema, +}; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -10,7 +12,7 @@ use crate::store::main::commit::commit; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; -/// Create the schema once, or reject a schema this server did not create. +/// Create or verify the schema, then add any missing additive indexes. pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { let mut transaction = begin(store, TransactionMode::Write).await?; (transaction, _) = query(transaction, &render(store, @@ -28,11 +30,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { for statement in recall_schema(store) { (transaction, _) = query(transaction, &statement, vec![]).await?; } - for statement in SCHEMA - .split(';') - .map(str::trim) - .filter(|statement| !statement.is_empty()) - { + for statement in statements(SCHEMA) { (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; } let now = transaction.now; @@ -46,5 +44,8 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ) .await?; } + for statement in statements(INDEXES) { + (transaction, _) = query(transaction, statement, vec![]).await?; + } commit(transaction).await } diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 0aba4ffc27..2d684232f3 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -25,7 +25,7 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::ticks::test_types::{ - FaultAnomalies, FaultStep, IdleLauncher, TransactionFault, + CycleRecorder, FaultAnomalies, FaultStep, RecordedCycle, TransactionFault, }; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::helpers::read_recall_events; @@ -729,12 +729,13 @@ async fn triggers_behind(store: &ControlStore, latest: DateTime) -> i64 { rows[0]["behind"].as_i64().expect("count") } -/// Run one real scheduler loop until `deadline`, then stop it and return the trigger's ticks. +/// Run one real scheduler loop until `deadline`, then return the trigger's ticks and cycle timings. pub(crate) async fn run_scheduler_until( store: &ControlStore, trigger_id: Uuid, - deadline: DateTime, -) -> Vec { + (deadline, poll_delay): (DateTime, std::time::Duration), +) -> (Vec, Vec) { + let mut cycles = Vec::new(); let (stop, shutdown) = tokio::sync::watch::channel(false); let stopping = async { tokio::time::sleep((deadline - Utc::now()).to_std().unwrap_or_default()).await; @@ -745,12 +746,33 @@ pub(crate) async fn run_scheduler_until( store, SchedulerState::new(Uuid::now_v7()), shutdown, - IdleLauncher + CycleRecorder { + cycles: &mut cycles, + poll_delay, + launching: false, + } ), stopping ); finished.expect("scheduler loop"); - list_ticks(store, trigger_id, 100).await.expect("ticks") + let ticks = list_ticks(store, trigger_id, 100).await.expect("ticks"); + (ticks, cycles) +} + +/// How late the cycle that made a decision woke: after `ready` or the previous poll, whichever is later. +pub(crate) fn decision_wake( + cycles: &[RecordedCycle], + decided_at: DateTime, + ready: DateTime, +) -> (TimeDelta, &RecordedCycle) { + let index = cycles + .iter() + .rposition(|cycle| cycle.timings.started_at <= decided_at) + .expect("a cycle started before the decision"); + let free = index + .checked_sub(1) + .map_or(ready, |previous| ready.max(cycles[previous].polled_at)); + (cycles[index].timings.started_at - free, &cycles[index]) } pub(crate) async fn queued_times( diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs index 71291a8feb..6496af2506 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -1,4 +1,6 @@ -//! Bounds tick-to-decision latency, where a decision queues or covers a run, and standby catch-up. +//! Bounds how late the scheduler wakes for due times and standby catch-up behind a slow launcher. + +use std::time::Duration; use chrono::{DurationRound, SecondsFormat, TimeDelta, Utc}; use serde_json::json; @@ -6,7 +8,7 @@ use serde_json::json; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::scheduler::tests::contracts::ticks::helpers::{ - queued_times, run_scheduler_until, snapshot, + decision_wake, queued_times, run_scheduler_until, snapshot, }; use crate::scheduler::tests::contracts::ticks::test_types::SchedulerLatencyTestCase; use crate::snapshots::main::publish_snapshot::publish_snapshot; @@ -25,16 +27,18 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks backend: Backend::Sqlite, expected_ticks: 3, expected_ticks_after_failover: 4, - expected_max_tick_to_decision: TimeDelta::milliseconds(500), - expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + expected_max_wake_after_due: TimeDelta::milliseconds(500), + expected_max_failover_wake: TimeDelta::milliseconds(1_500), + launcher_poll: Duration::from_millis(700), }, SchedulerLatencyTestCase { description: "PostgreSQL scheduler latency", backend: Backend::Postgres, expected_ticks: 3, expected_ticks_after_failover: 4, - expected_max_tick_to_decision: TimeDelta::milliseconds(500), - expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + expected_max_wake_after_due: TimeDelta::milliseconds(500), + expected_max_failover_wake: TimeDelta::milliseconds(1_500), + launcher_poll: Duration::from_millis(700), }, ]; for test_case in test_cases { @@ -68,7 +72,8 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks .await .expect("trigger")[0] .trigger_id; - let ticks = run_scheduler_until(store, trigger_id, at(6)).await; + let (ticks, cycles) = + run_scheduler_until(store, trigger_id, (at(6), test_case.launcher_poll)).await; let queued = queued_times(store, location).await; let earliest = ticks .iter() @@ -88,24 +93,26 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks "{}: later times are covered by the waiting run", test_case.description ); - let latencies: Vec = ticks - .iter() - .map(|tick| tick.evaluated_at - tick.scheduled_for) - .collect(); - println!("{} tick-to-decision: {latencies:?}", test_case.description); assert_eq!( - latencies.len(), + ticks.len(), test_case.expected_ticks, "{}", test_case.description ); - assert!( - latencies - .iter() - .all(|latency| *latency <= test_case.expected_max_tick_to_decision), - "{}: {latencies:?}", - test_case.description - ); + for tick in &ticks { + let (wake, cycle) = decision_wake(&cycles, tick.evaluated_at, tick.scheduled_for); + println!( + "{}: due {} woke {wake:?}, store {:?} to decision, cycle {cycle:?}", + test_case.description, + tick.scheduled_for, + tick.evaluated_at - cycle.timings.started_at, + ); + assert!( + wake <= test_case.expected_max_wake_after_due, + "{}: woke {wake:?} after the due time", + test_case.description + ); + } // The first leader stopped without releasing its lease while two more times fell due; // the standby catches up on the latest one only. // A standby is already polling when that lease lapses, so its own retry wait counts. @@ -116,8 +123,10 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks expire_lease(store).await; Utc::now() }; - let (ticks, available) = - tokio::join!(run_scheduler_until(&second, trigger_id, at(14)), lapse); + let ((ticks, cycles), available) = tokio::join!( + run_scheduler_until(&second, trigger_id, (at(14), Duration::ZERO)), + lapse + ); assert_eq!( ticks.len(), test_case.expected_ticks_after_failover, @@ -128,11 +137,15 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks .iter() .find(|tick| tick.scheduled_for == at(9)) .expect("the latest overdue time is decided"); - let catch_up = latest.evaluated_at - available; - println!("{} failover catch-up: {catch_up:?}", test_case.description); + let (wake, cycle) = decision_wake(&cycles, latest.evaluated_at, available); + println!( + "{} failover: woke {wake:?} after the lapse, store {:?} to decision, cycle {cycle:?}", + test_case.description, + latest.evaluated_at - cycle.timings.started_at, + ); assert!( - catch_up <= test_case.expected_max_failover_catch_up, - "{}: {catch_up:?}", + wake <= test_case.expected_max_failover_wake, + "{}: woke {wake:?} after the lapse", test_case.description ); close_store(second).await; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index f6cb23edc0..146b81d23b 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -36,6 +36,52 @@ impl crate::scheduler::types::RunLauncher for IdleLauncher { } } +/// A cycle's phase timings and when the launcher poll after it finished, freeing the loop to sleep. +#[derive(Clone, Debug)] +pub(crate) struct RecordedCycle { + pub(crate) timings: crate::scheduler::models::CycleTimings, + pub(crate) polled_at: chrono::DateTime, +} + +/// Records cycle timings; polls take `poll_delay` after a cycle that decided a trigger, as launches do. +pub(crate) struct CycleRecorder<'a> { + pub(crate) cycles: &'a mut Vec, + pub(crate) poll_delay: std::time::Duration, + pub(crate) launching: bool, +} + +impl crate::scheduler::types::RunLauncher for CycleRecorder<'_> { + async fn poll( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + if self.launching { + tokio::time::sleep(self.poll_delay).await; + } + if let Some(cycle) = self.cycles.last_mut() { + cycle.polled_at = chrono::Utc::now(); + } + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + Ok(()) + } + + fn cycled(&mut self, cycle: &crate::scheduler::models::SchedulerCycle) { + self.cycles.push(RecordedCycle { + timings: cycle.timings.clone(), + polled_at: cycle.timings.started_at, + }); + self.launching = cycle.evaluated > 0; + } +} + pub(crate) struct TickRetentionTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, @@ -88,6 +134,8 @@ pub(crate) struct SchedulerLatencyTestCase { pub(crate) backend: Backend, pub(crate) expected_ticks: usize, pub(crate) expected_ticks_after_failover: usize, - pub(crate) expected_max_tick_to_decision: chrono::TimeDelta, - pub(crate) expected_max_failover_catch_up: chrono::TimeDelta, + pub(crate) expected_max_wake_after_due: chrono::TimeDelta, + pub(crate) expected_max_failover_wake: chrono::TimeDelta, + /// Work after the cycle's next-due read, which must not delay the wake. + pub(crate) launcher_poll: std::time::Duration, } diff --git a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs new file mode 100644 index 0000000000..bc192f33ba --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs @@ -0,0 +1,7 @@ +//! Row readers shared by the control-store schema contracts. + +use crate::store::types::Row; + +pub(crate) fn text(row: &Row, key: &str) -> String { + row[key].as_str().expect("text column").to_owned() +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs new file mode 100644 index 0000000000..dd9741305f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs @@ -0,0 +1,10 @@ +//! Control-store schema contracts for SQLite and PostgreSQL. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_foreign_key_indexes.rs"] +mod test_foreign_key_indexes; +#[path = "test_store_upgrade.rs"] +mod test_store_upgrade; +#[path = "test_types.rs"] +pub(crate) mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs new file mode 100644 index 0000000000..96c5c50768 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs @@ -0,0 +1,86 @@ +//! Every foreign key is indexed on its referencing columns, so deleting a parent stays bounded. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::TransactionMode; +use crate::store::tests::schema::helpers::text; +use crate::store::tests::schema::test_types::ForeignKeyIndexTestCase; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +const FOREIGN_KEYS: &str = "SELECT m.name AS table_name, f.id AS key_id, f.\"from\" AS column_name \ + FROM sqlite_master m JOIN pragma_foreign_key_list(m.name) f WHERE m.type = 'table'"; +const INDEX_COLUMNS: &str = "SELECT m.name AS table_name, l.name AS index_name, \ + l.\"unique\" AS is_unique, i.seqno AS position, COALESCE(i.name, '') AS column_name \ + FROM sqlite_master m JOIN pragma_index_list(m.name) l JOIN pragma_index_info(l.name) i \ + WHERE m.type = 'table' ORDER BY m.name, l.name, i.seqno"; + +#[test] +fn given_control_store_schema_when_listing_foreign_keys_then_each_is_indexed() { + let test_cases = [ForeignKeyIndexTestCase { + description: "SQLite schema", + backend: Backend::Sqlite, + expected_uncovered: Vec::new(), + }]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + migrate_store(&fixture.store).await.expect("migrate store"); + let transaction = begin(&fixture.store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, keys) = query(transaction, FOREIGN_KEYS, vec![]) + .await + .expect("list foreign keys"); + let (transaction, columns) = query(transaction, INDEX_COLUMNS, vec![]) + .await + .expect("list index columns"); + commit(transaction).await.expect("finish read"); + + let mut foreign_keys: BTreeMap<(String, i64), BTreeSet> = BTreeMap::new(); + for row in &keys { + foreign_keys + .entry(( + text(row, "table_name"), + row["key_id"].as_i64().expect("key id"), + )) + .or_default() + .insert(text(row, "column_name")); + } + let mut indexes: BTreeMap<(String, String), (bool, Vec)> = BTreeMap::new(); + for row in &columns { + let entry = indexes + .entry((text(row, "table_name"), text(row, "index_name"))) + .or_insert((row["is_unique"].as_i64() == Some(1), Vec::new())); + entry.1.push(text(row, "column_name")); + } + let uncovered: Vec<(String, BTreeSet)> = foreign_keys + .into_iter() + .filter(|((table, _), referencing)| { + !indexes.iter().any(|((index_table, _), (unique, indexed))| { + index_table == table + && (indexed.len() >= referencing.len() + && indexed[..referencing.len()] + .iter() + .all(|column| referencing.contains(column)) + || *unique + && indexed.iter().all(|column| referencing.contains(column))) + }) + }) + .map(|((table, _), referencing)| (table, referencing)) + .collect(); + assert_eq!( + uncovered, test_case.expected_uncovered, + "{}: foreign keys without an index on their referencing columns", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_store_upgrade.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_store_upgrade.rs new file mode 100644 index 0000000000..cffb7e9bac --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_store_upgrade.rs @@ -0,0 +1,108 @@ +//! Opening a store created before the foreign-key indexes adds them in place and keeps history. + +use std::collections::BTreeSet; + +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::TransactionMode; +use crate::store::tests::schema::helpers::text; +use crate::store::tests::schema::test_types::StoreUpgradeTestCase; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +const FOREIGN_KEY_INDEXES: [&str; 10] = [ + "sqb_jobs_snapshot", + "sqb_location_work_session", + "sqb_location_work_snapshot", + "sqb_role_assignments_location", + "sqb_run_requests_run", + "sqb_runs_rerun_of", + "sqb_runs_trigger", + "sqb_triggers_removed", + "sqb_triggers_snapshot", + "sqb_warehouse_fingerprints_text", +]; + +#[test] +fn given_store_from_previous_schema_when_opening_then_indexes_are_added_and_history_is_kept() { + let test_cases = [ + StoreUpgradeTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + list_indexes: "SELECT name FROM sqlite_master WHERE type = 'index'", + expected_indexes: FOREIGN_KEY_INDEXES.to_vec(), + }, + StoreUpgradeTestCase { + description: "PostgreSQL store", + backend: Backend::Postgres, + list_indexes: "SELECT indexname AS name FROM pg_indexes \ + WHERE schemaname = current_schema()", + expected_indexes: FOREIGN_KEY_INDEXES.to_vec(), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("create store"); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("begin downgrade"); + for index in FOREIGN_KEY_INDEXES { + (transaction, _) = query(transaction, &format!("DROP INDEX {index}"), vec![]) + .await + .expect("drop index to recreate the previous schema"); + } + let (transaction, before) = query( + transaction, + "SELECT installation_id FROM sqb_installation", + vec![], + ) + .await + .expect("read installation before upgrade"); + commit(transaction).await.expect("commit downgrade"); + + migrate_store(store) + .await + .expect("open store from the previous schema"); + + let transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, after) = query( + transaction, + "SELECT installation_id FROM sqb_installation", + vec![], + ) + .await + .expect("read installation after upgrade"); + let (transaction, indexes) = query(transaction, test_case.list_indexes, vec![]) + .await + .expect("list indexes"); + commit(transaction).await.expect("finish read"); + let present: BTreeSet = indexes.iter().map(|row| text(row, "name")).collect(); + let added: Vec<&str> = test_case + .expected_indexes + .iter() + .copied() + .filter(|index| present.contains(*index)) + .collect(); + assert_eq!( + added, test_case.expected_indexes, + "{}: indexes after upgrade", + test_case.description + ); + assert_eq!( + (before.len(), &before), + (1, &after), + "{}: installation history changed", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs new file mode 100644 index 0000000000..748d56da48 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs @@ -0,0 +1,18 @@ +//! Expected control-store schema outcomes for both supported databases. + +use std::collections::BTreeSet; + +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct ForeignKeyIndexTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_uncovered: Vec<(String, BTreeSet)>, +} + +pub(crate) struct StoreUpgradeTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) list_indexes: &'static str, + pub(crate) expected_indexes: Vec<&'static str>, +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs index 868430a934..b2b30bae97 100644 --- a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -12,6 +12,8 @@ pub(crate) mod lease_helpers; pub(crate) mod logical_state; #[path = "replicas/helpers.rs"] pub(crate) mod replica_helpers; +#[path = "schema/test_contracts.rs"] +mod schema; #[path = "helpers.rs"] pub(crate) mod store_helpers; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 34a83cb59a..a0a27b13ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -23,7 +23,7 @@ const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; /// The selected target's name and merged config, or `(None, None)`. -fn resolved_target<'py>( +pub(crate) fn resolved_target<'py>( discovered: &Bound<'py, PyAny>, selected_target: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 8c6a9ca065..d8924b4abb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -1,5 +1,7 @@ //! Compiled project and project graph construction: frontier stages, target defaults and checks. +use std::collections::HashSet; + use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -14,8 +16,9 @@ use crate::bindings::_helpers::compile_driver::target_defaults::{ TargetDefaults, apply_target_defaults, }; use crate::bindings::_helpers::compile_driver::targets::{ - managed_loader_target_isolation, managed_write_schemas, named_target_schema_strategy, - project_targets, requires_explicit_targets, + managed_loader_target_isolation, managed_source_loaders, managed_write_schemas, + named_target_schema_strategy, preserved_loader_schemas, project_targets, + requires_explicit_targets, }; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; @@ -156,7 +159,11 @@ fn python_compiled_project<'py>( ) -> PyResult> { let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); - named_target_schema_strategy(discovered, adapter, request.selected_target.bind(py))?; + named_target_schema_strategy( + discovered, + adapter, + (request.selected_target.bind(py), request.cli_vars.bind(py)), + )?; managed_loader_target_isolation(discovered, adapter, request.resolved_connection.bind(py))?; let selection: &Bound<'py, PyAny> = request.analysis_selection.bind(py); let no_cache: bool = @@ -209,6 +216,19 @@ fn targeted_project<'py>( ) -> PyResult> { let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); + let source_loaders: HashSet = managed_source_loaders(project)?; + let mut intermediate_loaders: HashSet = HashSet::new(); + for loader in native_field(discovered, "loader_functions")?.try_iter()? { + let name: String = native_field(&loader?, "name")?.extract()?; + if !source_loaders.contains(&name) { + intermediate_loaders.insert(name); + } + } + preserved_loader_schemas( + discovered, + request.selected_target.bind(py), + &intermediate_loaders, + )?; managed_write_schemas(adapter, project)?; let effective_schema: Bound<'py, PyAny> = native_field(project, "effective_target_schema")?; let default_schema: Bound<'py, PyAny> = if effective_schema.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index bba27d6475..652b64cceb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -10,7 +10,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::target_checks::constants::{ LOADER_COLLISION_CODE, MANAGED_WRITE_SCHEMA_CODE, MISSING_TARGET_PARTS_CODE, - NAMED_TARGET_SCHEMA_CODE, + NAMED_TARGET_SCHEMA_CODE, PRESERVE_TARGET_VALUE, }; use sqlbuild_analysis::assembly::target_checks::errors::TargetError; use sqlbuild_analysis::assembly::target_checks::main::explicit_write_schema::explicit_write_schema; @@ -42,8 +42,8 @@ const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; +use crate::bindings::_helpers::compile_attachment::target::resolved_target; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::discovery::guidance::setting_help; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -148,21 +148,21 @@ fn project_setting_help( /// Require shared-warehouse named targets to choose a schema strategy. #[pyfunction] -#[pyo3(signature = (**options))] +#[pyo3(signature = (*, discovered_inputs, adapter, selected_target, cli_vars = None))] fn validate_named_target_schema_strategy<'py>( - py: Python<'py>, - options: Option>, + discovered_inputs: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, + cli_vars: Option>, ) -> Result<(), PyErr> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "validate_named_target_schema_strategy", - options, - (&["discovered_inputs", "adapter", "selected_target"], &[]), - )?; + let py: Python<'py> = discovered_inputs.py(); named_target_schema_strategy( - &arguments.value("discovered_inputs")?, - &arguments.value("adapter")?, - &arguments.value("selected_target")?, + discovered_inputs, + adapter, + ( + selected_target, + &cli_vars.unwrap_or_else(|| py.None().into_bound(py)), + ), ) } @@ -170,35 +170,28 @@ fn validate_named_target_schema_strategy<'py>( pub(crate) fn named_target_schema_strategy( discovered: &Bound<'_, PyAny>, adapter: &Bound<'_, PyAny>, - selected_target: &Bound<'_, PyAny>, + (selected_target, cli_vars): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), ) -> PyResult<()> { let py: Python<'_> = discovered.py(); if native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? { return Ok(()); } - let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; - let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; - let target_name: Bound<'_, PyAny> = resolve_target_name( - py, - &project_config.clone(), - &local_config.clone(), - &selected_target.clone(), - ) - .and_then(|value| value.into_bound_py_any(py))?; + let (target_name, target_config) = resolved_target(discovered, selected_target)?; if target_name.is_none() { return Ok(()); } - let target_config: Bound<'_, PyAny> = - resolve_target_config(py, &project_config, &local_config, &target_name.clone()) - .and_then(|value| value.into_bound_py_any(py))?; let schema: Option = text_or_none(&native_field(&target_config, "schema")?)?; if schema.is_some_and(|schema| !python_strip(&schema).is_empty()) { return Ok(()); } let name: String = target_name.str()?.to_cow()?.into_owned(); let mut message: String = target_error_message(&TargetError::NamedTargetSchema(name.clone())); - let connection: Bound<'_, PyAny> = - build_effective_connection_config(discovered, Some(target_name), None, false)?; + let connection: Bound<'_, PyAny> = build_effective_connection_config( + discovered, + Some(target_name), + Some(cli_vars.clone()).filter(|value| !value.is_none()), + false, + )?; let help: Option = match text_or_none(&mapping_get(&connection, "schema")?)? { Some(value) => { message.push_str(&ignored_connection_note("schema", &value)); @@ -403,18 +396,7 @@ fn managed_source_namespace( let value: Bound<'_, PyDict> = PyDict::new(py); pyo3::types::PyDictMethods::set_item(&value, "database", database)?; pyo3::types::PyDictMethods::set_item(&value, "schema", schema)?; - let expanded: Bound<'_, PyAny> = expanded_template_data( - value.as_any(), - (effective_vars.downcast()?, &PyDict::new(py)), - ( - TemplateOptions { - allow_context: false, - preserve_context_tokens: false, - preserve_unknown_context: false, - }, - LOADER_NAMESPACE_LABEL, - ), - )?; + let expanded: Bound<'_, PyAny> = expanded_loader_namespace(value.as_any(), effective_vars)?; let part = |key: &str| -> PyResult> { let value: Bound<'_, PyAny> = expanded.get_item(key)?; if value.is_none() { @@ -425,6 +407,25 @@ fn managed_source_namespace( Ok((part("database")?, part("schema")?)) } +/// A managed source's `database` and `schema` with project vars expanded and no context templates. +pub(crate) fn expanded_loader_namespace<'py>( + value: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + expanded_template_data( + value, + (effective_vars.downcast()?, &PyDict::new(value.py())), + ( + TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }, + LOADER_NAMESPACE_LABEL, + ), + ) +} + /// Validate compiled model and seed locations for the effective adapter. #[pyfunction] #[pyo3(signature = (**options))] @@ -598,15 +599,7 @@ pub(crate) fn managed_write_schemas( if ignored.is_none() { return Ok(()); } - let mut source_loaders: HashSet = HashSet::new(); - for source in native_field(project, "sources")?.try_iter()? { - if let Some(loader) = text_or_none(&native_field( - &native_field(&source?, "source_entry")?, - "loader", - )?)? { - source_loaders.insert(loader); - } - } + let source_loaders: HashSet = managed_source_loaders(project)?; for loader in native_field(project, "loader_functions")?.try_iter()? { let loader: Bound<'_, PyAny> = loader?; let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; @@ -627,6 +620,57 @@ pub(crate) fn managed_write_schemas( Ok(()) } +/// The loader names that load managed sources, which write to the source's own location. +pub(crate) fn managed_source_loaders(project: &Bound<'_, PyAny>) -> PyResult> { + let mut loaders: HashSet = HashSet::new(); + for source in native_field(project, "sources")?.try_iter()? { + if let Some(loader) = text_or_none(&native_field( + &native_field(&source?, "source_entry")?, + "loader", + )?)? { + loaders.insert(loader); + } + } + Ok(loaders) +} + +/// Raise when the selected target's schema is `preserve` and one of `loaders` names no schema. +pub(crate) fn preserved_loader_schemas( + discovered: &Bound<'_, PyAny>, + selected_target: &Bound<'_, PyAny>, + loaders: &HashSet, +) -> PyResult<()> { + let py: Python<'_> = discovered.py(); + let (target_name, target_config) = resolved_target(discovered, selected_target)?; + if target_name.is_none() { + return Ok(()); + } + if !text_or_none(&native_field(&target_config, "schema")?)? + .is_some_and(|schema| python_strip(&schema) == PRESERVE_TARGET_VALUE) + { + return Ok(()); + } + for loader in native_field(discovered, "loader_functions")?.try_iter()? { + let loader: Bound<'_, PyAny> = loader?; + let name: String = native_field(&loader, "name")?.extract()?; + let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; + if !loaders.contains(&name) || destination.as_deref().is_some_and(|d| d.contains('.')) { + continue; + } + let table: String = destination.unwrap_or_else(|| format!("__loader__{name}")); + return Err(planner_error( + py, + target_error_message(&TargetError::PreservedLoaderSchema(name)), + MANAGED_WRITE_SCHEMA_CODE, + Some(format!( + "to choose where it writes, set its destination in its @loader decorator:\n \ + @loader(destination=\"my_schema.{table}\")" + )), + )); + } + Ok(()) +} + /// Raise when neither the resource nor the project names a physical write schema. pub(crate) fn managed_write_schema( resource_label: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 456358a009..1ebb3f2c60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -41,6 +41,7 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::record_class; @@ -298,6 +299,8 @@ fn build_report( })?; let connection_config: Bound<'_, PyDict> = effective_connection(&context, (&adapter_name, &base_connection))?.cast_into()?; + let namespace_adapter: String = adapter_name.str()?.extract()?; + let namespaceless: bool = NAMESPACELESS_ADAPTERS.contains(&namespace_adapter.as_str()); let namespace: Bound<'_, PyTuple> = build_effective_target_namespace( py, Some(keywords( @@ -309,6 +312,7 @@ fn build_report( "connection_database", connection_config .get_item("database")? + .filter(|_| !namespaceless) .unwrap_or_else(|| py.None().into_bound(py)), ), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index f9a27a370b..5c3c84438d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -35,7 +35,7 @@ use sqlbuild_analysis::diff_selection::main::raw_query_shape::raw_query_shape; use sqlbuild_analysis::diff_selection::models::RawQueryShape; const GENERIC_DIALECT: &str = "generic"; -const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; +pub(crate) const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; fn failure(py: Python<'_>, message: String, code: &str, help: Option<&str>) -> PyErr { cli_error( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index d82b7fb7be..beae4031db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -33,7 +33,7 @@ use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; use crate::bindings::_helpers::load::load_selection::{ - LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, + LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, target_vars, }; use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; @@ -110,7 +110,11 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = LoadProject::new(&discovered, &target_config)?; + let vars: Bound<'py, PyAny> = target_vars( + &discovered, + (&selected_target, &native_field(request, "cli_vars")?), + )?; + let project: LoadProject<'py> = LoadProject::new(&discovered, (&target_config, &vars))?; let selected: Vec> = project.selected_entries( ( &select.extract::>()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 8aa416d540..88e816a2a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -1,12 +1,16 @@ //! Resolve which loaders and sources `sqb load` runs, as `SourceEntry` objects. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use pyo3::exceptions::PyValueError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::errors::AssemblyError; +use sqlbuild_analysis::assembly::project::main::preserved_logical_namespace::preserved_logical_namespace; +use sqlbuild_analysis::assembly::project::models::TargetNamespace; use sqlbuild_core::load_command::errors::UnknownLoadSelector; use sqlbuild_core::load_command::main::load_reference_loaders::load_reference_loaders; use sqlbuild_core::load_command::main::select_load_entries::select_load_entries as select_entries; @@ -14,21 +18,27 @@ use sqlbuild_core::load_command::models::{LoadEntry, LoadGraph}; use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; +use crate::bindings::_helpers::compile_attachment::effective_config::{ + build_effective_connection_config, build_effective_runtime_config, +}; use crate::bindings::_helpers::compile_attachment::target::{ connection_is_default_target, project_location, }; -use crate::bindings::_helpers::compile_driver::targets::managed_write_schema; +use crate::bindings::_helpers::compile_driver::targets::{ + expanded_loader_namespace, managed_write_schema, preserved_loader_schemas, +}; use crate::bindings::_helpers::contracts::spec_resolution::loader_destination_parts; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::construct::{new_helped_error, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; +const COMPILE_INPUT_CODE: &str = "P001"; const LOADER_TABLE_PREFIX: &str = "__loader__"; const UNKNOWN_SELECTOR_CODE: &str = "C901"; const UNKNOWN_SELECTOR_HELP: &str = "Use exact managed source names or loader function names."; @@ -46,10 +56,10 @@ pub(crate) struct LoadProject<'py> { graph: LoadGraph, } -/// Every declared source with the target's database and schema filled in where unset. +/// Every declared source with the target's database and schema, or a managed source's loader_schema expanded with `vars`, filled in where unset. pub(crate) fn environment_sources<'py>( discovered: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, + (target_config, vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult>> { let py: Python<'py> = discovered.py(); let mut sources: Vec> = Vec::new(); @@ -57,13 +67,40 @@ pub(crate) fn environment_sources<'py>( for source in native_field(&source_file?, "source_entries")?.try_iter()? { let source: Bound<'py, PyAny> = source?; let changes: Bound<'py, PyDict> = PyDict::new(py); - for field in ["database", "schema"] { + let managed: bool = native_field(&source, "managed")?.is_truthy()?; + if managed && !target_config.is_none() { + preserved_source_namespace(&source, target_config)?; + } + for (field, loader_first) in [("database", false), ("schema", managed)] { let value: Bound<'py, PyAny> = native_field(&source, field)?; - let effective: Bound<'py, PyAny> = if !value.is_none() || target_config.is_none() { + let fallback: bool = value.is_none() && !target_config.is_none(); + let effective: Bound<'py, PyAny> = if !fallback { value + } else if loader_first { + let loader_schema: Bound<'py, PyAny> = + native_field(target_config, "loader_schema")?; + if loader_schema.is_truthy()? { + loader_schema + } else { + native_field(target_config, field)? + } } else { native_field(target_config, field)? }; + let effective: Bound<'py, PyAny> = + if managed && fallback && !vars.is_none() && !effective.is_none() { + let location: Bound<'py, PyDict> = PyDict::new(py); + location.set_item(field, effective)?; + let expanded: Bound<'py, PyAny> = + expanded_loader_namespace(location.as_any(), vars)?.get_item(field)?; + if expanded.is_none() { + expanded + } else { + expanded.str()?.into_any() + } + } else { + effective + }; changes.set_item(field, effective)?; } sources.push(replaced_keywords(&source, &changes)?); @@ -72,13 +109,53 @@ pub(crate) fn environment_sources<'py>( Ok(sources) } +/// Raise build's error when a managed source has no schema of its own and its target value is `preserve`. +fn preserved_source_namespace( + source: &Bound<'_, PyAny>, + target_config: &Bound<'_, PyAny>, +) -> PyResult<()> { + let text = |record: &Bound<'_, PyAny>, field: &str| -> PyResult> { + native_field(record, field)?.extract() + }; + let loader_schema: Option = + text(target_config, "loader_schema")?.filter(|value| !value.is_empty()); + let target: TargetNamespace = TargetNamespace { + database: text(target_config, "database")?, + schema: loader_schema.clone().or(text(target_config, "schema")?), + loader_schema, + }; + let label: String = format!( + "Managed source '{}'", + text(source, "name")?.unwrap_or_default() + ); + match preserved_logical_namespace( + &label, + Some(&target), + ( + text(source, "database")?.as_ref(), + text(source, "schema")?.as_ref(), + ), + ) { + Ok(()) => Ok(()), + Err(AssemblyError::CompileInput { message, help }) => Err(new_helped_error( + source.py(), + ErrorId::CompileInput, + (message, COMPILE_INPUT_CODE), + help.as_deref(), + )), + Err(other) => Err(PyValueError::new_err(format!("{other:?}"))), + } +} + impl<'py> LoadProject<'py> { + /// The load project, with `vars` expanding managed sources' target locations, or None. pub(crate) fn new( discovered: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, + (target_config, vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult { let py: Python<'py> = discovered.py(); - let sources: Vec> = environment_sources(discovered, target_config)?; + let sources: Vec> = + environment_sources(discovered, (target_config, vars))?; let mut graph: LoadGraph = LoadGraph::default(); for source in &sources { let loader: Bound<'py, PyAny> = native_field(source, "loader")?; @@ -285,18 +362,63 @@ pub(crate) fn loader_defaults<'py>( connection_value.into_iter().filter(|_| inline).collect(), ) }; + let values: Bound<'py, PyDict> = PyDict::new(py); + values.set_item("database", location("database", connection_database)?)?; + values.set_item( + "schema", + location("schema", non_empty_text(connection.get_item("schema")?)?)?, + )?; + let runtime: Bound<'py, PyTuple> = build_effective_runtime_config( + discovered, + Some(selected_target.clone()).filter(|value| !value.is_none()), + Some(cli_vars.clone()).filter(|value| !value.is_none()), + )?; + let resolved: Bound<'py, PyAny> = early_model_templates( + values.as_any(), + &runtime.get_item(1)?, + (&runtime.get_item(0)?, &runtime.get_item(2)?), + )?; Ok(LoaderDefaults { - database: location("database", connection_database)?, - schema: location("schema", non_empty_text(connection.get_item("schema")?)?)?, + database: resolved.get_item("database")?, + schema: resolved.get_item("schema")?, }) } -/// Raise when a selected loader has no schema because the connection's schema is not used. +/// The project, target and command-line vars that expand a managed source's target location. +pub(crate) fn target_vars<'py>( + discovered: &Bound<'py, PyAny>, + (selected_target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + build_effective_runtime_config( + discovered, + Some(selected_target.clone()).filter(|value| !value.is_none()), + Some(cli_vars.clone()).filter(|value| !value.is_none()), + )? + .get_item(1) +} + +/// Raise when a selected loader would write to `preserve` or lost its schema to an ignored connection. pub(crate) fn loader_write_schemas<'py>( discovered: &Bound<'py, PyAny>, (selected_target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), entries: &[Bound<'py, PyAny>], ) -> PyResult<()> { + let mut intermediate_loaders: HashSet = HashSet::new(); + for entry in entries { + let meta: Bound<'py, PyAny> = native_field(entry, "meta")?; + let flag: Option> = match meta.downcast::() { + Ok(meta) => meta.get_item(LOADER_NODE_KEY)?, + Err(_) => None, + }; + if flag + .map(|flag| flag.is_truthy()) + .transpose()? + .unwrap_or(false) + { + intermediate_loaders.insert(native_field(entry, "name")?.extract()?); + } + } + preserved_loader_schemas(discovered, selected_target, &intermediate_loaders)?; if connection_is_default_target(discovered)? { return Ok(()); } @@ -315,11 +437,14 @@ pub(crate) fn loader_write_schemas<'py>( if native_field(entry, "loader")?.is_none() { continue; } + let name: String = native_field(entry, "name")?.extract()?; + let kind: &str = if intermediate_loaders.contains(&name) { + "Intermediate loader" + } else { + "Managed source" + }; managed_write_schema( - &format!( - "Intermediate loader '{}'", - native_field(entry, "name")?.str()? - ), + &format!("{kind} '{name}'"), &native_field(entry, "schema")?, (None, Some(&ignored)), )?; @@ -334,11 +459,33 @@ fn names(value: &Bound<'_, PyAny>) -> PyResult> { value.extract() } -fn passed_defaults<'py>(arguments: &KeywordArguments<'py>) -> PyResult> { - Ok(LoaderDefaults { +/// A Python selection request's arguments, the project it names and its loader defaults. +fn selection_request<'py>( + py: Python<'py>, + (function, required): (&str, &[&str]), + options: Option>, +) -> PyResult<(KeywordArguments<'py>, LoadProject<'py>, LoaderDefaults<'py>)> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + function, + options, + ( + required, + &["loader_default_database", "loader_default_schema"], + ), + )?; + let project: LoadProject<'py> = LoadProject::new( + &arguments.value("discovered_inputs")?, + ( + &arguments.value("target_config")?, + &py.None().into_bound(py), + ), + )?; + let defaults: LoaderDefaults<'py> = LoaderDefaults { database: arguments.value("loader_default_database")?, schema: arguments.value("loader_default_schema")?, - }) + }; + Ok((arguments, project, defaults)) } /// Select source and intermediate loader execution entries for `sqb load`. @@ -348,25 +495,20 @@ fn select_load_entries_entry<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( + let (arguments, project, defaults) = selection_request( py, - "select_load_entries", - options, ( + "select_load_entries", &["discovered_inputs", "select", "exclude", "target_config"], - &["loader_default_database", "loader_default_schema"], ), - )?; - let project: LoadProject<'py> = LoadProject::new( - &arguments.value("discovered_inputs")?, - &arguments.value("target_config")?, + options, )?; let entries: Vec> = project.selected_entries( ( &names(&arguments.value("select")?)?, &names(&arguments.value("exclude")?)?, ), - &passed_defaults(&arguments)?, + &defaults, )?; PyTuple::new(py, entries) } @@ -378,27 +520,19 @@ fn select_load_reference_entries<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( + let (arguments, project, defaults) = selection_request( py, - "select_load_reference_entries", - options, ( + "select_load_reference_entries", &["discovered_inputs", "selected_sources", "target_config"], - &["loader_default_database", "loader_default_schema"], ), - )?; - let project: LoadProject<'py> = LoadProject::new( - &arguments.value("discovered_inputs")?, - &arguments.value("target_config")?, + options, )?; let selected: Vec> = arguments .value("selected_sources")? .try_iter()? .collect::>()?; - PyTuple::new( - py, - project.reference_entries(&selected, &passed_defaults(&arguments)?)?, - ) + PyTuple::new(py, project.reference_entries(&selected, &defaults)?) } /// The default database and schema intermediate loader tables are written to. diff --git a/deploy/README.md b/deploy/README.md index 531a9ff768..e743652bea 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -166,6 +166,121 @@ In `.env`, set `DOCKER_SOCKET` to the socket path and `DOCKER_GID` to `podman compose`. Check that it uses Docker Compose: `podman compose version` prints `Docker Compose version`. +## Run as Podman Quadlet units + +Quadlet runs the server and a location as systemd services, without Compose. These rootful units +use the socket group set up in [Run on Podman](#run-on-podman) and images you built or pulled. + +Name the location container in the server's launcher, in `/etc/sqlbuild/server.toml`: + +```toml +[server] +host = "0.0.0.0" +port = 8000 + +[[launchers.docker]] +location = "orders" +target = "development" +server_url = "http://sqlbuild-server:8000" +max_workers = 1 +socket = "/var/run/docker.sock" +location_container = "orders-location" +executable = "/usr/local/bin/sqb" +``` + +Store the control database URL and, after `sqb admin create-location orders` and +`sqb admin issue-location-token`, the location's bearer as Podman secrets: + +```sh +printf '%s' "$SQLBUILD_CONTROL_DATABASE_URL" | sudo podman secret create sqlbuild-control-database-url - +printf '%s' "$SQLBUILD_LOCATION_TOKEN" | sudo podman secret create sqlbuild-orders-location-token - +``` + +Put these three files in `/etc/containers/systemd/`. `/etc/containers/systemd/sqlbuild.network`: + +```ini +[Network] +NetworkName=sqlbuild +``` + +`/etc/containers/systemd/sqlbuild-server.container`, with `GroupAdd` set to +`getent group sqlbuild-socket | cut -d: -f3`: + +```ini +[Unit] +Description=SQLBuild server and scheduler +Requires=podman.socket +After=podman.socket network-online.target + +[Container] +Image=localhost/sqlbuild-server:0.224.0 +ContainerName=sqlbuild-server +Network=sqlbuild.network +PublishPort=127.0.0.1:8000:8000 +Volume=/etc/sqlbuild/server.toml:/etc/sqlbuild/server.toml:ro +Volume=/run/podman/podman.sock:/var/run/docker.sock +GroupAdd=993 +SecurityLabelDisable=true +Secret=sqlbuild-control-database-url,type=env,target=SQLBUILD_CONTROL_DATABASE_URL +HealthCmd=curl --fail --silent --output /dev/null http://localhost:8000/api/openapi.json +HealthInterval=10s +HealthStartPeriod=30s + +[Service] +Restart=always +TimeoutStartSec=120 + +[Install] +WantedBy=multi-user.target +``` + +`/etc/containers/systemd/orders-location.container`, with the `location_id` that +`create-location` returned: + +```ini +[Unit] +Description=SQLBuild orders code location +Wants=sqlbuild-server.service +After=sqlbuild-server.service + +[Container] +Image=localhost/sqlbuild-orders:0.224.0 +ContainerName=orders-location +Network=sqlbuild.network +Volume=sqlbuild-warehouse:/warehouse +Secret=sqlbuild-orders-location-token,type=env,target=SQLBUILD_LOCATION_TOKEN +Exec=location serve --server http://sqlbuild-server:8000 --location-id 00000000-0000-0000-0000-000000000001 --target development +HealthCmd=sqb location health --server http://sqlbuild-server:8000 --location-id 00000000-0000-0000-0000-000000000001 --target development +HealthInterval=30s +HealthStartPeriod=60s +PodmanArgs=--cpus 2 --memory 1g + +[Service] +Restart=always +TimeoutStartSec=120 + +[Install] +WantedBy=multi-user.target +``` + +Then load and start them: + +```sh +sudo systemctl daemon-reload +sudo systemctl start orders-location.service +sudo podman healthcheck run orders-location +``` + +- `ContainerName=` must equal the launcher's `location_container`: the server finds the location's + image, mounts, network and limits through that container, and workers join its network. +- Write the location ID literally in `Exec=` and `HealthCmd=`. systemd expands `${...}` in those + lines from the service's environment, not the container's. +- `Restart=always` restarts either container when it exits. The location only `Wants=` the server, + so a server restart leaves it running; it checks in again when the server is back. +- The server resolves launcher locations when it starts. To add a location, create it, add its + launcher and container unit, then restart the server: `sudo systemctl restart sqlbuild-server`. +- `SecurityLabelDisable=true` lets the server use the socket on SELinux hosts. + ## Credentials, identity, and platform events Rotate a location credential with `docker compose run --rm --no-deps server admin rotate-token diff --git a/fensu.toml b/fensu.toml index e7cd8dcc61..3f1d83277e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1441,6 +1441,10 @@ reason = "Each callback or worker report forwards one distinct build or check ev paths = ["crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs"] reason = "Two Python entry points with different keyword contracts, project and named target, delegate to project_connection_config." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs"] +reason = "Two Python entry points with different keyword contracts, selected and referenced loader entries, delegate to selection_request and LoadProject." + [[dupes.allowlist]] paths = ["crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs", "crates/sqlbuild-cli/src/native_commands/main/_location_work.rs"] reason = "Metadata sync keeps its pending page for retry and work polling drops the failed request; each reports its own failure." diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index cd462648a0..9d4aca5f7e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -329,6 +329,11 @@ def raw_accounts(ctx: LoaderContext): ... ``` +An intermediate loader without a schema in its `destination` writes to the target's schema. When +the target's schema is `preserve`, which keeps each model's own schema, the loader has no schema to +keep, so `compile`, `build` and `load` stop with `S103` until its `destination` names one, such as +`destination="staging.__loader__raw_accounts"`. + ## Decorator parameters The `@loader` decorator accepts optional parameters that can also be set in the source YAML. When both are specified, the YAML takes precedence. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 53f0f73730..b465d0857c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3663,7 +3663,11 @@ def build_discovered_project_inputs( extract_output_column_locations: bool = True, ) -> DiscoveredProjectInputs: ... def validate_named_target_schema_strategy( - *, discovered_inputs: DiscoveredProjectInputs, adapter: BaseAdapter, selected_target: str | None + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + selected_target: str | None, + cli_vars: dict[str, object] | None = None, ) -> None: ... def validate_managed_loader_target_isolation( *, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 30354664b9..af1bd65b96 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -134,6 +134,7 @@ DELTA_RELATION_FORMAT, DESCRIBE_DETAIL_ROW_WIDTH, DETAILED_TABLE_INFORMATION_MARKER, + MERGE_OPERATION_COLUMN, NON_ROW_RESULT_COLUMN_NAMES, RELATION_NOT_FOUND_ERROR_CLASSES, SERVER_OPERATION_ERROR_CLASS_NAME, @@ -1775,6 +1776,12 @@ def render_delete_insert( unique_key: tuple[str, ...], columns: tuple[str, ...] | None = None, ) -> tuple[str, ...]: + if columns is not None: + return ( + self._render_keyed_replace_merge( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) key_condition: str = " AND ".join( f"{destination}.{self.render_identifier(key)} = __source.{self.render_identifier(key)}" for key in unique_key @@ -1825,6 +1832,20 @@ def render_delete_insert_cursor( replace_where: str = ( f"{quoted_cursor_column} >= {start_bound} AND {quoted_cursor_column} < {end_bound}" ) + if unique_key and columns is not None: + window: str = ( + f"__target.{quoted_cursor_column} >= {start_bound} " + f"AND __target.{quoted_cursor_column} < {end_bound}" + ) + return ( + self._render_keyed_replace_merge( + destination=destination, + sql=sql, + unique_key=unique_key, + columns=columns, + window=window, + ), + ) if unique_key: key_delete, *insert_statements = self.render_delete_insert( destination=destination, sql=sql, unique_key=unique_key, columns=columns @@ -1836,6 +1857,44 @@ def render_delete_insert_cursor( ) return (f"INSERT INTO {destination}{column_list} REPLACE WHERE {replace_where} {sql}",) + def _render_keyed_replace_merge( + self, + *, + destination: str, + sql: str, + unique_key: tuple[str, ...], + columns: tuple[str, ...], + window: str | None = None, + ) -> str: + """One MERGE deleting the window and delta keys, once each, then inserting every row.""" + + quoted: tuple[str, ...] = tuple(self.render_identifier(column) for column in columns) + keys: frozenset[str] = frozenset(key.lower() for key in unique_key) + key_values: str = ", ".join( + self.render_identifier(column) if column.lower() in keys else "NULL" + for column in columns + ) + key_condition: str = " AND ".join( + f"__target.{self.render_identifier(key)} = __source.{self.render_identifier(key)}" + for key in unique_key + ) + window_clause: str = ( + "" if window is None else f" WHEN NOT MATCHED BY SOURCE AND {window} THEN DELETE" + ) + return ( + f"MERGE INTO {destination} AS __target USING (" + f"SELECT {', '.join(quoted)}, 'insert' AS {MERGE_OPERATION_COLUMN} " + f"FROM ({sql}) AS __delta UNION ALL " + f"SELECT DISTINCT {key_values}, 'delete' AS {MERGE_OPERATION_COLUMN} " + f"FROM ({sql}) AS __delta) AS __source " + f"ON __source.{MERGE_OPERATION_COLUMN} = 'delete' AND {key_condition} " + "WHEN MATCHED THEN DELETE " + f"WHEN NOT MATCHED AND __source.{MERGE_OPERATION_COLUMN} = 'insert' THEN INSERT " + f"({', '.join(quoted)}) VALUES " + f"({', '.join(f'__source.{column}' for column in quoted)})" + f"{window_clause}" + ) + def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER TABLE {origin} RENAME TO {destination}",) diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index e90a625426..fd7fa393f3 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -15,6 +15,7 @@ ) UNDEFINED_TABLE_SQLSTATE: str = "42P01" SERVER_OPERATION_ERROR_CLASS_NAME: str = "ServerOperationError" +MERGE_OPERATION_COLUMN: str = "__sqlbuild_op" DETAILED_TABLE_INFORMATION_MARKER: str = "# detailed table information" DESCRIBE_DETAIL_ROW_WIDTH: int = 2 TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 4780bbf52e..59199946b8 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -1565,8 +1565,7 @@ def delete_insert( destination=destination, sql=sql, unique_key=keys, columns=columns ) statement_recorder.record_many(statements) - with self.transaction(connection): - return self._inserted_rows(connection=connection, statements=statements) + return self._replace_staged(connection=connection, statements=statements) def delete_insert_cursor( self, @@ -1582,8 +1581,6 @@ def delete_insert_cursor( cursor_type: str | None = None, unique_key: tuple[str, ...] = (), ) -> int | None: - """Replace a cursor window; staging DDL runs outside the transaction, which DDL commits.""" - statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, sql=sql, @@ -1595,6 +1592,11 @@ def delete_insert_cursor( unique_key=unique_key, ) statement_recorder.record_many(statements) + return self._replace_staged(connection=connection, statements=statements) + + def _replace_staged(self, *, connection: Any, statements: tuple[str, ...]) -> int | None: + """Run deletes and inserts in one transaction; staging DDL, which commits, runs outside.""" + staged: tuple[str, ...] = tuple(s for s in statements if s.startswith("CREATE ")) dropped: tuple[str, ...] = tuple(s for s in statements if s.startswith("DROP ")) for statement in staged: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index a742d1f25e..7c1bbe80f5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1437,8 +1437,10 @@ class ConnectionLocationTestCase: @dataclass(frozen=True) class ManagedSourceLoaderLocationTestCase: - """A command that runs a managed source's loader and the rows its loader schema must hold.""" + """A command that runs a managed source's loader and the rows the `raw` schema must hold.""" description: str + project_toml: str + source_schema: str command: tuple[str, ...] expected_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 088defa40c..3ee1d8dc14 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -17,6 +17,12 @@ ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +_PRESERVED_TARGET_PROJECT: str = ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[defaults]\nschema = "analytics"\n\n[targets.dev]\nschema = "preserve"\n' +) + @pytest.mark.parametrize( "test_case", @@ -55,6 +61,25 @@ ), expected_example_count=1, ), + MissingTargetDatabaseTestCase( + description="the ignored connection schema is shown with command-line vars applied", + project_toml=( + 'name = "orders"\nadapter = "snowflake"\ndefault_target = "dev"\n\n' + '[vars]\norders_schema = "ORDERS"\n\n' + '[connections.warehouse]\naccount = "example"\nschema = "${orders_schema}"\n\n' + '[targets.dev]\ndatabase = "ANALYTICS"\n' + ), + expected_error=( + "The connection's schema 'ORDERS_CI' is not used because the project defines " + "named connections or targets.\n" + " = help: to build this target into that schema, set this in " + "sqlbuild_project.toml:\n" + " [targets.dev]\n" + ' schema = "ORDERS_CI"\n' + ), + expected_example_count=0, + command=("--no-color", "compile", "--vars", '{"orders_schema": "ORDERS_CI"}'), + ), MissingTargetDatabaseTestCase( description="a named connection's schema is named as unused", project_toml=( @@ -94,9 +119,10 @@ }, expected_error=( "error[S103]: Intermediate loader 'raw_orders' has no explicitly resolved " - "physical write schema. Set schema on the resource or its defaults, selected " - "target, loader_schema, or connection. The connection's schema 'raw' is not used " - "because the project defines named connections or targets.\n" + "physical write schema. Set schema on the resource or its defaults, the selected " + "target or its loader_schema, or the project's only top-level [connection]. The " + "connection's schema 'raw' is not used because the project defines named " + "connections or targets.\n" " = help: to set project-wide defaults, set this in sqlbuild_project.toml:\n" " [defaults]\n" ' schema = "raw"\n' @@ -123,9 +149,10 @@ }, expected_error=( "error[S103]: Intermediate loader 'raw_orders' has no explicitly resolved " - "physical write schema. Set schema on the resource or its defaults, selected " - "target, loader_schema, or connection. The connection's schema 'raw' is not used " - "because the project defines named connections or targets.\n" + "physical write schema. Set schema on the resource or its defaults, the selected " + "target or its loader_schema, or the project's only top-level [connection]. The " + "connection's schema 'raw' is not used because the project defines named " + "connections or targets.\n" " = help: to set project-wide defaults, set this in sqlbuild_project.toml:\n" " [defaults]\n" ' schema = "raw"\n' @@ -133,6 +160,106 @@ expected_example_count=1, command=("--no-color", "load", "--select", "raw_orders"), ), + MissingTargetDatabaseTestCase( + description="a loader under a preserved target schema stops compile", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[defaults]\nschema = "analytics"\n\n[targets.dev]\nschema = "preserve"\n' + ), + extra_files={ + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error=( + "error[S103]: Intermediate loader 'raw_orders' would write to a schema named " + "'preserve'. The target's schema 'preserve' keeps each model's own schema, and a " + "loader has none, so its destination must name a schema.\n" + " = help: to choose where it writes, set its destination in its @loader " + "decorator:\n" + ' @loader(destination="my_schema.__loader__raw_orders")\n' + ), + expected_example_count=0, + ), + MissingTargetDatabaseTestCase( + description="sqb load stops a loader under a preserved target schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[defaults]\nschema = "analytics"\n\n[targets.dev]\nschema = "preserve"\n' + ), + extra_files={ + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error=( + "error[S103]: Intermediate loader 'raw_orders' would write to a schema named " + "'preserve'. The target's schema 'preserve' keeps each model's own schema, and a " + "loader has none, so its destination must name a schema.\n" + " = help: to choose where it writes, set its destination in its @loader " + "decorator:\n" + ' @loader(destination="my_schema.__loader__raw_orders")\n' + ), + expected_example_count=0, + command=("--no-color", "load", "--select", "raw_orders"), + ), + MissingTargetDatabaseTestCase( + description="sqb load stops a managed source under a preserved loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "preserve"\n' + ), + extra_files={ + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " managed: true\n write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error=( + "error[P001]: Managed source 'raw_orders' has no logical schema, but the " + "selected target sets schema to 'preserve'\n" + " = help: Set schema on the resource or its defaults, or configure a literal " + "target schema.\n" + ), + expected_example_count=0, + command=("--no-color", "load", "--select", "raw_orders"), + ), + MissingTargetDatabaseTestCase( + description="sqb load refuses a run context template in the target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "raw_${CTX:run.id}"\n' + ), + extra_files={ + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " managed: true\n write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error="managed loader target namespace does not allow CTX templates", + expected_example_count=0, + command=("--no-color", "load", "--select", "raw_orders"), + ), ), ids=lambda case: case.description, ) @@ -224,32 +351,93 @@ def test_given_connection_location_when_compiling_then_models_build_there( ( ManagedSourceLoaderLocationTestCase( description="build loads the managed source into the target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n[connection]\n' + 'database = "dev.duckdb"\nschema = "landing"\n\n' + '[connections.archive]\ndatabase = "archive.duckdb"\n\n' + '[targets.dev]\nloader_schema = "raw"\n' + ), + source_schema="", command=("--no-color", "build"), expected_rows=((1,),), ), + ManagedSourceLoaderLocationTestCase( + description="load loads the managed source into the target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n[connection]\n' + 'database = "dev.duckdb"\nschema = "landing"\n\n' + '[connections.archive]\ndatabase = "archive.duckdb"\n\n' + '[targets.dev]\nloader_schema = "raw"\n' + ), + source_schema="", + command=("--no-color", "load", "--select", "raw_orders"), + expected_rows=((1,),), + ), + ManagedSourceLoaderLocationTestCase( + description="load expands a templated target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[vars]\nraw_schema = "raw"\n\n[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "${raw_schema}"\n' + ), + source_schema="", + command=("--no-color", "load", "--select", "raw_orders"), + expected_rows=((1,),), + ), + ManagedSourceLoaderLocationTestCase( + description="build expands a templated target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[vars]\nraw_schema = "raw"\n\n[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "${raw_schema}"\n' + ), + source_schema="", + command=("--no-color", "build"), + expected_rows=((1,),), + ), + ManagedSourceLoaderLocationTestCase( + description="load ignores a target loader schema template no source uses", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "${missing_var}"\n' + ), + source_schema=" schema: raw\n", + command=("--no-color", "load", "--select", "raw_orders"), + expected_rows=((1,),), + ), + ManagedSourceLoaderLocationTestCase( + description="build under a preserved target loads the source into its own schema", + project_toml=_PRESERVED_TARGET_PROJECT, + source_schema=" schema: raw\n", + command=("--no-color", "build"), + expected_rows=((1,),), + ), + ManagedSourceLoaderLocationTestCase( + description="load under a preserved target loads the source into its own schema", + project_toml=_PRESERVED_TARGET_PROJECT, + source_schema=" schema: raw\n", + command=("--no-color", "load", "--select", "raw_orders"), + expected_rows=((1,),), + ), ), ids=lambda case: case.description, ) -def test_given_ignored_connection_schema_when_building_managed_source_then_loader_schema_is_used( +def test_given_managed_source_when_loading_then_source_lands_in_its_resolved_schema( test_case: ManagedSourceLoaderLocationTestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="orders", repo_files={ - "sqlbuild_project.toml": ( - 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n[connection]\n' - 'database = "dev.duckdb"\nschema = "landing"\n\n' - '[connections.archive]\ndatabase = "archive.duckdb"\n\n' - '[targets.dev]\nloader_schema = "raw"\n' - ), + "sqlbuild_project.toml": test_case.project_toml, "models/orders.sql": ( "MODEL (description 'Test model orders.', materialized view, " 'schema staging);\n\nSELECT order_id FROM __source("raw_orders")\n' ), "sources/raw.yml": ( "sources:\n - name: raw_orders\n description: Raw orders.\n" - " managed: true\n write_strategy: table\n" + f" managed: true\n{test_case.source_schema} write_strategy: table\n" " columns:\n - name: order_id\n type: INTEGER\n" ), "python/loaders/orders.py": ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/debug/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/debug/_test_types.py index dbffda67e6..6fe7801d38 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/debug/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/debug/_test_types.py @@ -9,6 +9,7 @@ class DebugCliTestCase: command: tuple[str, ...] expected_stdout_fragment: str expected_returncode: int = 0 + project_toml: str = "" @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py index d65d2d5f71..1f508ca121 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py @@ -77,25 +77,35 @@ def test_given_debug_command_when_running_then_outputs_checks( expected_stdout_fragment=( " database: DEFAULTS_DB [OK resolved]\n schema: DEFAULTS_SCHEMA [OK resolved]\n" ), + project_toml=( + 'name = "debug_defaults"\nadapter = "snowflake"\n\n' + '[connection]\naccount = "acme"\nuser = "u"\npassword = "p"\n' + 'database = "CONN_DB"\nschema = "CONN_SCHEMA"\n\n' + '[defaults]\ndatabase = "DEFAULTS_DB"\nschema = "DEFAULTS_SCHEMA"\n' + ), + ), + DebugCliTestCase( + description="a duckdb connection file is not reported as the models' database", + command=("debug", "--no-connection"), + expected_stdout_fragment=( + " database: not configured [OK resolved]\n schema: sales [OK resolved]\n" + ), + project_toml=( + 'name = "debug_defaults"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "shop.duckdb"\nschema = "sales"\n' + ), ), ], ids=lambda case: case.description, ) -def test_given_snowflake_defaults_location_when_debugging_then_reports_it( +def test_given_project_location_when_debugging_then_reports_where_models_build( tmp_path: Path, test_case: DebugCliTestCase, ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="debug_defaults", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "debug_defaults"\nadapter = "snowflake"\n\n' - '[connection]\naccount = "acme"\nuser = "u"\npassword = "p"\n' - 'database = "CONN_DB"\nschema = "CONN_SCHEMA"\n\n' - '[defaults]\ndatabase = "DEFAULTS_DB"\nschema = "DEFAULTS_SCHEMA"\n' - ), - }, + repo_files={"sqlbuild_project.toml": test_case.project_toml}, ) result: subprocess.CompletedProcess[str] = run_sqb( diff --git a/tests/goldens/compiler/failure/target-missing-write-schema.json b/tests/goldens/compiler/failure/target-missing-write-schema.json index dc9dd0833a..37133bf905 100644 --- a/tests/goldens/compiler/failure/target-missing-write-schema.json +++ b/tests/goldens/compiler/failure/target-missing-write-schema.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "S103", - "message": "Model 'customer_totals' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, selected target, loader_schema, or connection." + "message": "Model 'customer_totals' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, the selected target or its loader_schema, or the project's only top-level [connection]." } ] } diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index d43d91fcbc..e849504e9a 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -209,3 +209,11 @@ class DatabricksFreshnessBatchErrorTestCase: description: str batch_error: Exception expected_match: str + + +@dataclass(frozen=True) +class DatabricksKeyedDeleteInsertTestCase: + description: str + unique_key: tuple[str, ...] + columns: tuple[str, ...] + expected_statements: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py index 1dae5347e6..c3b604ba5d 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py @@ -133,3 +133,16 @@ def read_databricks_usage( return DatabricksAdapter().relation_usage( connection=connection, relations=relations, window_seconds=window_seconds ) + + +class RecordingDatabricksAdapter(DatabricksAdapter): + """Databricks adapter that records each statement instead of running it.""" + + def __init__(self) -> None: + super().__init__() + self.executed: list[str] = [] + + def execute(self, *, connection: Any, sql: str, **kwargs: Any) -> Any: + del connection, kwargs + self.executed.append(sql) + return None diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index 4e53389c01..3f02cd4442 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import FunctionArgument, FunctionReturnColumn, InferredNullability +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ExpressionInferenceProfile, RelationInfo, @@ -20,6 +21,7 @@ DatabricksDeltaDetailAgeTestCase, DatabricksExpressionInferenceProfileTestCase, DatabricksFreshnessBatchErrorTestCase, + DatabricksKeyedDeleteInsertTestCase, DatabricksMergeExclusionTestCase, DatabricksPruneSqlTestCase, DatabricksPythonFunctionSupportTestCase, @@ -38,6 +40,7 @@ FailingDatabricksMetadataCursor, FakeDatabricksMetadataConnection, FakeDatabricksMetadataCursor, + RecordingDatabricksAdapter, RequestError, ServerOperationError, ) @@ -356,21 +359,26 @@ def test_given_request_error_in_freshness_batch_when_getting_metadata_then_it_is ), ), DatabricksRenderDeleteInsertCursorTestCase( - description="keyed cursor deletes the window and the delta keys in one statement", + description="keyed cursor replaces the window and the delta keys in one merge", target="`workspace`.`test`.`orders`", sql="SELECT id, status FROM `workspace`.`test`.`orders__delta`", cursor_column="id", cursor_start="1", cursor_end="10", columns=("id", "status"), - unique_key=("id",), + unique_key=("ID",), expected_statements=( - "DELETE FROM `workspace`.`test`.`orders` WHERE (`id` >= 1 AND `id` < 10) " - "OR EXISTS (SELECT 1 FROM (SELECT id, status FROM " - "`workspace`.`test`.`orders__delta`) AS __source " - "WHERE `workspace`.`test`.`orders`.`id` = __source.`id`)", - "INSERT INTO `workspace`.`test`.`orders` (`id`, `status`) " - "SELECT id, status FROM `workspace`.`test`.`orders__delta`", + "MERGE INTO `workspace`.`test`.`orders` AS __target USING (" + "SELECT `id`, `status`, 'insert' AS __sqlbuild_op FROM " + "(SELECT id, status FROM `workspace`.`test`.`orders__delta`) AS __delta " + "UNION ALL SELECT DISTINCT `id`, NULL, 'delete' AS __sqlbuild_op FROM " + "(SELECT id, status FROM `workspace`.`test`.`orders__delta`) AS __delta) " + "AS __source ON __source.__sqlbuild_op = 'delete' " + "AND __target.`ID` = __source.`ID` WHEN MATCHED THEN DELETE " + "WHEN NOT MATCHED AND __source.__sqlbuild_op = 'insert' THEN INSERT " + "(`id`, `status`) VALUES (__source.`id`, __source.`status`) " + "WHEN NOT MATCHED BY SOURCE AND __target.`id` >= 1 AND __target.`id` < 10 " + "THEN DELETE", ), ), ], @@ -847,3 +855,42 @@ def test_given_databricks_schema_when_checking_existence_then_current_catalog_is assert exists is test_case.expected_exists, test_case.description assert cursor.executed_sql is not None assert test_case.expected_query_fragment in cursor.executed_sql, test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksKeyedDeleteInsertTestCase( + description="keyed delete_insert runs one merge, so no failure leaves rows deleted", + unique_key=("order_id",), + columns=("order_id", "status"), + expected_statements=( + "MERGE INTO `main`.`marts`.`orders` AS __target USING (" + "SELECT `order_id`, `status`, 'insert' AS __sqlbuild_op FROM " + "(SELECT order_id, status FROM delta) AS __delta UNION ALL " + "SELECT DISTINCT `order_id`, NULL, 'delete' AS __sqlbuild_op FROM " + "(SELECT order_id, status FROM delta) AS __delta) AS __source " + "ON __source.__sqlbuild_op = 'delete' " + "AND __target.`order_id` = __source.`order_id` WHEN MATCHED THEN DELETE " + "WHEN NOT MATCHED AND __source.__sqlbuild_op = 'insert' THEN INSERT " + "(`order_id`, `status`) VALUES (__source.`order_id`, __source.`status`)", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_keyed_delete_insert_when_writing_then_databricks_runs_one_merge( + test_case: DatabricksKeyedDeleteInsertTestCase, +) -> None: + adapter: RecordingDatabricksAdapter = RecordingDatabricksAdapter() + + adapter.delete_insert( + connection=object(), + destination="`main`.`marts`.`orders`", + sql="SELECT order_id, status FROM delta", + unique_key=test_case.unique_key, + columns=test_case.columns, + statement_recorder=StatementRecorder(), + ) + + assert tuple(adapter.executed) == test_case.expected_statements diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index a984ec19a2..a336d3e62d 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -434,3 +434,10 @@ class SnowflakeKeyedCursorDeleteInsertTestCase: unique_key: tuple[str, ...] expected_prefixes: tuple[str, ...] expected_inserted: int + + +@dataclass(frozen=True) +class SnowflakeKeyedDeleteInsertFailureTestCase: + description: str + failing_keyword: str + expected_prefixes: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index 8b15802d82..f6f14eecfc 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -3,12 +3,12 @@ from __future__ import annotations import sys -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import nullcontext from dataclasses import dataclass from io import StringIO from itertools import cycle, islice -from types import ModuleType, SimpleNamespace +from types import MappingProxyType, ModuleType, SimpleNamespace from typing import Any from sqlbuild._native import NativeProgressProjector @@ -687,11 +687,25 @@ def prefix_listing_statements(warehouse: RecordingSnowflakeWarehouse) -> tuple[s class RecordingSnowflakeAdapter(SnowflakeAdapter): """Snowflake adapter that records each statement instead of running it.""" - def __init__(self) -> None: + def __init__( + self, *, failures: Mapping[str, Callable[[], None]] = MappingProxyType({}) + ) -> None: super().__init__() self.executed: list[str] = [] + self.failures: Mapping[str, Callable[[], None]] = failures def execute(self, *, connection: Any, sql: str, **kwargs: Any) -> Any: del connection, kwargs self.executed.append(sql) + self.failures.get(sql.split(" ", 1)[0], _succeed)() return SimpleNamespace(rowcount=3) + + +def _succeed() -> None: + """A statement that runs without error.""" + + +def fail_statement() -> None: + """A statement the warehouse rejects.""" + + raise RuntimeError("statement failed") diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py index ca2daab813..c012fae3ba 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py @@ -7,9 +7,11 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeKeyedCursorDeleteInsertTestCase, + SnowflakeKeyedDeleteInsertFailureTestCase, ) from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( RecordingSnowflakeAdapter, + fail_statement, ) @@ -66,3 +68,43 @@ def test_given_cursor_window_when_replacing_then_snowflake_commits_deletes_with_ for statement, prefix in zip(adapter.executed, test_case.expected_prefixes, strict=True) ) assert inserted == test_case.expected_inserted + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeKeyedDeleteInsertFailureTestCase( + description="a failing insert rolls the key delete back", + failing_keyword="INSERT", + expected_prefixes=( + "CREATE", + "BEGIN", + "DELETE FROM ANALYTICS.MARTS.ORDERS USING", + "INSERT", + "ROLLBACK", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_failure_after_delete_when_keyed_delete_insert_then_snowflake_rolls_back( + test_case: SnowflakeKeyedDeleteInsertFailureTestCase, +) -> None: + adapter: RecordingSnowflakeAdapter = RecordingSnowflakeAdapter( + failures={test_case.failing_keyword: fail_statement} + ) + + with pytest.raises(RuntimeError, match="statement failed"): + adapter.delete_insert( + connection=object(), + destination="ANALYTICS.MARTS.ORDERS", + sql="SELECT * FROM ANALYTICS.MARTS.ORDERS__DELTA", + unique_key=("order_id",), + statement_recorder=StatementRecorder(), + ) + + assert len(adapter.executed) == len(test_case.expected_prefixes) + assert all( + statement.startswith(prefix) + for statement, prefix in zip(adapter.executed, test_case.expected_prefixes, strict=True) + ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py index 53e20809fd..da0338fa98 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py @@ -31,3 +31,5 @@ class LoaderDefaultResolutionTestCase: expected_defaults: tuple[str | None, str | None] project_defaults: tuple[str | None, str | None] = (None, None) connections: dict[str, dict[str, object]] = field(default_factory=dict) + project_vars: dict[str, str] = field(default_factory=dict) + cli_vars: dict[str, object] | None = None diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index 3bfc6126cc..8ff72a5fa5 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -167,6 +167,23 @@ def test_given_intermediate_loader_when_selecting_then_parses_destination_with_e project_defaults=("DEFAULTS_DB", "DEFAULTS_SCHEMA"), expected_defaults=("DEFAULTS_DB", "DEFAULTS_SCHEMA"), ), + LoaderDefaultResolutionTestCase( + description="templated project defaults expand project vars as models do", + adapter="postgres", + connection={"database": "connection_db", "schema": "connection_schema"}, + project_defaults=("${warehouse_db}", "orders_${environment}"), + project_vars={"warehouse_db": "analytics", "environment": "dev"}, + expected_defaults=("analytics", "orders_dev"), + ), + LoaderDefaultResolutionTestCase( + description="command-line vars override project vars in templated defaults", + adapter="postgres", + connection={"database": "connection_db", "schema": "connection_schema"}, + project_defaults=(None, "orders_${environment}"), + project_vars={"environment": "dev"}, + cli_vars={"environment": "ci"}, + expected_defaults=("connection_db", "orders_ci"), + ), ), ids=lambda case: case.description, ) @@ -182,6 +199,7 @@ def test_given_connection_namespace_when_resolving_load_defaults_then_matches_co defaults=DefaultsConfig( database=test_case.project_defaults[0], schema=test_case.project_defaults[1] ), + vars=test_case.project_vars, ), local_config=LocalConfig(), ) @@ -190,7 +208,7 @@ def test_given_connection_namespace_when_resolving_load_defaults_then_matches_co discovered_inputs=discovered_inputs, selected_target=None, target_config=None, - cli_vars=None, + cli_vars=test_case.cli_vars, ) assert defaults == test_case.expected_defaults diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 9719cc75c5..02bb37969d 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -309,6 +309,11 @@ def raw_accounts(ctx: LoaderContext): ... ``` +An intermediate loader without a schema in its `destination` writes to the target's schema. When +the target's schema is `preserve`, which keeps each model's own schema, the loader has no schema to +keep, so `compile`, `build` and `load` stop with `S103` until its `destination` names one, such as +`destination="staging.__loader__raw_accounts"`. + ## Decorator parameters The `@loader` decorator accepts optional parameters that can also be set in the source YAML. When both are specified, the YAML takes precedence.