From 14a9c18ec7bd8eba6469b275b51cdf69b9fee4e1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:16:58 +0000 Subject: [PATCH 01/18] fix(server): index every control-store foreign key --- .../src/store/schema.sql | 10 +++ .../store/schema/test_foreign_key_indexes.rs | 80 +++++++++++++++++++ .../tests/store/test_contracts.rs | 2 + 3 files changed, 92 insertions(+) create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql index cd25e62407..ebcc54a1ea 100644 --- a/crates/sqlbuild-orchestration/src/store/schema.sql +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -92,6 +92,7 @@ CREATE TABLE sqb_jobs ( definition_json {json} NOT NULL, PRIMARY KEY (location_id, target, job_key) ); +CREATE INDEX sqb_jobs_snapshot ON sqb_jobs(snapshot_id); CREATE TABLE sqb_triggers ( trigger_id {uuid} PRIMARY KEY, location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), @@ -104,6 +105,8 @@ CREATE TABLE sqb_triggers ( removed_in_snapshot_id {uuid} REFERENCES sqb_snapshots(snapshot_id), UNIQUE (location_id, target, kind, name) ); +CREATE INDEX sqb_triggers_snapshot ON sqb_triggers(snapshot_id); +CREATE INDEX sqb_triggers_removed ON sqb_triggers(removed_in_snapshot_id) WHERE removed_in_snapshot_id IS NOT NULL; CREATE TABLE sqb_trigger_state ( trigger_id {uuid} PRIMARY KEY REFERENCES sqb_triggers(trigger_id), status TEXT NOT NULL CHECK (status IN ('running', 'paused')), @@ -162,6 +165,8 @@ CREATE TABLE sqb_runs ( claimed_at {time}, last_heartbeat_at {time} ); +CREATE INDEX sqb_runs_trigger ON sqb_runs(trigger_id) WHERE trigger_id IS NOT NULL; +CREATE INDEX sqb_runs_rerun_of ON sqb_runs(rerun_of_run_id) WHERE rerun_of_run_id IS NOT NULL; CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); CREATE INDEX sqb_runs_location_ended ON sqb_runs(location_id, target, ended_at); CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; @@ -262,6 +267,8 @@ CREATE TABLE sqb_location_work ( result_json {json}, completed_at {time} ); +CREATE INDEX sqb_location_work_session ON sqb_location_work(session_id) WHERE session_id IS NOT NULL; +CREATE INDEX sqb_location_work_snapshot ON sqb_location_work(snapshot_id); CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); CREATE TABLE sqb_platform_sink_deliveries ( sink_id TEXT NOT NULL, @@ -288,6 +295,7 @@ CREATE TABLE sqb_run_requests ( run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), PRIMARY KEY (trigger_id, run_key) ); +CREATE INDEX sqb_run_requests_run ON sqb_run_requests(run_id); CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled'); CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; @@ -321,6 +329,7 @@ CREATE TABLE sqb_warehouse_fingerprints ( target_name TEXT, scan_id {uuid} ); +CREATE INDEX sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash); CREATE INDEX sqb_warehouse_fingerprints_scope ON sqb_warehouse_fingerprints (location_id, target, observed_at, event_id); CREATE TABLE sqb_development_locations ( project_path TEXT NOT NULL, @@ -372,6 +381,7 @@ CREATE TABLE sqb_role_assignments ( target TEXT, active_key TEXT NOT NULL UNIQUE ); +CREATE INDEX sqb_role_assignments_location ON sqb_role_assignments(location_id); CREATE INDEX sqb_role_assignments_principal ON sqb_role_assignments(principal_id); CREATE TABLE sqb_person_groups ( principal_id {uuid} NOT NULL REFERENCES sqb_principals(principal_id), diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs new file mode 100644 index 0000000000..a12ddec4bd --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs @@ -0,0 +1,80 @@ +//! Every foreign key is indexed on its referencing columns, so deleting a parent stays bounded. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::TransactionMode; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::store::types::Row; + +const FOREIGN_KEYS: &str = "SELECT m.name AS table_name, f.id AS key_id, f.\"from\" AS column_name \ + FROM sqlite_master m JOIN pragma_foreign_key_list(m.name) f WHERE m.type = 'table'"; +const INDEX_COLUMNS: &str = "SELECT m.name AS table_name, l.name AS index_name, \ + l.\"unique\" AS is_unique, i.seqno AS position, COALESCE(i.name, '') AS column_name \ + FROM sqlite_master m JOIN pragma_index_list(m.name) l JOIN pragma_index_info(l.name) i \ + WHERE m.type = 'table' ORDER BY m.name, l.name, i.seqno"; + +fn text(row: &Row, key: &str) -> String { + row[key].as_str().expect("text column").to_owned() +} + +#[test] +fn given_control_store_schema_when_listing_foreign_keys_then_each_is_indexed() { + run(async { + let fixture = open_test_store(Backend::Sqlite) + .await + .expect("open SQLite store"); + migrate_store(&fixture.store).await.expect("migrate store"); + let transaction = begin(&fixture.store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, keys) = query(transaction, FOREIGN_KEYS, vec![]) + .await + .expect("list foreign keys"); + let (transaction, columns) = query(transaction, INDEX_COLUMNS, vec![]) + .await + .expect("list index columns"); + commit(transaction).await.expect("finish read"); + + let mut foreign_keys: BTreeMap<(String, i64), BTreeSet> = BTreeMap::new(); + for row in &keys { + foreign_keys + .entry(( + text(row, "table_name"), + row["key_id"].as_i64().expect("key id"), + )) + .or_default() + .insert(text(row, "column_name")); + } + let mut indexes: BTreeMap<(String, String), (bool, Vec)> = BTreeMap::new(); + for row in &columns { + let entry = indexes + .entry((text(row, "table_name"), text(row, "index_name"))) + .or_insert((row["is_unique"].as_i64() == Some(1), Vec::new())); + entry.1.push(text(row, "column_name")); + } + let uncovered: Vec<(String, BTreeSet)> = foreign_keys + .into_iter() + .filter(|((table, _), referencing)| { + !indexes.iter().any(|((index_table, _), (unique, indexed))| { + index_table == table + && (indexed.len() >= referencing.len() + && indexed[..referencing.len()] + .iter() + .all(|column| referencing.contains(column)) + || *unique && indexed.iter().all(|column| referencing.contains(column))) + }) + }) + .map(|((table, _), referencing)| (table, referencing)) + .collect(); + assert!( + uncovered.is_empty(), + "foreign keys without an index on their referencing columns: {uncovered:?}" + ); + close_test_store(fixture).await; + }); +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs index 868430a934..ebf0f584dc 100644 --- a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -18,6 +18,8 @@ pub(crate) mod store_helpers; pub(crate) mod store_test_types; #[path = "recall/test_contract.rs"] mod test_contract; +#[path = "schema/test_foreign_key_indexes.rs"] +mod test_foreign_key_indexes; #[path = "recall/test_types.rs"] mod test_types; #[path = "recall/test_views.rs"] From 5665a3758f698c1ce0252774458aeadecb1d916d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:47:33 +0000 Subject: [PATCH 02/18] fix: write keyed delete_insert atomically, add store indexes in place and expand loader location templates --- .../src/store/_helpers/migration.rs | 9 ++ .../src/store/indexes.sql | 10 ++ .../src/store/main/migrate_store.rs | 15 +-- .../src/store/schema.sql | 10 -- .../tests/store/schema/helpers.rs | 7 ++ .../tests/store/schema/test_contracts.rs | 10 ++ .../store/schema/test_foreign_key_indexes.rs | 118 +++++++++--------- .../tests/store/schema/test_store_upgrade.rs | 108 ++++++++++++++++ .../tests/store/schema/test_types.rs | 18 +++ .../tests/store/test_contracts.rs | 4 +- .../bindings/_helpers/load/load_selection.rs | 25 +++- .../databricks/classes/databricks_adapter.py | 59 +++++++++ src/sqlbuild/adapters/databricks/constants.py | 1 + .../snowflake/classes/snowflake_adapter.py | 10 +- .../adapters/databricks/_test_types.py | 8 ++ .../sqlbuild/adapters/databricks/helpers.py | 13 ++ .../adapters/databricks/test_client.py | 63 ++++++++-- .../adapters/snowflake/_test_types.py | 7 ++ .../sqlbuild/adapters/snowflake/helpers.py | 20 ++- .../test_keyed_cursor_delete_insert.py | 42 +++++++ .../_helpers/load_selection/_test_types.py | 2 + .../_helpers/load_selection/test_selection.py | 20 ++- 22 files changed, 485 insertions(+), 94 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/store/indexes.sql create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/test_store_upgrade.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/test_types.rs diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 980caf2878..8a2ed7baa2 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -9,6 +9,15 @@ use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); +/// Additive indexes, created on every open and kept out of the digest so old stores gain them. +pub(crate) const INDEXES: &str = include_str!("../indexes.sql"); + +pub(crate) fn statements(sql: &str) -> impl Iterator { + sql.split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) +} + pub(crate) fn schema_digest(store: &ControlStore) -> String { sha256_hex(format!("{}\n{SCHEMA}", recall_schema(store).join(";\n")).as_bytes()) } diff --git a/crates/sqlbuild-orchestration/src/store/indexes.sql b/crates/sqlbuild-orchestration/src/store/indexes.sql new file mode 100644 index 0000000000..4f141c93ee --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/indexes.sql @@ -0,0 +1,10 @@ +CREATE INDEX IF NOT EXISTS sqb_jobs_snapshot ON sqb_jobs(snapshot_id); +CREATE INDEX IF NOT EXISTS sqb_triggers_snapshot ON sqb_triggers(snapshot_id); +CREATE INDEX IF NOT EXISTS sqb_triggers_removed ON sqb_triggers(removed_in_snapshot_id) WHERE removed_in_snapshot_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sqb_runs_trigger ON sqb_runs(trigger_id) WHERE trigger_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sqb_runs_rerun_of ON sqb_runs(rerun_of_run_id) WHERE rerun_of_run_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sqb_location_work_session ON sqb_location_work(session_id) WHERE session_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sqb_location_work_snapshot ON sqb_location_work(snapshot_id); +CREATE INDEX IF NOT EXISTS sqb_run_requests_run ON sqb_run_requests(run_id); +CREATE INDEX IF NOT EXISTS sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash); +CREATE INDEX IF NOT EXISTS sqb_role_assignments_location ON sqb_role_assignments(location_id); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index ed230c65a9..e82180367e 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -2,7 +2,9 @@ use uuid::Uuid; -use crate::store::_helpers::migration::{SCHEMA, render, schema_digest, verify_schema}; +use crate::store::_helpers::migration::{ + INDEXES, SCHEMA, render, schema_digest, statements, verify_schema, +}; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -10,7 +12,7 @@ use crate::store::main::commit::commit; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; -/// Create the schema once, or reject a schema this server did not create. +/// Create or verify the schema, then add any missing additive indexes. pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { let mut transaction = begin(store, TransactionMode::Write).await?; (transaction, _) = query(transaction, &render(store, @@ -28,11 +30,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { for statement in recall_schema(store) { (transaction, _) = query(transaction, &statement, vec![]).await?; } - for statement in SCHEMA - .split(';') - .map(str::trim) - .filter(|statement| !statement.is_empty()) - { + for statement in statements(SCHEMA) { (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; } let now = transaction.now; @@ -46,5 +44,8 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ) .await?; } + for statement in statements(INDEXES) { + (transaction, _) = query(transaction, statement, vec![]).await?; + } commit(transaction).await } diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql index ebcc54a1ea..cd25e62407 100644 --- a/crates/sqlbuild-orchestration/src/store/schema.sql +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -92,7 +92,6 @@ CREATE TABLE sqb_jobs ( definition_json {json} NOT NULL, PRIMARY KEY (location_id, target, job_key) ); -CREATE INDEX sqb_jobs_snapshot ON sqb_jobs(snapshot_id); CREATE TABLE sqb_triggers ( trigger_id {uuid} PRIMARY KEY, location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), @@ -105,8 +104,6 @@ CREATE TABLE sqb_triggers ( removed_in_snapshot_id {uuid} REFERENCES sqb_snapshots(snapshot_id), UNIQUE (location_id, target, kind, name) ); -CREATE INDEX sqb_triggers_snapshot ON sqb_triggers(snapshot_id); -CREATE INDEX sqb_triggers_removed ON sqb_triggers(removed_in_snapshot_id) WHERE removed_in_snapshot_id IS NOT NULL; CREATE TABLE sqb_trigger_state ( trigger_id {uuid} PRIMARY KEY REFERENCES sqb_triggers(trigger_id), status TEXT NOT NULL CHECK (status IN ('running', 'paused')), @@ -165,8 +162,6 @@ CREATE TABLE sqb_runs ( claimed_at {time}, last_heartbeat_at {time} ); -CREATE INDEX sqb_runs_trigger ON sqb_runs(trigger_id) WHERE trigger_id IS NOT NULL; -CREATE INDEX sqb_runs_rerun_of ON sqb_runs(rerun_of_run_id) WHERE rerun_of_run_id IS NOT NULL; CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); CREATE INDEX sqb_runs_location_ended ON sqb_runs(location_id, target, ended_at); CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; @@ -267,8 +262,6 @@ CREATE TABLE sqb_location_work ( result_json {json}, completed_at {time} ); -CREATE INDEX sqb_location_work_session ON sqb_location_work(session_id) WHERE session_id IS NOT NULL; -CREATE INDEX sqb_location_work_snapshot ON sqb_location_work(snapshot_id); CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); CREATE TABLE sqb_platform_sink_deliveries ( sink_id TEXT NOT NULL, @@ -295,7 +288,6 @@ CREATE TABLE sqb_run_requests ( run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), PRIMARY KEY (trigger_id, run_key) ); -CREATE INDEX sqb_run_requests_run ON sqb_run_requests(run_id); CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled'); CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; @@ -329,7 +321,6 @@ CREATE TABLE sqb_warehouse_fingerprints ( target_name TEXT, scan_id {uuid} ); -CREATE INDEX sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash); CREATE INDEX sqb_warehouse_fingerprints_scope ON sqb_warehouse_fingerprints (location_id, target, observed_at, event_id); CREATE TABLE sqb_development_locations ( project_path TEXT NOT NULL, @@ -381,7 +372,6 @@ CREATE TABLE sqb_role_assignments ( target TEXT, active_key TEXT NOT NULL UNIQUE ); -CREATE INDEX sqb_role_assignments_location ON sqb_role_assignments(location_id); CREATE INDEX sqb_role_assignments_principal ON sqb_role_assignments(principal_id); CREATE TABLE sqb_person_groups ( principal_id {uuid} NOT NULL REFERENCES sqb_principals(principal_id), diff --git a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs new file mode 100644 index 0000000000..bc192f33ba --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs @@ -0,0 +1,7 @@ +//! Row readers shared by the control-store schema contracts. + +use crate::store::types::Row; + +pub(crate) fn text(row: &Row, key: &str) -> String { + row[key].as_str().expect("text column").to_owned() +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs new file mode 100644 index 0000000000..dd9741305f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs @@ -0,0 +1,10 @@ +//! Control-store schema contracts for SQLite and PostgreSQL. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_foreign_key_indexes.rs"] +mod test_foreign_key_indexes; +#[path = "test_store_upgrade.rs"] +mod test_store_upgrade; +#[path = "test_types.rs"] +pub(crate) mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs index a12ddec4bd..96c5c50768 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_foreign_key_indexes.rs @@ -7,9 +7,10 @@ use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; use crate::store::main::query::query; use crate::store::models::TransactionMode; +use crate::store::tests::schema::helpers::text; +use crate::store::tests::schema::test_types::ForeignKeyIndexTestCase; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; -use crate::store::types::Row; const FOREIGN_KEYS: &str = "SELECT m.name AS table_name, f.id AS key_id, f.\"from\" AS column_name \ FROM sqlite_master m JOIN pragma_foreign_key_list(m.name) f WHERE m.type = 'table'"; @@ -18,63 +19,68 @@ const INDEX_COLUMNS: &str = "SELECT m.name AS table_name, l.name AS index_name, FROM sqlite_master m JOIN pragma_index_list(m.name) l JOIN pragma_index_info(l.name) i \ WHERE m.type = 'table' ORDER BY m.name, l.name, i.seqno"; -fn text(row: &Row, key: &str) -> String { - row[key].as_str().expect("text column").to_owned() -} - #[test] fn given_control_store_schema_when_listing_foreign_keys_then_each_is_indexed() { - run(async { - let fixture = open_test_store(Backend::Sqlite) - .await - .expect("open SQLite store"); - migrate_store(&fixture.store).await.expect("migrate store"); - let transaction = begin(&fixture.store, TransactionMode::Read) - .await - .expect("begin read"); - let (transaction, keys) = query(transaction, FOREIGN_KEYS, vec![]) - .await - .expect("list foreign keys"); - let (transaction, columns) = query(transaction, INDEX_COLUMNS, vec![]) - .await - .expect("list index columns"); - commit(transaction).await.expect("finish read"); + let test_cases = [ForeignKeyIndexTestCase { + description: "SQLite schema", + backend: Backend::Sqlite, + expected_uncovered: Vec::new(), + }]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + migrate_store(&fixture.store).await.expect("migrate store"); + let transaction = begin(&fixture.store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, keys) = query(transaction, FOREIGN_KEYS, vec![]) + .await + .expect("list foreign keys"); + let (transaction, columns) = query(transaction, INDEX_COLUMNS, vec![]) + .await + .expect("list index columns"); + commit(transaction).await.expect("finish read"); - let mut foreign_keys: BTreeMap<(String, i64), BTreeSet> = BTreeMap::new(); - for row in &keys { - foreign_keys - .entry(( - text(row, "table_name"), - row["key_id"].as_i64().expect("key id"), - )) - .or_default() - .insert(text(row, "column_name")); - } - let mut indexes: BTreeMap<(String, String), (bool, Vec)> = BTreeMap::new(); - for row in &columns { - let entry = indexes - .entry((text(row, "table_name"), text(row, "index_name"))) - .or_insert((row["is_unique"].as_i64() == Some(1), Vec::new())); - entry.1.push(text(row, "column_name")); - } - let uncovered: Vec<(String, BTreeSet)> = foreign_keys - .into_iter() - .filter(|((table, _), referencing)| { - !indexes.iter().any(|((index_table, _), (unique, indexed))| { - index_table == table - && (indexed.len() >= referencing.len() - && indexed[..referencing.len()] - .iter() - .all(|column| referencing.contains(column)) - || *unique && indexed.iter().all(|column| referencing.contains(column))) + let mut foreign_keys: BTreeMap<(String, i64), BTreeSet> = BTreeMap::new(); + for row in &keys { + foreign_keys + .entry(( + text(row, "table_name"), + row["key_id"].as_i64().expect("key id"), + )) + .or_default() + .insert(text(row, "column_name")); + } + let mut indexes: BTreeMap<(String, String), (bool, Vec)> = BTreeMap::new(); + for row in &columns { + let entry = indexes + .entry((text(row, "table_name"), text(row, "index_name"))) + .or_insert((row["is_unique"].as_i64() == Some(1), Vec::new())); + entry.1.push(text(row, "column_name")); + } + let uncovered: Vec<(String, BTreeSet)> = foreign_keys + .into_iter() + .filter(|((table, _), referencing)| { + !indexes.iter().any(|((index_table, _), (unique, indexed))| { + index_table == table + && (indexed.len() >= referencing.len() + && indexed[..referencing.len()] + .iter() + .all(|column| referencing.contains(column)) + || *unique + && indexed.iter().all(|column| referencing.contains(column))) + }) }) - }) - .map(|((table, _), referencing)| (table, referencing)) - .collect(); - assert!( - uncovered.is_empty(), - "foreign keys without an index on their referencing columns: {uncovered:?}" - ); - close_test_store(fixture).await; - }); + .map(|((table, _), referencing)| (table, referencing)) + .collect(); + assert_eq!( + uncovered, test_case.expected_uncovered, + "{}: foreign keys without an index on their referencing columns", + test_case.description + ); + close_test_store(fixture).await; + }); + } } diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_store_upgrade.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_store_upgrade.rs new file mode 100644 index 0000000000..cffb7e9bac --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_store_upgrade.rs @@ -0,0 +1,108 @@ +//! Opening a store created before the foreign-key indexes adds them in place and keeps history. + +use std::collections::BTreeSet; + +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::TransactionMode; +use crate::store::tests::schema::helpers::text; +use crate::store::tests::schema::test_types::StoreUpgradeTestCase; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +const FOREIGN_KEY_INDEXES: [&str; 10] = [ + "sqb_jobs_snapshot", + "sqb_location_work_session", + "sqb_location_work_snapshot", + "sqb_role_assignments_location", + "sqb_run_requests_run", + "sqb_runs_rerun_of", + "sqb_runs_trigger", + "sqb_triggers_removed", + "sqb_triggers_snapshot", + "sqb_warehouse_fingerprints_text", +]; + +#[test] +fn given_store_from_previous_schema_when_opening_then_indexes_are_added_and_history_is_kept() { + let test_cases = [ + StoreUpgradeTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + list_indexes: "SELECT name FROM sqlite_master WHERE type = 'index'", + expected_indexes: FOREIGN_KEY_INDEXES.to_vec(), + }, + StoreUpgradeTestCase { + description: "PostgreSQL store", + backend: Backend::Postgres, + list_indexes: "SELECT indexname AS name FROM pg_indexes \ + WHERE schemaname = current_schema()", + expected_indexes: FOREIGN_KEY_INDEXES.to_vec(), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("create store"); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("begin downgrade"); + for index in FOREIGN_KEY_INDEXES { + (transaction, _) = query(transaction, &format!("DROP INDEX {index}"), vec![]) + .await + .expect("drop index to recreate the previous schema"); + } + let (transaction, before) = query( + transaction, + "SELECT installation_id FROM sqb_installation", + vec![], + ) + .await + .expect("read installation before upgrade"); + commit(transaction).await.expect("commit downgrade"); + + migrate_store(store) + .await + .expect("open store from the previous schema"); + + let transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, after) = query( + transaction, + "SELECT installation_id FROM sqb_installation", + vec![], + ) + .await + .expect("read installation after upgrade"); + let (transaction, indexes) = query(transaction, test_case.list_indexes, vec![]) + .await + .expect("list indexes"); + commit(transaction).await.expect("finish read"); + let present: BTreeSet = indexes.iter().map(|row| text(row, "name")).collect(); + let added: Vec<&str> = test_case + .expected_indexes + .iter() + .copied() + .filter(|index| present.contains(*index)) + .collect(); + assert_eq!( + added, test_case.expected_indexes, + "{}: indexes after upgrade", + test_case.description + ); + assert_eq!( + (before.len(), &before), + (1, &after), + "{}: installation history changed", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs new file mode 100644 index 0000000000..748d56da48 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs @@ -0,0 +1,18 @@ +//! Expected control-store schema outcomes for both supported databases. + +use std::collections::BTreeSet; + +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct ForeignKeyIndexTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_uncovered: Vec<(String, BTreeSet)>, +} + +pub(crate) struct StoreUpgradeTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) list_indexes: &'static str, + pub(crate) expected_indexes: Vec<&'static str>, +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs index ebf0f584dc..b2b30bae97 100644 --- a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -12,14 +12,14 @@ pub(crate) mod lease_helpers; pub(crate) mod logical_state; #[path = "replicas/helpers.rs"] pub(crate) mod replica_helpers; +#[path = "schema/test_contracts.rs"] +mod schema; #[path = "helpers.rs"] pub(crate) mod store_helpers; #[path = "test_types.rs"] pub(crate) mod store_test_types; #[path = "recall/test_contract.rs"] mod test_contract; -#[path = "schema/test_foreign_key_indexes.rs"] -mod test_foreign_key_indexes; #[path = "recall/test_types.rs"] mod test_types; #[path = "recall/test_views.rs"] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 8aa416d540..9c3105360b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -14,13 +14,16 @@ use sqlbuild_core::load_command::models::{LoadEntry, LoadGraph}; use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; +use crate::bindings::_helpers::compile_attachment::effective_config::{ + build_effective_connection_config, build_effective_runtime_config, +}; use crate::bindings::_helpers::compile_attachment::target::{ connection_is_default_target, project_location, }; use crate::bindings::_helpers::compile_driver::targets::managed_write_schema; use crate::bindings::_helpers::contracts::spec_resolution::loader_destination_parts; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record_with; @@ -285,9 +288,25 @@ pub(crate) fn loader_defaults<'py>( connection_value.into_iter().filter(|_| inline).collect(), ) }; + let values: Bound<'py, PyDict> = PyDict::new(py); + values.set_item("database", location("database", connection_database)?)?; + values.set_item( + "schema", + location("schema", non_empty_text(connection.get_item("schema")?)?)?, + )?; + let runtime: Bound<'py, PyTuple> = build_effective_runtime_config( + discovered, + Some(selected_target.clone()).filter(|value| !value.is_none()), + Some(cli_vars.clone()).filter(|value| !value.is_none()), + )?; + let resolved: Bound<'py, PyAny> = early_model_templates( + values.as_any(), + &runtime.get_item(1)?, + (&runtime.get_item(0)?, &runtime.get_item(2)?), + )?; Ok(LoaderDefaults { - database: location("database", connection_database)?, - schema: location("schema", non_empty_text(connection.get_item("schema")?)?)?, + database: resolved.get_item("database")?, + schema: resolved.get_item("schema")?, }) } diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 30354664b9..b4088cf7d8 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -134,6 +134,7 @@ DELTA_RELATION_FORMAT, DESCRIBE_DETAIL_ROW_WIDTH, DETAILED_TABLE_INFORMATION_MARKER, + MERGE_OPERATION_COLUMN, NON_ROW_RESULT_COLUMN_NAMES, RELATION_NOT_FOUND_ERROR_CLASSES, SERVER_OPERATION_ERROR_CLASS_NAME, @@ -1775,6 +1776,12 @@ def render_delete_insert( unique_key: tuple[str, ...], columns: tuple[str, ...] | None = None, ) -> tuple[str, ...]: + if columns is not None: + return ( + self._render_keyed_replace_merge( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) key_condition: str = " AND ".join( f"{destination}.{self.render_identifier(key)} = __source.{self.render_identifier(key)}" for key in unique_key @@ -1825,6 +1832,20 @@ def render_delete_insert_cursor( replace_where: str = ( f"{quoted_cursor_column} >= {start_bound} AND {quoted_cursor_column} < {end_bound}" ) + if unique_key and columns is not None: + window: str = ( + f"__target.{quoted_cursor_column} >= {start_bound} " + f"AND __target.{quoted_cursor_column} < {end_bound}" + ) + return ( + self._render_keyed_replace_merge( + destination=destination, + sql=sql, + unique_key=unique_key, + columns=columns, + window=window, + ), + ) if unique_key: key_delete, *insert_statements = self.render_delete_insert( destination=destination, sql=sql, unique_key=unique_key, columns=columns @@ -1836,6 +1857,44 @@ def render_delete_insert_cursor( ) return (f"INSERT INTO {destination}{column_list} REPLACE WHERE {replace_where} {sql}",) + def _render_keyed_replace_merge( + self, + *, + destination: str, + sql: str, + unique_key: tuple[str, ...], + columns: tuple[str, ...], + window: str | None = None, + ) -> str: + """One MERGE deleting the window and the delta's keys, once each, then inserting every row.""" + + quoted: tuple[str, ...] = tuple(self.render_identifier(column) for column in columns) + keys: frozenset[str] = frozenset(key.lower() for key in unique_key) + key_values: str = ", ".join( + self.render_identifier(column) if column.lower() in keys else "NULL" + for column in columns + ) + key_condition: str = " AND ".join( + f"__target.{self.render_identifier(key)} = __source.{self.render_identifier(key)}" + for key in unique_key + ) + window_clause: str = ( + "" if window is None else f" WHEN NOT MATCHED BY SOURCE AND {window} THEN DELETE" + ) + return ( + f"MERGE INTO {destination} AS __target USING (" + f"SELECT {', '.join(quoted)}, 'insert' AS {MERGE_OPERATION_COLUMN} " + f"FROM ({sql}) AS __delta UNION ALL " + f"SELECT DISTINCT {key_values}, 'delete' AS {MERGE_OPERATION_COLUMN} " + f"FROM ({sql}) AS __delta) AS __source " + f"ON __source.{MERGE_OPERATION_COLUMN} = 'delete' AND {key_condition} " + "WHEN MATCHED THEN DELETE " + f"WHEN NOT MATCHED AND __source.{MERGE_OPERATION_COLUMN} = 'insert' THEN INSERT " + f"({', '.join(quoted)}) VALUES " + f"({', '.join(f'__source.{column}' for column in quoted)})" + f"{window_clause}" + ) + def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER TABLE {origin} RENAME TO {destination}",) diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index e90a625426..fd7fa393f3 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -15,6 +15,7 @@ ) UNDEFINED_TABLE_SQLSTATE: str = "42P01" SERVER_OPERATION_ERROR_CLASS_NAME: str = "ServerOperationError" +MERGE_OPERATION_COLUMN: str = "__sqlbuild_op" DETAILED_TABLE_INFORMATION_MARKER: str = "# detailed table information" DESCRIBE_DETAIL_ROW_WIDTH: int = 2 TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 4780bbf52e..59199946b8 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -1565,8 +1565,7 @@ def delete_insert( destination=destination, sql=sql, unique_key=keys, columns=columns ) statement_recorder.record_many(statements) - with self.transaction(connection): - return self._inserted_rows(connection=connection, statements=statements) + return self._replace_staged(connection=connection, statements=statements) def delete_insert_cursor( self, @@ -1582,8 +1581,6 @@ def delete_insert_cursor( cursor_type: str | None = None, unique_key: tuple[str, ...] = (), ) -> int | None: - """Replace a cursor window; staging DDL runs outside the transaction, which DDL commits.""" - statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, sql=sql, @@ -1595,6 +1592,11 @@ def delete_insert_cursor( unique_key=unique_key, ) statement_recorder.record_many(statements) + return self._replace_staged(connection=connection, statements=statements) + + def _replace_staged(self, *, connection: Any, statements: tuple[str, ...]) -> int | None: + """Run deletes and inserts in one transaction; staging DDL, which commits, runs outside.""" + staged: tuple[str, ...] = tuple(s for s in statements if s.startswith("CREATE ")) dropped: tuple[str, ...] = tuple(s for s in statements if s.startswith("DROP ")) for statement in staged: diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index d43d91fcbc..e849504e9a 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -209,3 +209,11 @@ class DatabricksFreshnessBatchErrorTestCase: description: str batch_error: Exception expected_match: str + + +@dataclass(frozen=True) +class DatabricksKeyedDeleteInsertTestCase: + description: str + unique_key: tuple[str, ...] + columns: tuple[str, ...] + expected_statements: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py index 1dae5347e6..c3b604ba5d 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py @@ -133,3 +133,16 @@ def read_databricks_usage( return DatabricksAdapter().relation_usage( connection=connection, relations=relations, window_seconds=window_seconds ) + + +class RecordingDatabricksAdapter(DatabricksAdapter): + """Databricks adapter that records each statement instead of running it.""" + + def __init__(self) -> None: + super().__init__() + self.executed: list[str] = [] + + def execute(self, *, connection: Any, sql: str, **kwargs: Any) -> Any: + del connection, kwargs + self.executed.append(sql) + return None diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index 4e53389c01..3f02cd4442 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import FunctionArgument, FunctionReturnColumn, InferredNullability +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ExpressionInferenceProfile, RelationInfo, @@ -20,6 +21,7 @@ DatabricksDeltaDetailAgeTestCase, DatabricksExpressionInferenceProfileTestCase, DatabricksFreshnessBatchErrorTestCase, + DatabricksKeyedDeleteInsertTestCase, DatabricksMergeExclusionTestCase, DatabricksPruneSqlTestCase, DatabricksPythonFunctionSupportTestCase, @@ -38,6 +40,7 @@ FailingDatabricksMetadataCursor, FakeDatabricksMetadataConnection, FakeDatabricksMetadataCursor, + RecordingDatabricksAdapter, RequestError, ServerOperationError, ) @@ -356,21 +359,26 @@ def test_given_request_error_in_freshness_batch_when_getting_metadata_then_it_is ), ), DatabricksRenderDeleteInsertCursorTestCase( - description="keyed cursor deletes the window and the delta keys in one statement", + description="keyed cursor replaces the window and the delta keys in one merge", target="`workspace`.`test`.`orders`", sql="SELECT id, status FROM `workspace`.`test`.`orders__delta`", cursor_column="id", cursor_start="1", cursor_end="10", columns=("id", "status"), - unique_key=("id",), + unique_key=("ID",), expected_statements=( - "DELETE FROM `workspace`.`test`.`orders` WHERE (`id` >= 1 AND `id` < 10) " - "OR EXISTS (SELECT 1 FROM (SELECT id, status FROM " - "`workspace`.`test`.`orders__delta`) AS __source " - "WHERE `workspace`.`test`.`orders`.`id` = __source.`id`)", - "INSERT INTO `workspace`.`test`.`orders` (`id`, `status`) " - "SELECT id, status FROM `workspace`.`test`.`orders__delta`", + "MERGE INTO `workspace`.`test`.`orders` AS __target USING (" + "SELECT `id`, `status`, 'insert' AS __sqlbuild_op FROM " + "(SELECT id, status FROM `workspace`.`test`.`orders__delta`) AS __delta " + "UNION ALL SELECT DISTINCT `id`, NULL, 'delete' AS __sqlbuild_op FROM " + "(SELECT id, status FROM `workspace`.`test`.`orders__delta`) AS __delta) " + "AS __source ON __source.__sqlbuild_op = 'delete' " + "AND __target.`ID` = __source.`ID` WHEN MATCHED THEN DELETE " + "WHEN NOT MATCHED AND __source.__sqlbuild_op = 'insert' THEN INSERT " + "(`id`, `status`) VALUES (__source.`id`, __source.`status`) " + "WHEN NOT MATCHED BY SOURCE AND __target.`id` >= 1 AND __target.`id` < 10 " + "THEN DELETE", ), ), ], @@ -847,3 +855,42 @@ def test_given_databricks_schema_when_checking_existence_then_current_catalog_is assert exists is test_case.expected_exists, test_case.description assert cursor.executed_sql is not None assert test_case.expected_query_fragment in cursor.executed_sql, test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksKeyedDeleteInsertTestCase( + description="keyed delete_insert runs one merge, so no failure leaves rows deleted", + unique_key=("order_id",), + columns=("order_id", "status"), + expected_statements=( + "MERGE INTO `main`.`marts`.`orders` AS __target USING (" + "SELECT `order_id`, `status`, 'insert' AS __sqlbuild_op FROM " + "(SELECT order_id, status FROM delta) AS __delta UNION ALL " + "SELECT DISTINCT `order_id`, NULL, 'delete' AS __sqlbuild_op FROM " + "(SELECT order_id, status FROM delta) AS __delta) AS __source " + "ON __source.__sqlbuild_op = 'delete' " + "AND __target.`order_id` = __source.`order_id` WHEN MATCHED THEN DELETE " + "WHEN NOT MATCHED AND __source.__sqlbuild_op = 'insert' THEN INSERT " + "(`order_id`, `status`) VALUES (__source.`order_id`, __source.`status`)", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_keyed_delete_insert_when_writing_then_databricks_runs_one_merge( + test_case: DatabricksKeyedDeleteInsertTestCase, +) -> None: + adapter: RecordingDatabricksAdapter = RecordingDatabricksAdapter() + + adapter.delete_insert( + connection=object(), + destination="`main`.`marts`.`orders`", + sql="SELECT order_id, status FROM delta", + unique_key=test_case.unique_key, + columns=test_case.columns, + statement_recorder=StatementRecorder(), + ) + + assert tuple(adapter.executed) == test_case.expected_statements diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index a984ec19a2..a336d3e62d 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -434,3 +434,10 @@ class SnowflakeKeyedCursorDeleteInsertTestCase: unique_key: tuple[str, ...] expected_prefixes: tuple[str, ...] expected_inserted: int + + +@dataclass(frozen=True) +class SnowflakeKeyedDeleteInsertFailureTestCase: + description: str + failing_keyword: str + expected_prefixes: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index 8b15802d82..f6f14eecfc 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -3,12 +3,12 @@ from __future__ import annotations import sys -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import nullcontext from dataclasses import dataclass from io import StringIO from itertools import cycle, islice -from types import ModuleType, SimpleNamespace +from types import MappingProxyType, ModuleType, SimpleNamespace from typing import Any from sqlbuild._native import NativeProgressProjector @@ -687,11 +687,25 @@ def prefix_listing_statements(warehouse: RecordingSnowflakeWarehouse) -> tuple[s class RecordingSnowflakeAdapter(SnowflakeAdapter): """Snowflake adapter that records each statement instead of running it.""" - def __init__(self) -> None: + def __init__( + self, *, failures: Mapping[str, Callable[[], None]] = MappingProxyType({}) + ) -> None: super().__init__() self.executed: list[str] = [] + self.failures: Mapping[str, Callable[[], None]] = failures def execute(self, *, connection: Any, sql: str, **kwargs: Any) -> Any: del connection, kwargs self.executed.append(sql) + self.failures.get(sql.split(" ", 1)[0], _succeed)() return SimpleNamespace(rowcount=3) + + +def _succeed() -> None: + """A statement that runs without error.""" + + +def fail_statement() -> None: + """A statement the warehouse rejects.""" + + raise RuntimeError("statement failed") diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py index ca2daab813..c012fae3ba 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py @@ -7,9 +7,11 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeKeyedCursorDeleteInsertTestCase, + SnowflakeKeyedDeleteInsertFailureTestCase, ) from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( RecordingSnowflakeAdapter, + fail_statement, ) @@ -66,3 +68,43 @@ def test_given_cursor_window_when_replacing_then_snowflake_commits_deletes_with_ for statement, prefix in zip(adapter.executed, test_case.expected_prefixes, strict=True) ) assert inserted == test_case.expected_inserted + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeKeyedDeleteInsertFailureTestCase( + description="a failing insert rolls the key delete back", + failing_keyword="INSERT", + expected_prefixes=( + "CREATE", + "BEGIN", + "DELETE FROM ANALYTICS.MARTS.ORDERS USING", + "INSERT", + "ROLLBACK", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_failure_after_delete_when_keyed_delete_insert_then_snowflake_rolls_back( + test_case: SnowflakeKeyedDeleteInsertFailureTestCase, +) -> None: + adapter: RecordingSnowflakeAdapter = RecordingSnowflakeAdapter( + failures={test_case.failing_keyword: fail_statement} + ) + + with pytest.raises(RuntimeError, match="statement failed"): + adapter.delete_insert( + connection=object(), + destination="ANALYTICS.MARTS.ORDERS", + sql="SELECT * FROM ANALYTICS.MARTS.ORDERS__DELTA", + unique_key=("order_id",), + statement_recorder=StatementRecorder(), + ) + + assert len(adapter.executed) == len(test_case.expected_prefixes) + assert all( + statement.startswith(prefix) + for statement, prefix in zip(adapter.executed, test_case.expected_prefixes, strict=True) + ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py index 53e20809fd..da0338fa98 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py @@ -31,3 +31,5 @@ class LoaderDefaultResolutionTestCase: expected_defaults: tuple[str | None, str | None] project_defaults: tuple[str | None, str | None] = (None, None) connections: dict[str, dict[str, object]] = field(default_factory=dict) + project_vars: dict[str, str] = field(default_factory=dict) + cli_vars: dict[str, object] | None = None diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index 3bfc6126cc..8ff72a5fa5 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -167,6 +167,23 @@ def test_given_intermediate_loader_when_selecting_then_parses_destination_with_e project_defaults=("DEFAULTS_DB", "DEFAULTS_SCHEMA"), expected_defaults=("DEFAULTS_DB", "DEFAULTS_SCHEMA"), ), + LoaderDefaultResolutionTestCase( + description="templated project defaults expand project vars as models do", + adapter="postgres", + connection={"database": "connection_db", "schema": "connection_schema"}, + project_defaults=("${warehouse_db}", "orders_${environment}"), + project_vars={"warehouse_db": "analytics", "environment": "dev"}, + expected_defaults=("analytics", "orders_dev"), + ), + LoaderDefaultResolutionTestCase( + description="command-line vars override project vars in templated defaults", + adapter="postgres", + connection={"database": "connection_db", "schema": "connection_schema"}, + project_defaults=(None, "orders_${environment}"), + project_vars={"environment": "dev"}, + cli_vars={"environment": "ci"}, + expected_defaults=("connection_db", "orders_ci"), + ), ), ids=lambda case: case.description, ) @@ -182,6 +199,7 @@ def test_given_connection_namespace_when_resolving_load_defaults_then_matches_co defaults=DefaultsConfig( database=test_case.project_defaults[0], schema=test_case.project_defaults[1] ), + vars=test_case.project_vars, ), local_config=LocalConfig(), ) @@ -190,7 +208,7 @@ def test_given_connection_namespace_when_resolving_load_defaults_then_matches_co discovered_inputs=discovered_inputs, selected_target=None, target_config=None, - cli_vars=None, + cli_vars=test_case.cli_vars, ) assert defaults == test_case.expected_defaults From 329044b80f1d89ade58164af4a761a25b3bfb615 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:45:46 +0000 Subject: [PATCH 03/18] style: shorten the Databricks keyed MERGE docstring --- src/sqlbuild/adapters/databricks/classes/databricks_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index b4088cf7d8..af1bd65b96 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -1866,7 +1866,7 @@ def _render_keyed_replace_merge( columns: tuple[str, ...], window: str | None = None, ) -> str: - """One MERGE deleting the window and the delta's keys, once each, then inserting every row.""" + """One MERGE deleting the window and delta keys, once each, then inserting every row.""" quoted: tuple[str, ...] = tuple(self.render_identifier(column) for column in columns) keys: frozenset[str] = frozenset(key.lower() for key in unique_key) From 03f1605ee13db3176c3a3a5cc24c32e0e89aa49d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:45:56 +0000 Subject: [PATCH 04/18] fix: stop a loader from writing to a schema named preserve --- .../src/assembly/target_checks/errors.rs | 2 + .../main/target_error_message.rs | 5 ++ .../target_checks/tests/test_target_checks.rs | 5 ++ .../_helpers/compile_driver/targets.rs | 48 +++++++++++++++++- .../bindings/_helpers/load/load_selection.rs | 20 +++++++- .../docs/concepts/python-nodes/loaders.md | 5 ++ .../compile/test_missing_target_database.py | 49 +++++++++++++++++++ .../docs/concepts/python-nodes/loaders.mdx | 5 ++ 8 files changed, 136 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs index 9f066da682..9c1dcf2dc0 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs @@ -13,4 +13,6 @@ pub enum TargetError { ManagedWriteSchema(String), /// A named target without an explicit schema. NamedTargetSchema(String), + /// An intermediate loader, by name, whose schema resolves to the target's `preserve`. + PreservedLoaderSchema(String), } diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs index 1adc077120..2c4cc04e1c 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs @@ -42,5 +42,10 @@ pub fn target_error_message(error: &TargetError) -> String { '{PRESERVE_TARGET_VALUE}'. Resource schemas, defaults, and connection schemas do \ not select a named target schema strategy." ), + TargetError::PreservedLoaderSchema(loader) => format!( + "Intermediate loader '{loader}' would write to a schema named \ + '{PRESERVE_TARGET_VALUE}'. The target's schema '{PRESERVE_TARGET_VALUE}' keeps each \ + model's own schema, and a loader has none, so its destination must name a schema." + ), } } diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs index e7a26a098f..5fcdd06a52 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs @@ -53,6 +53,11 @@ fn given_target_error_when_formatting_then_matches_python_message() { error: TargetError::NamedTargetSchema("ci".to_owned()), expected_message: "Named target 'ci' must explicitly set schema to a nonblank literal or 'preserve'. Resource schemas, defaults, and connection schemas do not select a named target schema strategy.", }, + MessageTestCase { + description: "an intermediate loader under a preserved target schema", + error: TargetError::PreservedLoaderSchema("raw_orders".to_owned()), + expected_message: "Intermediate loader 'raw_orders' would write to a schema named 'preserve'. The target's schema 'preserve' keeps each model's own schema, and a loader has none, so its destination must name a schema.", + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index bba27d6475..f9565fae71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -10,7 +10,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::target_checks::constants::{ LOADER_COLLISION_CODE, MANAGED_WRITE_SCHEMA_CODE, MISSING_TARGET_PARTS_CODE, - NAMED_TARGET_SCHEMA_CODE, + NAMED_TARGET_SCHEMA_CODE, PRESERVE_TARGET_VALUE, }; use sqlbuild_analysis::assembly::target_checks::errors::TargetError; use sqlbuild_analysis::assembly::target_checks::main::explicit_write_schema::explicit_write_schema; @@ -554,6 +554,28 @@ pub(crate) fn managed_write_schemas( adapter: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, ) -> PyResult<()> { + let target_schema: Option = + text_or_none(&native_field(project, "effective_target_schema")?)?; + for loader in native_field(project, "loader_functions")?.try_iter()? { + let loader: Bound<'_, PyAny> = loader?; + let name: String = native_field(&loader, "name")?.str()?.to_cow()?.into_owned(); + let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; + if loader_schema(destination.as_deref(), target_schema.as_deref()) + .is_some_and(|schema| python_strip(schema) == PRESERVE_TARGET_VALUE) + { + let table: String = destination.as_deref().map_or_else( + || format!("__loader__{name}"), + |destination| { + destination + .rsplit('.') + .next() + .unwrap_or(destination) + .to_owned() + }, + ); + preserved_loader_schema(loader.py(), &name, &table)?; + } + } let ignored: Option = ignored_connection_value(project, "schema")?; if ignored.is_none() && native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? @@ -627,6 +649,30 @@ pub(crate) fn managed_write_schemas( Ok(()) } +/// Raise when loader `name` would write into a schema literally named `preserve`. +pub(crate) fn preserved_loader_schema(py: Python<'_>, name: &str, table: &str) -> PyResult<()> { + Err(planner_error( + py, + target_error_message(&TargetError::PreservedLoaderSchema(name.to_owned())), + MANAGED_WRITE_SCHEMA_CODE, + Some(format!( + "to choose where it writes, set its destination in its @loader decorator:\n \ + @loader(destination=\"my_schema.{table}\")" + )), + )) +} + +/// The schema a loader destination names, or `target_schema` when it names none. +fn loader_schema<'a>( + destination: Option<&'a str>, + target_schema: Option<&'a str>, +) -> Option<&'a str> { + match destination.map(|destination| destination.rsplit('.').collect::>()) { + Some(parts) if parts.len() > 1 => Some(parts[1]), + _ => target_schema, + } +} + /// Raise when neither the resource nor the project names a physical write schema. pub(crate) fn managed_write_schema( resource_label: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 9c3105360b..d34f0f21a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_core::load_command::errors::UnknownLoadSelector; use sqlbuild_core::load_command::main::load_reference_loaders::load_reference_loaders; use sqlbuild_core::load_command::main::select_load_entries::select_load_entries as select_entries; @@ -20,7 +21,9 @@ use crate::bindings::_helpers::compile_attachment::effective_config::{ use crate::bindings::_helpers::compile_attachment::target::{ connection_is_default_target, project_location, }; -use crate::bindings::_helpers::compile_driver::targets::managed_write_schema; +use crate::bindings::_helpers::compile_driver::targets::{ + managed_write_schema, preserved_loader_schema, +}; use crate::bindings::_helpers::contracts::spec_resolution::loader_destination_parts; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::model_config::template_data::early_model_templates; @@ -310,12 +313,25 @@ pub(crate) fn loader_defaults<'py>( }) } -/// Raise when a selected loader has no schema because the connection's schema is not used. +/// Raise when a selected loader would write to `preserve` or lost its schema to an ignored connection. pub(crate) fn loader_write_schemas<'py>( discovered: &Bound<'py, PyAny>, (selected_target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), entries: &[Bound<'py, PyAny>], ) -> PyResult<()> { + for entry in entries { + if native_field(entry, "loader")?.is_none() { + continue; + } + let schema: Option = native_field(entry, "schema")?.extract()?; + if schema.is_some_and(|schema| schema.trim() == PRESERVE_TARGET_VALUE) { + preserved_loader_schema( + entry.py(), + &native_field(entry, "name")?.str()?.to_string(), + &native_field(entry, "table")?.str()?.to_string(), + )?; + } + } if connection_is_default_target(discovered)? { return Ok(()); } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index cd462648a0..9d4aca5f7e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -329,6 +329,11 @@ def raw_accounts(ctx: LoaderContext): ... ``` +An intermediate loader without a schema in its `destination` writes to the target's schema. When +the target's schema is `preserve`, which keeps each model's own schema, the loader has no schema to +keep, so `compile`, `build` and `load` stop with `S103` until its `destination` names one, such as +`destination="staging.__loader__raw_accounts"`. + ## Decorator parameters The `@loader` decorator accepts optional parameters that can also be set in the source YAML. When both are specified, the YAML takes precedence. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 088defa40c..895978fe6a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -133,6 +133,55 @@ expected_example_count=1, command=("--no-color", "load", "--select", "raw_orders"), ), + MissingTargetDatabaseTestCase( + description="a loader under a preserved target schema stops compile", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[defaults]\nschema = "analytics"\n\n[targets.dev]\nschema = "preserve"\n' + ), + extra_files={ + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error=( + "error[S103]: Intermediate loader 'raw_orders' would write to a schema named " + "'preserve'. The target's schema 'preserve' keeps each model's own schema, and a " + "loader has none, so its destination must name a schema.\n" + " = help: to choose where it writes, set its destination in its @loader " + "decorator:\n" + ' @loader(destination="my_schema.__loader__raw_orders")\n' + ), + expected_example_count=0, + ), + MissingTargetDatabaseTestCase( + description="sqb load stops a loader under a preserved target schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[defaults]\nschema = "analytics"\n\n[targets.dev]\nschema = "preserve"\n' + ), + extra_files={ + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error=( + "error[S103]: Intermediate loader 'raw_orders' would write to a schema named " + "'preserve'. The target's schema 'preserve' keeps each model's own schema, and a " + "loader has none, so its destination must name a schema.\n" + " = help: to choose where it writes, set its destination in its @loader " + "decorator:\n" + ' @loader(destination="my_schema.__loader__raw_orders")\n' + ), + expected_example_count=0, + command=("--no-color", "load", "--select", "raw_orders"), + ), ), ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 9719cc75c5..02bb37969d 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -309,6 +309,11 @@ def raw_accounts(ctx: LoaderContext): ... ``` +An intermediate loader without a schema in its `destination` writes to the target's schema. When +the target's schema is `preserve`, which keeps each model's own schema, the loader has no schema to +keep, so `compile`, `build` and `load` stop with `S103` until its `destination` names one, such as +`destination="staging.__loader__raw_accounts"`. + ## Decorator parameters The `@loader` decorator accepts optional parameters that can also be set in the source YAML. When both are specified, the YAML takes precedence. From 268e6156f2957b46dbe0eeee4e8cb78c209d5dda Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:48:19 +0000 Subject: [PATCH 05/18] fix: say a connection schema counts only for a project's only top-level connection --- .../target_checks/main/target_error_message.rs | 3 ++- .../target_checks/tests/test_target_checks.rs | 2 +- .../main/compile/test_missing_target_database.py | 14 ++++++++------ .../failure/target-missing-write-schema.json | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs index 2c4cc04e1c..ba4b45c4a6 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs @@ -35,7 +35,8 @@ pub fn target_error_message(error: &TargetError) -> String { ), TargetError::ManagedWriteSchema(label) => format!( "{label} has no explicitly resolved physical write schema. Set schema on the \ - resource or its defaults, selected target, loader_schema, or connection." + resource or its defaults, the selected target or its loader_schema, or the \ + project's only top-level [connection]." ), TargetError::NamedTargetSchema(target) => format!( "Named target '{target}' must explicitly set schema to a nonblank literal or \ diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs index 5fcdd06a52..b1fd264dac 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/tests/test_target_checks.rs @@ -46,7 +46,7 @@ fn given_target_error_when_formatting_then_matches_python_message() { MessageTestCase { description: "a managed write without a schema", error: TargetError::ManagedWriteSchema("Model 'orders'".to_owned()), - expected_message: "Model 'orders' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, selected target, loader_schema, or connection.", + expected_message: "Model 'orders' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, the selected target or its loader_schema, or the project's only top-level [connection].", }, MessageTestCase { description: "a named target without a schema", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 895978fe6a..89f8217ea1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -94,9 +94,10 @@ }, expected_error=( "error[S103]: Intermediate loader 'raw_orders' has no explicitly resolved " - "physical write schema. Set schema on the resource or its defaults, selected " - "target, loader_schema, or connection. The connection's schema 'raw' is not used " - "because the project defines named connections or targets.\n" + "physical write schema. Set schema on the resource or its defaults, the selected " + "target or its loader_schema, or the project's only top-level [connection]. The " + "connection's schema 'raw' is not used because the project defines named " + "connections or targets.\n" " = help: to set project-wide defaults, set this in sqlbuild_project.toml:\n" " [defaults]\n" ' schema = "raw"\n' @@ -123,9 +124,10 @@ }, expected_error=( "error[S103]: Intermediate loader 'raw_orders' has no explicitly resolved " - "physical write schema. Set schema on the resource or its defaults, selected " - "target, loader_schema, or connection. The connection's schema 'raw' is not used " - "because the project defines named connections or targets.\n" + "physical write schema. Set schema on the resource or its defaults, the selected " + "target or its loader_schema, or the project's only top-level [connection]. The " + "connection's schema 'raw' is not used because the project defines named " + "connections or targets.\n" " = help: to set project-wide defaults, set this in sqlbuild_project.toml:\n" " [defaults]\n" ' schema = "raw"\n' diff --git a/tests/goldens/compiler/failure/target-missing-write-schema.json b/tests/goldens/compiler/failure/target-missing-write-schema.json index dc9dd0833a..37133bf905 100644 --- a/tests/goldens/compiler/failure/target-missing-write-schema.json +++ b/tests/goldens/compiler/failure/target-missing-write-schema.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "S103", - "message": "Model 'customer_totals' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, selected target, loader_schema, or connection." + "message": "Model 'customer_totals' has no explicitly resolved physical write schema. Set schema on the resource or its defaults, the selected target or its loader_schema, or the project's only top-level [connection]." } ] } From 5d14d6722a6373ea4f35b996333d256723418e1e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:59:47 +0000 Subject: [PATCH 06/18] fix: show the S104 hint's connection schema with command-line vars applied --- .../_helpers/compile_driver/project.rs | 6 +++++- .../_helpers/compile_driver/targets.rs | 20 ++++++++++++++----- src/sqlbuild/_native.pyi | 6 +++++- .../compile/test_missing_target_database.py | 19 ++++++++++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 8c6a9ca065..cb97ef9474 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -156,7 +156,11 @@ fn python_compiled_project<'py>( ) -> PyResult> { let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); - named_target_schema_strategy(discovered, adapter, request.selected_target.bind(py))?; + named_target_schema_strategy( + discovered, + adapter, + (request.selected_target.bind(py), request.cli_vars.bind(py)), + )?; managed_loader_target_isolation(discovered, adapter, request.resolved_connection.bind(py))?; let selection: &Bound<'py, PyAny> = request.analysis_selection.bind(py); let no_cache: bool = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index f9565fae71..aa02100f6a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -157,12 +157,18 @@ fn validate_named_target_schema_strategy<'py>( py, "validate_named_target_schema_strategy", options, - (&["discovered_inputs", "adapter", "selected_target"], &[]), + ( + &["discovered_inputs", "adapter", "selected_target"], + &["cli_vars"], + ), )?; named_target_schema_strategy( &arguments.value("discovered_inputs")?, &arguments.value("adapter")?, - &arguments.value("selected_target")?, + ( + &arguments.value("selected_target")?, + &arguments.value("cli_vars")?, + ), ) } @@ -170,7 +176,7 @@ fn validate_named_target_schema_strategy<'py>( pub(crate) fn named_target_schema_strategy( discovered: &Bound<'_, PyAny>, adapter: &Bound<'_, PyAny>, - selected_target: &Bound<'_, PyAny>, + (selected_target, cli_vars): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), ) -> PyResult<()> { let py: Python<'_> = discovered.py(); if native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? { @@ -197,8 +203,12 @@ pub(crate) fn named_target_schema_strategy( } let name: String = target_name.str()?.to_cow()?.into_owned(); let mut message: String = target_error_message(&TargetError::NamedTargetSchema(name.clone())); - let connection: Bound<'_, PyAny> = - build_effective_connection_config(discovered, Some(target_name), None, false)?; + let connection: Bound<'_, PyAny> = build_effective_connection_config( + discovered, + Some(target_name), + Some(cli_vars.clone()).filter(|value| !value.is_none()), + false, + )?; let help: Option = match text_or_none(&mapping_get(&connection, "schema")?)? { Some(value) => { message.push_str(&ignored_connection_note("schema", &value)); diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 53f0f73730..b465d0857c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3663,7 +3663,11 @@ def build_discovered_project_inputs( extract_output_column_locations: bool = True, ) -> DiscoveredProjectInputs: ... def validate_named_target_schema_strategy( - *, discovered_inputs: DiscoveredProjectInputs, adapter: BaseAdapter, selected_target: str | None + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + selected_target: str | None, + cli_vars: dict[str, object] | None = None, ) -> None: ... def validate_managed_loader_target_isolation( *, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 89f8217ea1..79dcc20847 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -55,6 +55,25 @@ ), expected_example_count=1, ), + MissingTargetDatabaseTestCase( + description="the ignored connection schema is shown with command-line vars applied", + project_toml=( + 'name = "orders"\nadapter = "snowflake"\ndefault_target = "dev"\n\n' + '[vars]\norders_schema = "ORDERS"\n\n' + '[connections.warehouse]\naccount = "example"\nschema = "${orders_schema}"\n\n' + '[targets.dev]\ndatabase = "ANALYTICS"\n' + ), + expected_error=( + "The connection's schema 'ORDERS_CI' is not used because the project defines " + "named connections or targets.\n" + " = help: to build this target into that schema, set this in " + "sqlbuild_project.toml:\n" + " [targets.dev]\n" + ' schema = "ORDERS_CI"\n' + ), + expected_example_count=0, + command=("--no-color", "compile", "--vars", '{"orders_schema": "ORDERS_CI"}'), + ), MissingTargetDatabaseTestCase( description="a named connection's schema is named as unused", project_toml=( From c3a2bbea0b01fd037715a0caca0712368d6262d4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:08:34 +0000 Subject: [PATCH 07/18] fix(debug): report no database for DuckDB targets Debug passed the DuckDB connection file path as the target database, while models build in the file's own catalog. Debug now drops it for namespaceless adapters, as diff already does. --- .../_helpers/debug_command/run_debug.rs | 4 +++ .../bindings/_helpers/diff_report/query.rs | 2 +- .../cli/commands/main/debug/_test_types.py | 1 + .../cli/commands/main/debug/test_debug.py | 28 +++++++++++++------ 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 456358a009..1ebb3f2c60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -41,6 +41,7 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::record_class; @@ -298,6 +299,8 @@ fn build_report( })?; let connection_config: Bound<'_, PyDict> = effective_connection(&context, (&adapter_name, &base_connection))?.cast_into()?; + let namespace_adapter: String = adapter_name.str()?.extract()?; + let namespaceless: bool = NAMESPACELESS_ADAPTERS.contains(&namespace_adapter.as_str()); let namespace: Bound<'_, PyTuple> = build_effective_target_namespace( py, Some(keywords( @@ -309,6 +312,7 @@ fn build_report( "connection_database", connection_config .get_item("database")? + .filter(|_| !namespaceless) .unwrap_or_else(|| py.None().into_bound(py)), ), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index f9a27a370b..5c3c84438d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -35,7 +35,7 @@ use sqlbuild_analysis::diff_selection::main::raw_query_shape::raw_query_shape; use sqlbuild_analysis::diff_selection::models::RawQueryShape; const GENERIC_DIALECT: &str = "generic"; -const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; +pub(crate) const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; fn failure(py: Python<'_>, message: String, code: &str, help: Option<&str>) -> PyErr { cli_error( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/debug/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/debug/_test_types.py index dbffda67e6..6fe7801d38 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/debug/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/debug/_test_types.py @@ -9,6 +9,7 @@ class DebugCliTestCase: command: tuple[str, ...] expected_stdout_fragment: str expected_returncode: int = 0 + project_toml: str = "" @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py index d65d2d5f71..1f508ca121 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py @@ -77,25 +77,35 @@ def test_given_debug_command_when_running_then_outputs_checks( expected_stdout_fragment=( " database: DEFAULTS_DB [OK resolved]\n schema: DEFAULTS_SCHEMA [OK resolved]\n" ), + project_toml=( + 'name = "debug_defaults"\nadapter = "snowflake"\n\n' + '[connection]\naccount = "acme"\nuser = "u"\npassword = "p"\n' + 'database = "CONN_DB"\nschema = "CONN_SCHEMA"\n\n' + '[defaults]\ndatabase = "DEFAULTS_DB"\nschema = "DEFAULTS_SCHEMA"\n' + ), + ), + DebugCliTestCase( + description="a duckdb connection file is not reported as the models' database", + command=("debug", "--no-connection"), + expected_stdout_fragment=( + " database: not configured [OK resolved]\n schema: sales [OK resolved]\n" + ), + project_toml=( + 'name = "debug_defaults"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "shop.duckdb"\nschema = "sales"\n' + ), ), ], ids=lambda case: case.description, ) -def test_given_snowflake_defaults_location_when_debugging_then_reports_it( +def test_given_project_location_when_debugging_then_reports_where_models_build( tmp_path: Path, test_case: DebugCliTestCase, ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="debug_defaults", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "debug_defaults"\nadapter = "snowflake"\n\n' - '[connection]\naccount = "acme"\nuser = "u"\npassword = "p"\n' - 'database = "CONN_DB"\nschema = "CONN_SCHEMA"\n\n' - '[defaults]\ndatabase = "DEFAULTS_DB"\nschema = "DEFAULTS_SCHEMA"\n' - ), - }, + repo_files={"sqlbuild_project.toml": test_case.project_toml}, ) result: subprocess.CompletedProcess[str] = run_sqb( From 9fd9c7340b789924c4e9aafe4e049317829f7fec Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:36:50 +0000 Subject: [PATCH 08/18] fix: check only intermediate loaders under a preserved target schema A managed source's loader writes to the source's own schema or the target's loader_schema, so a target schema of preserve no longer stops it. The check now applies when the selected target's own schema is preserve and the loader's destination names no schema, the same way for compile, build and load. --- .../_helpers/compile_driver/project.rs | 20 ++- .../_helpers/compile_driver/targets.rs | 137 ++++++++++-------- .../bindings/_helpers/load/load_selection.rs | 22 ++- .../cli/commands/main/compile/_test_types.py | 4 +- .../compile/test_missing_target_database.py | 38 ++++- 5 files changed, 133 insertions(+), 88 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index cb97ef9474..d8924b4abb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -1,5 +1,7 @@ //! Compiled project and project graph construction: frontier stages, target defaults and checks. +use std::collections::HashSet; + use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -14,8 +16,9 @@ use crate::bindings::_helpers::compile_driver::target_defaults::{ TargetDefaults, apply_target_defaults, }; use crate::bindings::_helpers::compile_driver::targets::{ - managed_loader_target_isolation, managed_write_schemas, named_target_schema_strategy, - project_targets, requires_explicit_targets, + managed_loader_target_isolation, managed_source_loaders, managed_write_schemas, + named_target_schema_strategy, preserved_loader_schemas, project_targets, + requires_explicit_targets, }; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; @@ -213,6 +216,19 @@ fn targeted_project<'py>( ) -> PyResult> { let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); + let source_loaders: HashSet = managed_source_loaders(project)?; + let mut intermediate_loaders: HashSet = HashSet::new(); + for loader in native_field(discovered, "loader_functions")?.try_iter()? { + let name: String = native_field(&loader?, "name")?.extract()?; + if !source_loaders.contains(&name) { + intermediate_loaders.insert(name); + } + } + preserved_loader_schemas( + discovered, + request.selected_target.bind(py), + &intermediate_loaders, + )?; managed_write_schemas(adapter, project)?; let effective_schema: Bound<'py, PyAny> = native_field(project, "effective_target_schema")?; let default_schema: Bound<'py, PyAny> = if effective_schema.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index aa02100f6a..f24e9e7019 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -182,21 +182,10 @@ pub(crate) fn named_target_schema_strategy( if native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? { return Ok(()); } - let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; - let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; - let target_name: Bound<'_, PyAny> = resolve_target_name( - py, - &project_config.clone(), - &local_config.clone(), - &selected_target.clone(), - ) - .and_then(|value| value.into_bound_py_any(py))?; - if target_name.is_none() { + let Some((target_name, target_config)) = selected_target_config(discovered, selected_target)? + else { return Ok(()); - } - let target_config: Bound<'_, PyAny> = - resolve_target_config(py, &project_config, &local_config, &target_name.clone()) - .and_then(|value| value.into_bound_py_any(py))?; + }; let schema: Option = text_or_none(&native_field(&target_config, "schema")?)?; if schema.is_some_and(|schema| !python_strip(&schema).is_empty()) { return Ok(()); @@ -564,28 +553,6 @@ pub(crate) fn managed_write_schemas( adapter: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, ) -> PyResult<()> { - let target_schema: Option = - text_or_none(&native_field(project, "effective_target_schema")?)?; - for loader in native_field(project, "loader_functions")?.try_iter()? { - let loader: Bound<'_, PyAny> = loader?; - let name: String = native_field(&loader, "name")?.str()?.to_cow()?.into_owned(); - let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; - if loader_schema(destination.as_deref(), target_schema.as_deref()) - .is_some_and(|schema| python_strip(schema) == PRESERVE_TARGET_VALUE) - { - let table: String = destination.as_deref().map_or_else( - || format!("__loader__{name}"), - |destination| { - destination - .rsplit('.') - .next() - .unwrap_or(destination) - .to_owned() - }, - ); - preserved_loader_schema(loader.py(), &name, &table)?; - } - } let ignored: Option = ignored_connection_value(project, "schema")?; if ignored.is_none() && native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? @@ -630,15 +597,7 @@ pub(crate) fn managed_write_schemas( if ignored.is_none() { return Ok(()); } - let mut source_loaders: HashSet = HashSet::new(); - for source in native_field(project, "sources")?.try_iter()? { - if let Some(loader) = text_or_none(&native_field( - &native_field(&source?, "source_entry")?, - "loader", - )?)? { - source_loaders.insert(loader); - } - } + let source_loaders: HashSet = managed_source_loaders(project)?; for loader in native_field(project, "loader_functions")?.try_iter()? { let loader: Bound<'_, PyAny> = loader?; let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; @@ -659,28 +618,78 @@ pub(crate) fn managed_write_schemas( Ok(()) } -/// Raise when loader `name` would write into a schema literally named `preserve`. -pub(crate) fn preserved_loader_schema(py: Python<'_>, name: &str, table: &str) -> PyResult<()> { - Err(planner_error( - py, - target_error_message(&TargetError::PreservedLoaderSchema(name.to_owned())), - MANAGED_WRITE_SCHEMA_CODE, - Some(format!( - "to choose where it writes, set its destination in its @loader decorator:\n \ - @loader(destination=\"my_schema.{table}\")" - )), - )) +/// The loader names that load managed sources, which write to the source's own location. +pub(crate) fn managed_source_loaders(project: &Bound<'_, PyAny>) -> PyResult> { + let mut loaders: HashSet = HashSet::new(); + for source in native_field(project, "sources")?.try_iter()? { + if let Some(loader) = text_or_none(&native_field( + &native_field(&source?, "source_entry")?, + "loader", + )?)? { + loaders.insert(loader); + } + } + Ok(loaders) } -/// The schema a loader destination names, or `target_schema` when it names none. -fn loader_schema<'a>( - destination: Option<&'a str>, - target_schema: Option<&'a str>, -) -> Option<&'a str> { - match destination.map(|destination| destination.rsplit('.').collect::>()) { - Some(parts) if parts.len() > 1 => Some(parts[1]), - _ => target_schema, +/// Raise when the selected target's schema is `preserve` and one of `loaders` names no schema. +pub(crate) fn preserved_loader_schemas( + discovered: &Bound<'_, PyAny>, + selected_target: &Bound<'_, PyAny>, + loaders: &HashSet, +) -> PyResult<()> { + let py: Python<'_> = discovered.py(); + let Some((_, target_config)) = selected_target_config(discovered, selected_target)? else { + return Ok(()); + }; + if !text_or_none(&native_field(&target_config, "schema")?)? + .is_some_and(|schema| python_strip(&schema) == PRESERVE_TARGET_VALUE) + { + return Ok(()); } + for loader in native_field(discovered, "loader_functions")?.try_iter()? { + let loader: Bound<'_, PyAny> = loader?; + let name: String = native_field(&loader, "name")?.extract()?; + let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; + if !loaders.contains(&name) || destination.as_deref().is_some_and(|d| d.contains('.')) { + continue; + } + let table: String = destination.unwrap_or_else(|| format!("__loader__{name}")); + return Err(planner_error( + py, + target_error_message(&TargetError::PreservedLoaderSchema(name)), + MANAGED_WRITE_SCHEMA_CODE, + Some(format!( + "to choose where it writes, set its destination in its @loader decorator:\n \ + @loader(destination=\"my_schema.{table}\")" + )), + )); + } + Ok(()) +} + +/// The selected target's name and configuration, or `None` when no named target is selected. +fn selected_target_config<'py>( + discovered: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, +) -> PyResult, Bound<'py, PyAny>)>> { + let py: Python<'py> = discovered.py(); + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; + let target_name: Bound<'py, PyAny> = resolve_target_name( + py, + &project_config.clone(), + &local_config.clone(), + &selected_target.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; + if target_name.is_none() { + return Ok(None); + } + let target_config: Bound<'py, PyAny> = + resolve_target_config(py, &project_config, &local_config, &target_name.clone()) + .and_then(|value| value.into_bound_py_any(py))?; + Ok(Some((target_name, target_config))) } /// Raise when neither the resource nor the project names a physical write schema. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index d34f0f21a5..c5684505ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -1,13 +1,12 @@ //! Resolve which loaders and sources `sqb load` runs, as `SourceEntry` objects. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_core::load_command::errors::UnknownLoadSelector; use sqlbuild_core::load_command::main::load_reference_loaders::load_reference_loaders; use sqlbuild_core::load_command::main::select_load_entries::select_load_entries as select_entries; @@ -22,7 +21,7 @@ use crate::bindings::_helpers::compile_attachment::target::{ connection_is_default_target, project_location, }; use crate::bindings::_helpers::compile_driver::targets::{ - managed_write_schema, preserved_loader_schema, + managed_write_schema, preserved_loader_schemas, }; use crate::bindings::_helpers::contracts::spec_resolution::loader_destination_parts; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -319,19 +318,16 @@ pub(crate) fn loader_write_schemas<'py>( (selected_target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), entries: &[Bound<'py, PyAny>], ) -> PyResult<()> { + let mut intermediate_loaders: HashSet = HashSet::new(); for entry in entries { - if native_field(entry, "loader")?.is_none() { - continue; - } - let schema: Option = native_field(entry, "schema")?.extract()?; - if schema.is_some_and(|schema| schema.trim() == PRESERVE_TARGET_VALUE) { - preserved_loader_schema( - entry.py(), - &native_field(entry, "name")?.str()?.to_string(), - &native_field(entry, "table")?.str()?.to_string(), - )?; + if native_field(entry, "meta")? + .call_method1("get", (LOADER_NODE_KEY,))? + .is_truthy()? + { + intermediate_loaders.insert(native_field(entry, "name")?.extract()?); } } + preserved_loader_schemas(discovered, selected_target, &intermediate_loaders)?; if connection_is_default_target(discovered)? { return Ok(()); } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index a742d1f25e..7c1bbe80f5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1437,8 +1437,10 @@ class ConnectionLocationTestCase: @dataclass(frozen=True) class ManagedSourceLoaderLocationTestCase: - """A command that runs a managed source's loader and the rows its loader schema must hold.""" + """A command that runs a managed source's loader and the rows the `raw` schema must hold.""" description: str + project_toml: str + source_schema: str command: tuple[str, ...] expected_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 79dcc20847..9eaae1fba0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -17,6 +17,12 @@ ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +_PRESERVED_TARGET_PROJECT: str = ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[defaults]\nschema = "analytics"\n\n[targets.dev]\nschema = "preserve"\n' +) + @pytest.mark.parametrize( "test_case", @@ -294,32 +300,48 @@ def test_given_connection_location_when_compiling_then_models_build_there( ( ManagedSourceLoaderLocationTestCase( description="build loads the managed source into the target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n[connection]\n' + 'database = "dev.duckdb"\nschema = "landing"\n\n' + '[connections.archive]\ndatabase = "archive.duckdb"\n\n' + '[targets.dev]\nloader_schema = "raw"\n' + ), + source_schema="", + command=("--no-color", "build"), + expected_rows=((1,),), + ), + ManagedSourceLoaderLocationTestCase( + description="build under a preserved target loads the source into its own schema", + project_toml=_PRESERVED_TARGET_PROJECT, + source_schema=" schema: raw\n", command=("--no-color", "build"), expected_rows=((1,),), ), + ManagedSourceLoaderLocationTestCase( + description="load under a preserved target loads the source into its own schema", + project_toml=_PRESERVED_TARGET_PROJECT, + source_schema=" schema: raw\n", + command=("--no-color", "load", "--select", "raw_orders"), + expected_rows=((1,),), + ), ), ids=lambda case: case.description, ) -def test_given_ignored_connection_schema_when_building_managed_source_then_loader_schema_is_used( +def test_given_managed_source_when_loading_then_source_lands_in_its_resolved_schema( test_case: ManagedSourceLoaderLocationTestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="orders", repo_files={ - "sqlbuild_project.toml": ( - 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n[connection]\n' - 'database = "dev.duckdb"\nschema = "landing"\n\n' - '[connections.archive]\ndatabase = "archive.duckdb"\n\n' - '[targets.dev]\nloader_schema = "raw"\n' - ), + "sqlbuild_project.toml": test_case.project_toml, "models/orders.sql": ( "MODEL (description 'Test model orders.', materialized view, " 'schema staging);\n\nSELECT order_id FROM __source("raw_orders")\n' ), "sources/raw.yml": ( "sources:\n - name: raw_orders\n description: Raw orders.\n" - " managed: true\n write_strategy: table\n" + f" managed: true\n{test_case.source_schema} write_strategy: table\n" " columns:\n - name: order_id\n type: INTEGER\n" ), "python/loaders/orders.py": ( From b01defaea628266d5f694f33273e7b3c0ad699d6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:01:22 +0000 Subject: [PATCH 09/18] refactor: read the loader node flag without a Python call --- .../src/bindings/_helpers/load/load_selection.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index c5684505ee..81d3bad041 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -320,9 +320,15 @@ pub(crate) fn loader_write_schemas<'py>( ) -> PyResult<()> { let mut intermediate_loaders: HashSet = HashSet::new(); for entry in entries { - if native_field(entry, "meta")? - .call_method1("get", (LOADER_NODE_KEY,))? - .is_truthy()? + let meta: Bound<'py, PyAny> = native_field(entry, "meta")?; + let flag: Option> = match meta.downcast::() { + Ok(meta) => meta.get_item(LOADER_NODE_KEY)?, + Err(_) => None, + }; + if flag + .map(|flag| flag.is_truthy()) + .transpose()? + .unwrap_or(false) { intermediate_loaders.insert(native_field(entry, "name")?.extract()?); } From d61540a84370eba64dff1d090720261302f82bcd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:04:30 +0000 Subject: [PATCH 10/18] fix(load): load a managed source into the target's loader_schema sqb load filled a managed source's missing schema from the target's schema, while build uses the target's loader_schema first, so the two commands wrote the same source to different schemas. Load now resolves it the same way, and its missing-schema error names a managed source as one. --- .../bindings/_helpers/load/load_selection.rs | 22 ++++++++++++++----- .../compile/test_missing_target_database.py | 12 ++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 81d3bad041..0cc371211e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -51,7 +51,7 @@ pub(crate) struct LoadProject<'py> { graph: LoadGraph, } -/// Every declared source with the target's database and schema filled in where unset. +/// Every declared source with the target's database and schema, or a managed source's loader_schema, filled in where unset. pub(crate) fn environment_sources<'py>( discovered: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, @@ -62,10 +62,19 @@ pub(crate) fn environment_sources<'py>( for source in native_field(&source_file?, "source_entries")?.try_iter()? { let source: Bound<'py, PyAny> = source?; let changes: Bound<'py, PyDict> = PyDict::new(py); + let managed: bool = native_field(&source, "managed")?.is_truthy()?; for field in ["database", "schema"] { let value: Bound<'py, PyAny> = native_field(&source, field)?; let effective: Bound<'py, PyAny> = if !value.is_none() || target_config.is_none() { value + } else if managed && field == "schema" { + let loader_schema: Bound<'py, PyAny> = + native_field(target_config, "loader_schema")?; + if loader_schema.is_truthy()? { + loader_schema + } else { + native_field(target_config, field)? + } } else { native_field(target_config, field)? }; @@ -352,11 +361,14 @@ pub(crate) fn loader_write_schemas<'py>( if native_field(entry, "loader")?.is_none() { continue; } + let name: String = native_field(entry, "name")?.extract()?; + let kind: &str = if intermediate_loaders.contains(&name) { + "Intermediate loader" + } else { + "Managed source" + }; managed_write_schema( - &format!( - "Intermediate loader '{}'", - native_field(entry, "name")?.str()? - ), + &format!("{kind} '{name}'"), &native_field(entry, "schema")?, (None, Some(&ignored)), )?; diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 9eaae1fba0..d3fdbb4978 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -310,6 +310,18 @@ def test_given_connection_location_when_compiling_then_models_build_there( command=("--no-color", "build"), expected_rows=((1,),), ), + ManagedSourceLoaderLocationTestCase( + description="load loads the managed source into the target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n[connection]\n' + 'database = "dev.duckdb"\nschema = "landing"\n\n' + '[connections.archive]\ndatabase = "archive.duckdb"\n\n' + '[targets.dev]\nloader_schema = "raw"\n' + ), + source_schema="", + command=("--no-color", "load", "--select", "raw_orders"), + expected_rows=((1,),), + ), ManagedSourceLoaderLocationTestCase( description="build under a preserved target loads the source into its own schema", project_toml=_PRESERVED_TARGET_PROJECT, From c63997b7ed100fee18be9afd2bada2fc3f637dc5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:07:11 +0000 Subject: [PATCH 11/18] refactor(load): pick a managed source's schema without comparing field names --- .../src/bindings/_helpers/load/load_selection.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 0cc371211e..0107137c38 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -63,11 +63,11 @@ pub(crate) fn environment_sources<'py>( let source: Bound<'py, PyAny> = source?; let changes: Bound<'py, PyDict> = PyDict::new(py); let managed: bool = native_field(&source, "managed")?.is_truthy()?; - for field in ["database", "schema"] { + for (field, loader_first) in [("database", false), ("schema", managed)] { let value: Bound<'py, PyAny> = native_field(&source, field)?; let effective: Bound<'py, PyAny> = if !value.is_none() || target_config.is_none() { value - } else if managed && field == "schema" { + } else if loader_first { let loader_schema: Bound<'py, PyAny> = native_field(target_config, "loader_schema")?; if loader_schema.is_truthy()? { From 0be3958385879ecda6e3015004d0d376e8ce05e5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:21:56 +0000 Subject: [PATCH 12/18] fix(load): stop a managed source that would load into a schema named preserve With loader_schema or schema set to preserve and no schema on the source, build stops with P001 while load wrote into a schema literally named preserve. Load now raises the same error. --- .../bindings/_helpers/load/load_selection.rs | 48 ++++++++++++++++++- .../compile/test_missing_target_database.py | 28 +++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 0107137c38..35b25dcbfe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -2,11 +2,15 @@ use std::collections::{HashMap, HashSet}; +use pyo3::exceptions::PyValueError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::errors::AssemblyError; +use sqlbuild_analysis::assembly::project::main::preserved_logical_namespace::preserved_logical_namespace; +use sqlbuild_analysis::assembly::project::models::TargetNamespace; use sqlbuild_core::load_command::errors::UnknownLoadSelector; use sqlbuild_core::load_command::main::load_reference_loaders::load_reference_loaders; use sqlbuild_core::load_command::main::select_load_entries::select_load_entries as select_entries; @@ -28,12 +32,13 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_ada use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::construct::{new_helped_error, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; +const COMPILE_INPUT_CODE: &str = "P001"; const LOADER_TABLE_PREFIX: &str = "__loader__"; const UNKNOWN_SELECTOR_CODE: &str = "C901"; const UNKNOWN_SELECTOR_HELP: &str = "Use exact managed source names or loader function names."; @@ -63,6 +68,9 @@ pub(crate) fn environment_sources<'py>( let source: Bound<'py, PyAny> = source?; let changes: Bound<'py, PyDict> = PyDict::new(py); let managed: bool = native_field(&source, "managed")?.is_truthy()?; + if managed && !target_config.is_none() { + preserved_source_namespace(&source, target_config)?; + } for (field, loader_first) in [("database", false), ("schema", managed)] { let value: Bound<'py, PyAny> = native_field(&source, field)?; let effective: Bound<'py, PyAny> = if !value.is_none() || target_config.is_none() { @@ -86,6 +94,44 @@ pub(crate) fn environment_sources<'py>( Ok(sources) } +/// Raise build's error when a managed source has no schema of its own and its target value is `preserve`. +fn preserved_source_namespace( + source: &Bound<'_, PyAny>, + target_config: &Bound<'_, PyAny>, +) -> PyResult<()> { + let text = |record: &Bound<'_, PyAny>, field: &str| -> PyResult> { + native_field(record, field)?.extract() + }; + let loader_schema: Option = + text(target_config, "loader_schema")?.filter(|value| !value.is_empty()); + let target: TargetNamespace = TargetNamespace { + database: text(target_config, "database")?, + schema: loader_schema.clone().or(text(target_config, "schema")?), + loader_schema, + }; + let label: String = format!( + "Managed source '{}'", + text(source, "name")?.unwrap_or_default() + ); + match preserved_logical_namespace( + &label, + Some(&target), + ( + text(source, "database")?.as_ref(), + text(source, "schema")?.as_ref(), + ), + ) { + Ok(()) => Ok(()), + Err(AssemblyError::CompileInput { message, help }) => Err(new_helped_error( + source.py(), + ErrorId::CompileInput, + (message, COMPILE_INPUT_CODE), + help.as_deref(), + )), + Err(other) => Err(PyValueError::new_err(format!("{other:?}"))), + } +} + impl<'py> LoadProject<'py> { pub(crate) fn new( discovered: &Bound<'py, PyAny>, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index d3fdbb4978..f1f074afba 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -209,6 +209,34 @@ expected_example_count=0, command=("--no-color", "load", "--select", "raw_orders"), ), + MissingTargetDatabaseTestCase( + description="sqb load stops a managed source under a preserved loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "preserve"\n' + ), + extra_files={ + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " managed: true\n write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error=( + "error[P001]: Managed source 'raw_orders' has no logical schema, but the " + "selected target sets schema to 'preserve'\n" + " = help: Set schema on the resource or its defaults, or configure a literal " + "target schema.\n" + ), + expected_example_count=0, + command=("--no-color", "load", "--select", "raw_orders"), + ), ), ids=lambda case: case.description, ) From 519b010fd70d5c1345d83d0039cf388b77970c21 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:59:52 +0000 Subject: [PATCH 13/18] fix(load): expand templated target locations for managed sources sqb load read the target's database, schema and loader_schema as written when it placed a managed source, so a value such as ${raw_schema} failed with a parser error while build expanded it. Load now expands them with the project and command-line vars, as build does. --- .../bindings/_helpers/load/load_command.rs | 13 ++++- .../bindings/_helpers/load/load_selection.rs | 57 ++++++++++++++++--- .../compile/test_missing_target_database.py | 22 +++++++ 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index d82b7fb7be..3789ba52af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -33,7 +33,7 @@ use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; use crate::bindings::_helpers::load::load_selection::{ - LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, + LoadProject, LoaderDefaults, expanded_target_config, loader_defaults, loader_write_schemas, }; use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; @@ -110,7 +110,16 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = LoadProject::new(&discovered, &target_config)?; + let expanded_target: Bound<'py, PyAny> = expanded_target_config( + &discovered, + ( + &selected_target, + &target_config, + &native_field(request, "cli_vars")?, + ), + )?; + let project: LoadProject<'py> = + LoadProject::new(&discovered, (&target_config, &expanded_target))?; let selected: Vec> = project.selected_entries( ( &select.extract::>()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 35b25dcbfe..94d8830e22 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -39,6 +39,7 @@ use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const COMPILE_INPUT_CODE: &str = "P001"; +const TARGET_LOCATION_FIELDS: [&str; 3] = ["database", "schema", "loader_schema"]; const LOADER_TABLE_PREFIX: &str = "__loader__"; const UNKNOWN_SELECTOR_CODE: &str = "C901"; const UNKNOWN_SELECTOR_HELP: &str = "Use exact managed source names or loader function names."; @@ -56,10 +57,10 @@ pub(crate) struct LoadProject<'py> { graph: LoadGraph, } -/// Every declared source with the target's database and schema, or a managed source's loader_schema, filled in where unset. +/// Every declared source with the expanded target's database and schema, or a managed source's loader_schema, filled in where unset. pub(crate) fn environment_sources<'py>( discovered: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, + (target_config, expanded_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult>> { let py: Python<'py> = discovered.py(); let mut sources: Vec> = Vec::new(); @@ -77,14 +78,14 @@ pub(crate) fn environment_sources<'py>( value } else if loader_first { let loader_schema: Bound<'py, PyAny> = - native_field(target_config, "loader_schema")?; + native_field(expanded_target, "loader_schema")?; if loader_schema.is_truthy()? { loader_schema } else { - native_field(target_config, field)? + native_field(expanded_target, field)? } } else { - native_field(target_config, field)? + native_field(expanded_target, field)? }; changes.set_item(field, effective)?; } @@ -133,12 +134,14 @@ fn preserved_source_namespace( } impl<'py> LoadProject<'py> { + /// The load project, with `expanded_target` the target config after template expansion. pub(crate) fn new( discovered: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, + (target_config, expanded_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult { let py: Python<'py> = discovered.py(); - let sources: Vec> = environment_sources(discovered, target_config)?; + let sources: Vec> = + environment_sources(discovered, (target_config, expanded_target))?; let mut graph: LoadGraph = LoadGraph::default(); for source in &sources { let loader: Bound<'py, PyAny> = native_field(source, "loader")?; @@ -367,6 +370,40 @@ pub(crate) fn loader_defaults<'py>( }) } +/// `target_config` with its database, schema and loader_schema templates expanded with `cli_vars`. +pub(crate) fn expanded_target_config<'py>( + discovered: &Bound<'py, PyAny>, + (selected_target, target_config, cli_vars): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + if target_config.is_none() { + return Ok(target_config.clone()); + } + let py: Python<'py> = discovered.py(); + let values: Bound<'py, PyDict> = PyDict::new(py); + for field in TARGET_LOCATION_FIELDS { + values.set_item(field, native_field(target_config, field)?)?; + } + let runtime: Bound<'py, PyTuple> = build_effective_runtime_config( + discovered, + Some(selected_target.clone()).filter(|value| !value.is_none()), + Some(cli_vars.clone()).filter(|value| !value.is_none()), + )?; + let resolved: Bound<'py, PyAny> = early_model_templates( + values.as_any(), + &runtime.get_item(1)?, + (&runtime.get_item(0)?, &runtime.get_item(2)?), + )?; + let changes: Bound<'py, PyDict> = PyDict::new(py); + for field in TARGET_LOCATION_FIELDS { + changes.set_item(field, resolved.get_item(field)?)?; + } + replaced_keywords(target_config, &changes) +} + /// Raise when a selected loader would write to `preserve` or lost its schema to an ignored connection. pub(crate) fn loader_write_schemas<'py>( discovered: &Bound<'py, PyAny>, @@ -452,9 +489,10 @@ fn select_load_entries_entry<'py>( &["loader_default_database", "loader_default_schema"], ), )?; + let target_config: Bound<'py, PyAny> = arguments.value("target_config")?; let project: LoadProject<'py> = LoadProject::new( &arguments.value("discovered_inputs")?, - &arguments.value("target_config")?, + (&target_config, &target_config), )?; let entries: Vec> = project.selected_entries( ( @@ -482,9 +520,10 @@ fn select_load_reference_entries<'py>( &["loader_default_database", "loader_default_schema"], ), )?; + let target_config: Bound<'py, PyAny> = arguments.value("target_config")?; let project: LoadProject<'py> = LoadProject::new( &arguments.value("discovered_inputs")?, - &arguments.value("target_config")?, + (&target_config, &target_config), )?; let selected: Vec> = arguments .value("selected_sources")? diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index f1f074afba..a5506e3eb9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -350,6 +350,28 @@ def test_given_connection_location_when_compiling_then_models_build_there( command=("--no-color", "load", "--select", "raw_orders"), expected_rows=((1,),), ), + ManagedSourceLoaderLocationTestCase( + description="load expands a templated target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[vars]\nraw_schema = "raw"\n\n[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "${raw_schema}"\n' + ), + source_schema="", + command=("--no-color", "load", "--select", "raw_orders"), + expected_rows=((1,),), + ), + ManagedSourceLoaderLocationTestCase( + description="build expands a templated target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[vars]\nraw_schema = "raw"\n\n[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "${raw_schema}"\n' + ), + source_schema="", + command=("--no-color", "build"), + expected_rows=((1,),), + ), ManagedSourceLoaderLocationTestCase( description="build under a preserved target loads the source into its own schema", project_toml=_PRESERVED_TARGET_PROJECT, From 4d958c7a13f32d9b772e9234037ecee7ec7e7633 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:13:29 +0000 Subject: [PATCH 14/18] fix(load): expand only the target location a managed source falls back to Expanding the whole target up front failed load on unused templates that build ignores, and accepted run context templates that build refuses. Load now expands just the fallback value, with build's own options. --- .../_helpers/compile_driver/targets.rs | 32 +++++---- .../bindings/_helpers/load/load_command.rs | 13 ++-- .../bindings/_helpers/load/load_selection.rs | 67 ++++++++----------- .../compile/test_missing_target_database.py | 34 ++++++++++ 4 files changed, 85 insertions(+), 61 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index f24e9e7019..b67ff4e718 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -402,18 +402,7 @@ fn managed_source_namespace( let value: Bound<'_, PyDict> = PyDict::new(py); pyo3::types::PyDictMethods::set_item(&value, "database", database)?; pyo3::types::PyDictMethods::set_item(&value, "schema", schema)?; - let expanded: Bound<'_, PyAny> = expanded_template_data( - value.as_any(), - (effective_vars.downcast()?, &PyDict::new(py)), - ( - TemplateOptions { - allow_context: false, - preserve_context_tokens: false, - preserve_unknown_context: false, - }, - LOADER_NAMESPACE_LABEL, - ), - )?; + let expanded: Bound<'_, PyAny> = expanded_loader_namespace(value.as_any(), effective_vars)?; let part = |key: &str| -> PyResult> { let value: Bound<'_, PyAny> = expanded.get_item(key)?; if value.is_none() { @@ -424,6 +413,25 @@ fn managed_source_namespace( Ok((part("database")?, part("schema")?)) } +/// A managed source's `database` and `schema` with project vars expanded and no context templates. +pub(crate) fn expanded_loader_namespace<'py>( + value: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + expanded_template_data( + value, + (effective_vars.downcast()?, &PyDict::new(value.py())), + ( + TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }, + LOADER_NAMESPACE_LABEL, + ), + ) +} + /// Validate compiled model and seed locations for the effective adapter. #[pyfunction] #[pyo3(signature = (**options))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 3789ba52af..beae4031db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -33,7 +33,7 @@ use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; use crate::bindings::_helpers::load::load_selection::{ - LoadProject, LoaderDefaults, expanded_target_config, loader_defaults, loader_write_schemas, + LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, target_vars, }; use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; @@ -110,16 +110,11 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = expanded_target_config( + let vars: Bound<'py, PyAny> = target_vars( &discovered, - ( - &selected_target, - &target_config, - &native_field(request, "cli_vars")?, - ), + (&selected_target, &native_field(request, "cli_vars")?), )?; - let project: LoadProject<'py> = - LoadProject::new(&discovered, (&target_config, &expanded_target))?; + let project: LoadProject<'py> = LoadProject::new(&discovered, (&target_config, &vars))?; let selected: Vec> = project.selected_entries( ( &select.extract::>()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 94d8830e22..18e55e6e12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::compile_attachment::target::{ connection_is_default_target, project_location, }; use crate::bindings::_helpers::compile_driver::targets::{ - managed_write_schema, preserved_loader_schemas, + expanded_loader_namespace, managed_write_schema, preserved_loader_schemas, }; use crate::bindings::_helpers::contracts::spec_resolution::loader_destination_parts; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -39,7 +39,6 @@ use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const COMPILE_INPUT_CODE: &str = "P001"; -const TARGET_LOCATION_FIELDS: [&str; 3] = ["database", "schema", "loader_schema"]; const LOADER_TABLE_PREFIX: &str = "__loader__"; const UNKNOWN_SELECTOR_CODE: &str = "C901"; const UNKNOWN_SELECTOR_HELP: &str = "Use exact managed source names or loader function names."; @@ -57,10 +56,10 @@ pub(crate) struct LoadProject<'py> { graph: LoadGraph, } -/// Every declared source with the expanded target's database and schema, or a managed source's loader_schema, filled in where unset. +/// Every declared source with the target's database and schema, or a managed source's loader_schema expanded with `vars`, filled in where unset. pub(crate) fn environment_sources<'py>( discovered: &Bound<'py, PyAny>, - (target_config, expanded_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (target_config, vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult>> { let py: Python<'py> = discovered.py(); let mut sources: Vec> = Vec::new(); @@ -74,19 +73,28 @@ pub(crate) fn environment_sources<'py>( } for (field, loader_first) in [("database", false), ("schema", managed)] { let value: Bound<'py, PyAny> = native_field(&source, field)?; - let effective: Bound<'py, PyAny> = if !value.is_none() || target_config.is_none() { + let fallback: bool = value.is_none() && !target_config.is_none(); + let effective: Bound<'py, PyAny> = if !fallback { value } else if loader_first { let loader_schema: Bound<'py, PyAny> = - native_field(expanded_target, "loader_schema")?; + native_field(target_config, "loader_schema")?; if loader_schema.is_truthy()? { loader_schema } else { - native_field(expanded_target, field)? + native_field(target_config, field)? } } else { - native_field(expanded_target, field)? + native_field(target_config, field)? }; + let effective: Bound<'py, PyAny> = + if managed && fallback && !vars.is_none() && !effective.is_none() { + let location: Bound<'py, PyDict> = PyDict::new(py); + location.set_item(field, effective)?; + expanded_loader_namespace(location.as_any(), vars)?.get_item(field)? + } else { + effective + }; changes.set_item(field, effective)?; } sources.push(replaced_keywords(&source, &changes)?); @@ -134,14 +142,14 @@ fn preserved_source_namespace( } impl<'py> LoadProject<'py> { - /// The load project, with `expanded_target` the target config after template expansion. + /// The load project, with `vars` expanding managed sources' target locations, or None. pub(crate) fn new( discovered: &Bound<'py, PyAny>, - (target_config, expanded_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (target_config, vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult { let py: Python<'py> = discovered.py(); let sources: Vec> = - environment_sources(discovered, (target_config, expanded_target))?; + environment_sources(discovered, (target_config, vars))?; let mut graph: LoadGraph = LoadGraph::default(); for source in &sources { let loader: Bound<'py, PyAny> = native_field(source, "loader")?; @@ -370,38 +378,17 @@ pub(crate) fn loader_defaults<'py>( }) } -/// `target_config` with its database, schema and loader_schema templates expanded with `cli_vars`. -pub(crate) fn expanded_target_config<'py>( +/// The project, target and command-line vars that expand a managed source's target location. +pub(crate) fn target_vars<'py>( discovered: &Bound<'py, PyAny>, - (selected_target, target_config, cli_vars): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), + (selected_target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - if target_config.is_none() { - return Ok(target_config.clone()); - } - let py: Python<'py> = discovered.py(); - let values: Bound<'py, PyDict> = PyDict::new(py); - for field in TARGET_LOCATION_FIELDS { - values.set_item(field, native_field(target_config, field)?)?; - } - let runtime: Bound<'py, PyTuple> = build_effective_runtime_config( + build_effective_runtime_config( discovered, Some(selected_target.clone()).filter(|value| !value.is_none()), Some(cli_vars.clone()).filter(|value| !value.is_none()), - )?; - let resolved: Bound<'py, PyAny> = early_model_templates( - values.as_any(), - &runtime.get_item(1)?, - (&runtime.get_item(0)?, &runtime.get_item(2)?), - )?; - let changes: Bound<'py, PyDict> = PyDict::new(py); - for field in TARGET_LOCATION_FIELDS { - changes.set_item(field, resolved.get_item(field)?)?; - } - replaced_keywords(target_config, &changes) + )? + .get_item(1) } /// Raise when a selected loader would write to `preserve` or lost its schema to an ignored connection. @@ -492,7 +479,7 @@ fn select_load_entries_entry<'py>( let target_config: Bound<'py, PyAny> = arguments.value("target_config")?; let project: LoadProject<'py> = LoadProject::new( &arguments.value("discovered_inputs")?, - (&target_config, &target_config), + (&target_config, &py.None().into_bound(py)), )?; let entries: Vec> = project.selected_entries( ( @@ -523,7 +510,7 @@ fn select_load_reference_entries<'py>( let target_config: Bound<'py, PyAny> = arguments.value("target_config")?; let project: LoadProject<'py> = LoadProject::new( &arguments.value("discovered_inputs")?, - (&target_config, &target_config), + (&target_config, &py.None().into_bound(py)), )?; let selected: Vec> = arguments .value("selected_sources")? diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index a5506e3eb9..3ee1d8dc14 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -237,6 +237,29 @@ expected_example_count=0, command=("--no-color", "load", "--select", "raw_orders"), ), + MissingTargetDatabaseTestCase( + description="sqb load refuses a run context template in the target loader schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "raw_${CTX:run.id}"\n' + ), + extra_files={ + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " managed: true\n write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error="managed loader target namespace does not allow CTX templates", + expected_example_count=0, + command=("--no-color", "load", "--select", "raw_orders"), + ), ), ids=lambda case: case.description, ) @@ -372,6 +395,17 @@ def test_given_connection_location_when_compiling_then_models_build_there( command=("--no-color", "build"), expected_rows=((1,),), ), + ManagedSourceLoaderLocationTestCase( + description="load ignores a target loader schema template no source uses", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[targets.dev]\nschema = "analytics"\nloader_schema = "${missing_var}"\n' + ), + source_schema=" schema: raw\n", + command=("--no-color", "load", "--select", "raw_orders"), + expected_rows=((1,),), + ), ManagedSourceLoaderLocationTestCase( description="build under a preserved target loads the source into its own schema", project_toml=_PRESERVED_TARGET_PROJECT, From 6fa727286730ea4c409267d8dc5a19d8bc95225e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:19:11 +0000 Subject: [PATCH 15/18] fix(load): read an expanded target location as text, as build does --- .../src/bindings/_helpers/load/load_selection.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 18e55e6e12..df61fe16d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -91,7 +91,13 @@ pub(crate) fn environment_sources<'py>( if managed && fallback && !vars.is_none() && !effective.is_none() { let location: Bound<'py, PyDict> = PyDict::new(py); location.set_item(field, effective)?; - expanded_loader_namespace(location.as_any(), vars)?.get_item(field)? + let expanded: Bound<'py, PyAny> = + expanded_loader_namespace(location.as_any(), vars)?.get_item(field)?; + if expanded.is_none() { + expanded + } else { + expanded.str()?.into_any() + } } else { effective }; From 411f16f6a6bbbf203e9e72b401db31e73ba01e0a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:22:31 +0000 Subject: [PATCH 16/18] docs(deploy): run the server and a location as Podman Quadlet units --- deploy/README.md | 115 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/deploy/README.md b/deploy/README.md index 531a9ff768..e743652bea 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -166,6 +166,121 @@ In `.env`, set `DOCKER_SOCKET` to the socket path and `DOCKER_GID` to `podman compose`. Check that it uses Docker Compose: `podman compose version` prints `Docker Compose version`. +## Run as Podman Quadlet units + +Quadlet runs the server and a location as systemd services, without Compose. These rootful units +use the socket group set up in [Run on Podman](#run-on-podman) and images you built or pulled. + +Name the location container in the server's launcher, in `/etc/sqlbuild/server.toml`: + +```toml +[server] +host = "0.0.0.0" +port = 8000 + +[[launchers.docker]] +location = "orders" +target = "development" +server_url = "http://sqlbuild-server:8000" +max_workers = 1 +socket = "/var/run/docker.sock" +location_container = "orders-location" +executable = "/usr/local/bin/sqb" +``` + +Store the control database URL and, after `sqb admin create-location orders` and +`sqb admin issue-location-token`, the location's bearer as Podman secrets: + +```sh +printf '%s' "$SQLBUILD_CONTROL_DATABASE_URL" | sudo podman secret create sqlbuild-control-database-url - +printf '%s' "$SQLBUILD_LOCATION_TOKEN" | sudo podman secret create sqlbuild-orders-location-token - +``` + +Put these three files in `/etc/containers/systemd/`. `/etc/containers/systemd/sqlbuild.network`: + +```ini +[Network] +NetworkName=sqlbuild +``` + +`/etc/containers/systemd/sqlbuild-server.container`, with `GroupAdd` set to +`getent group sqlbuild-socket | cut -d: -f3`: + +```ini +[Unit] +Description=SQLBuild server and scheduler +Requires=podman.socket +After=podman.socket network-online.target + +[Container] +Image=localhost/sqlbuild-server:0.224.0 +ContainerName=sqlbuild-server +Network=sqlbuild.network +PublishPort=127.0.0.1:8000:8000 +Volume=/etc/sqlbuild/server.toml:/etc/sqlbuild/server.toml:ro +Volume=/run/podman/podman.sock:/var/run/docker.sock +GroupAdd=993 +SecurityLabelDisable=true +Secret=sqlbuild-control-database-url,type=env,target=SQLBUILD_CONTROL_DATABASE_URL +HealthCmd=curl --fail --silent --output /dev/null http://localhost:8000/api/openapi.json +HealthInterval=10s +HealthStartPeriod=30s + +[Service] +Restart=always +TimeoutStartSec=120 + +[Install] +WantedBy=multi-user.target +``` + +`/etc/containers/systemd/orders-location.container`, with the `location_id` that +`create-location` returned: + +```ini +[Unit] +Description=SQLBuild orders code location +Wants=sqlbuild-server.service +After=sqlbuild-server.service + +[Container] +Image=localhost/sqlbuild-orders:0.224.0 +ContainerName=orders-location +Network=sqlbuild.network +Volume=sqlbuild-warehouse:/warehouse +Secret=sqlbuild-orders-location-token,type=env,target=SQLBUILD_LOCATION_TOKEN +Exec=location serve --server http://sqlbuild-server:8000 --location-id 00000000-0000-0000-0000-000000000001 --target development +HealthCmd=sqb location health --server http://sqlbuild-server:8000 --location-id 00000000-0000-0000-0000-000000000001 --target development +HealthInterval=30s +HealthStartPeriod=60s +PodmanArgs=--cpus 2 --memory 1g + +[Service] +Restart=always +TimeoutStartSec=120 + +[Install] +WantedBy=multi-user.target +``` + +Then load and start them: + +```sh +sudo systemctl daemon-reload +sudo systemctl start orders-location.service +sudo podman healthcheck run orders-location +``` + +- `ContainerName=` must equal the launcher's `location_container`: the server finds the location's + image, mounts, network and limits through that container, and workers join its network. +- Write the location ID literally in `Exec=` and `HealthCmd=`. systemd expands `${...}` in those + lines from the service's environment, not the container's. +- `Restart=always` restarts either container when it exits. The location only `Wants=` the server, + so a server restart leaves it running; it checks in again when the server is back. +- The server resolves launcher locations when it starts. To add a location, create it, add its + launcher and container unit, then restart the server: `sudo systemctl restart sqlbuild-server`. +- `SecurityLabelDisable=true` lets the server use the socket on SELinux hosts. + ## Credentials, identity, and platform events Rotate a location credential with `docker compose run --rm --no-deps server admin rotate-token From dc9e1c7d140540e48d4957b13cac9f70328d97fa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:37:06 +0000 Subject: [PATCH 17/18] fix(scheduler): wake on a due time however long the store took, and log slow cycles The loop slept for the next due time's distance measured before the cycle's last commit, so a slow commit or WAL checkpoint pushed every wake back by its own length. It now sleeps until that deadline. Each cycle records per-phase timings, the worker supervisor logs cycles slower than 500 ms with them, and the latency test bounds how late the scheduler wakes while printing store time separately. --- .../src/workers/_helpers/supervisor.rs | 19 +++++- crates/sqlbuild-cli/src/workers/constants.rs | 6 +- .../scheduler/_helpers/drain_due_triggers.rs | 12 ++-- .../src/scheduler/main/_scheduler_cycle.rs | 28 +++++++- .../src/scheduler/main/run_scheduler.rs | 13 ++-- .../src/scheduler/models.rs | 20 ++++++ .../src/scheduler/types.rs | 5 +- .../tests/scheduler/ticks/helpers.rs | 34 ++++++++-- .../tests/scheduler/ticks/test_latency.rs | 65 +++++++++++-------- .../tests/scheduler/ticks/test_types.rs | 52 ++++++++++++++- 10 files changed, 204 insertions(+), 50 deletions(-) diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index 80d4e295dc..8a683b2b9c 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -16,7 +16,7 @@ use sqlbuild_orchestration::runs::models::{ }; use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; -use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::scheduler::models::{SchedulerCycle, SchedulerLease}; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::models::ControlStore; use sqlbuild_process_ownership::ownership::main::{ @@ -26,6 +26,7 @@ use tokio::process::Command; use uuid::Uuid; use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::constants::SLOW_SCHEDULER_CYCLE; use crate::workers::models::{ ClaimedWorker, DockerWorker, Launcher, LocalLauncher, LocalSpawn, WorkerChild, WorkerExit, Workers, @@ -149,6 +150,22 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { self.shutdown_workers(store, lease).await } + fn cycled(&mut self, cycle: &SchedulerCycle) { + let timings = &cycle.timings; + if timings.total() >= SLOW_SCHEDULER_CYCLE { + drop(writeln!( + std::io::stderr().lock(), + "Slow scheduler cycle: {} ms (lease {} ms, triggers {} ms, run monitor {} ms, tick cleanup {} ms, next due {} ms)", + timings.total().as_millis(), + timings.lease.as_millis(), + timings.triggers.as_millis(), + timings.monitor.as_millis(), + timings.prune.as_millis(), + timings.next_due.as_millis(), + )); + } + } + /// A claim whose commit outcome is unknown is adopted by the next recovery pass. fn retrying(&mut self, error: &ControlError) { self.recovered_epoch = None; diff --git a/crates/sqlbuild-cli/src/workers/constants.rs b/crates/sqlbuild-cli/src/workers/constants.rs index 881c48bea1..6db1d72d80 100644 --- a/crates/sqlbuild-cli/src/workers/constants.rs +++ b/crates/sqlbuild-cli/src/workers/constants.rs @@ -1,3 +1,7 @@ -//! Immutable Docker image identities use the full SHA-256 hexadecimal digest. +//! Worker supervision limits. +/// Immutable Docker image identities use the full SHA-256 hexadecimal digest. pub(super) const IMAGE_ID_HEX_LENGTH: usize = 64; + +/// Healthy scheduler cycles take tens of milliseconds; slower ones are logged with phase timings. +pub(super) const SLOW_SCHEDULER_CYCLE: std::time::Duration = std::time::Duration::from_millis(500); diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs index 7161888c49..30ed2819c1 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -1,6 +1,6 @@ //! Drain a bounded due-trigger page, renewing the lease between persisted evaluations. -use std::time::Duration; +use std::time::{Duration, Instant}; use chrono::{DateTime, Utc}; use uuid::Uuid; @@ -42,13 +42,14 @@ pub(crate) async fn drain_due_triggers( Ok((lease, cycle)) } -/// Shorten the cycle wait to the next running trigger's due time, never below zero. +/// Shorten the wait to the next due time, never below zero, measured from the returned instant. pub(crate) async fn wait_for_next_due( store: &ControlStore, wait_for: Duration, -) -> Result { +) -> Result<(Duration, Instant), ControlError> { let transaction = begin(store, TransactionMode::Read).await?; let now = transaction.now; + let read_at = Instant::now(); let (transaction, rows) = query(transaction, "SELECT MIN(s.next_due_at) AS next_due_at FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ JOIN sqb_locations l ON l.location_id=t.location_id \ @@ -56,10 +57,11 @@ pub(crate) async fn wait_for_next_due( let next: Option> = serde_json::from_value(rows[0]["next_due_at"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; commit(transaction).await?; - Ok(next.map_or(wait_for, |next| { + let wait_for = next.map_or(wait_for, |next| { (next - now) .to_std() .unwrap_or(Duration::ZERO) .min(wait_for) - })) + }); + Ok((wait_for, read_at)) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs index c6267cfc84..f27642c52d 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs @@ -1,6 +1,8 @@ //! A cycle renews leadership and drains a bounded set of persisted due triggers. -use std::time::Duration; +use std::time::{Duration, Instant}; + +use chrono::Utc; use crate::runs::constants::MONITOR_BATCH_SIZE; use crate::runs::main::monitor_runs::monitor_runs; @@ -9,7 +11,7 @@ use crate::scheduler::_helpers::tick_retention::prune_ticks_when_due; use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; use crate::scheduler::main::_acquire_lease::acquire_lease; use crate::scheduler::main::renew_lease::renew_lease; -use crate::scheduler::models::{SchedulerCycle, SchedulerState}; +use crate::scheduler::models::{CycleTimings, SchedulerCycle, SchedulerState}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -17,6 +19,15 @@ pub(crate) async fn scheduler_cycle( store: &ControlStore, mut state: SchedulerState, ) -> Result<(SchedulerState, SchedulerCycle), ControlError> { + let mut timings = CycleTimings { + started_at: Utc::now(), + lease: Duration::ZERO, + triggers: Duration::ZERO, + monitor: Duration::ZERO, + prune: Duration::ZERO, + next_due: Duration::ZERO, + }; + let phase = Instant::now(); let renewal = match state.lease.as_ref() { Some(lease) => renew_lease(store, lease).await.map(Some), None => acquire_lease(store, state.holder).await, @@ -26,23 +37,34 @@ pub(crate) async fn scheduler_cycle( Err(ControlError::Fenced) => None, Err(error) => return Err(error), }; + timings.lease = phase.elapsed(); let cycle = SchedulerCycle { leader: state.lease.is_some(), evaluated: 0, ticks: 0, unresponsive: 0, wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), + wait_from: Instant::now(), + timings, }; let Some(lease) = state.lease.clone() else { return Ok((state, cycle)); }; + let phase = Instant::now(); let (lease, mut cycle) = drain_due_triggers(store, lease, cycle).await?; + cycle.timings.triggers = phase.elapsed(); + let phase = Instant::now(); cycle.unresponsive = monitor_runs(store, &lease).await?; if cycle.unresponsive == MONITOR_BATCH_SIZE { cycle.wait_for = Duration::ZERO; } + cycle.timings.monitor = phase.elapsed(); + let phase = Instant::now(); state.pruned_at = prune_ticks_when_due(store, &lease, state.pruned_at).await?; state.lease = Some(lease); - cycle.wait_for = wait_for_next_due(store, cycle.wait_for).await?; + cycle.timings.prune = phase.elapsed(); + let phase = Instant::now(); + (cycle.wait_for, cycle.wait_from) = wait_for_next_due(store, cycle.wait_for).await?; + cycle.timings.next_due = phase.elapsed(); Ok((state, cycle)) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index 2917ec1447..42b2fb2ea4 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -3,6 +3,7 @@ use std::time::Duration; use tokio::sync::watch; +use tokio::time::Instant; use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; use crate::scheduler::models::SchedulerState; @@ -24,23 +25,25 @@ pub async fn run_scheduler( Ok((next_state, cycle)) => { state = next_state; last_lease = state.lease.clone(); + launcher.cycled(&cycle); launcher .poll(store, state.lease.as_ref()) .await - .map(|()| cycle.wait_for) + .map(|()| Instant::from_std(cycle.wait_from + cycle.wait_for)) } Err(error) => Err(error), }; - let wait_for = match polled { - Ok(wait_for) => wait_for, + let retry = Instant::now() + Duration::from_secs(1); + let due = match polled { + Ok(due) => due.min(retry), Err(error) if error.transient() => { launcher.retrying(&error); - Duration::from_secs(1) + retry } Err(error) => return Err(error), }; tokio::select! { - _ = tokio::time::sleep(wait_for.min(Duration::from_secs(1))) => {}, + _ = tokio::time::sleep_until(due) => {}, result = shutdown.changed() => { if result.is_err() { break; } }, } } diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index c448b4138e..c861f59ed6 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -40,6 +40,26 @@ pub struct SchedulerCycle { pub ticks: usize, pub unresponsive: usize, pub wait_for: std::time::Duration, + /// When `wait_for` was read, so the loop wakes on time however long the store takes afterwards. + pub wait_from: std::time::Instant, + pub timings: CycleTimings, +} + +/// Wall time per cycle phase, which separates a late wake from slow store writes. +#[derive(Clone, Debug)] +pub struct CycleTimings { + pub started_at: DateTime, + pub lease: std::time::Duration, + pub triggers: std::time::Duration, + pub monitor: std::time::Duration, + pub prune: std::time::Duration, + pub next_due: std::time::Duration, +} + +impl CycleTimings { + pub fn total(&self) -> std::time::Duration { + self.lease + self.triggers + self.monitor + self.prune + self.next_due + } } /// Validated at the native boundary and evaluated as wall-clock times with SQLBuild's DST policy. diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs index 18c32c7ac9..1515b87c9d 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -1,6 +1,6 @@ //! Compound scheduler return values. -use crate::scheduler::models::SchedulerLease; +use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -18,6 +18,9 @@ pub trait RunLauncher { lease: Option<&SchedulerLease>, ) -> impl Future> + Send; + /// Hears each completed cycle, with its phase timings, before the launcher polls. + fn cycled(&mut self, _cycle: &SchedulerCycle) {} + /// Hears each transient store failure the scheduler retries instead of stopping. fn retrying(&mut self, _error: &ControlError) {} } diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 0aba4ffc27..2d684232f3 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -25,7 +25,7 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::ticks::test_types::{ - FaultAnomalies, FaultStep, IdleLauncher, TransactionFault, + CycleRecorder, FaultAnomalies, FaultStep, RecordedCycle, TransactionFault, }; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::helpers::read_recall_events; @@ -729,12 +729,13 @@ async fn triggers_behind(store: &ControlStore, latest: DateTime) -> i64 { rows[0]["behind"].as_i64().expect("count") } -/// Run one real scheduler loop until `deadline`, then stop it and return the trigger's ticks. +/// Run one real scheduler loop until `deadline`, then return the trigger's ticks and cycle timings. pub(crate) async fn run_scheduler_until( store: &ControlStore, trigger_id: Uuid, - deadline: DateTime, -) -> Vec { + (deadline, poll_delay): (DateTime, std::time::Duration), +) -> (Vec, Vec) { + let mut cycles = Vec::new(); let (stop, shutdown) = tokio::sync::watch::channel(false); let stopping = async { tokio::time::sleep((deadline - Utc::now()).to_std().unwrap_or_default()).await; @@ -745,12 +746,33 @@ pub(crate) async fn run_scheduler_until( store, SchedulerState::new(Uuid::now_v7()), shutdown, - IdleLauncher + CycleRecorder { + cycles: &mut cycles, + poll_delay, + launching: false, + } ), stopping ); finished.expect("scheduler loop"); - list_ticks(store, trigger_id, 100).await.expect("ticks") + let ticks = list_ticks(store, trigger_id, 100).await.expect("ticks"); + (ticks, cycles) +} + +/// How late the cycle that made a decision woke: after `ready` or the previous poll, whichever is later. +pub(crate) fn decision_wake( + cycles: &[RecordedCycle], + decided_at: DateTime, + ready: DateTime, +) -> (TimeDelta, &RecordedCycle) { + let index = cycles + .iter() + .rposition(|cycle| cycle.timings.started_at <= decided_at) + .expect("a cycle started before the decision"); + let free = index + .checked_sub(1) + .map_or(ready, |previous| ready.max(cycles[previous].polled_at)); + (cycles[index].timings.started_at - free, &cycles[index]) } pub(crate) async fn queued_times( diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs index 71291a8feb..6496af2506 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -1,4 +1,6 @@ -//! Bounds tick-to-decision latency, where a decision queues or covers a run, and standby catch-up. +//! Bounds how late the scheduler wakes for due times and standby catch-up behind a slow launcher. + +use std::time::Duration; use chrono::{DurationRound, SecondsFormat, TimeDelta, Utc}; use serde_json::json; @@ -6,7 +8,7 @@ use serde_json::json; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::scheduler::tests::contracts::ticks::helpers::{ - queued_times, run_scheduler_until, snapshot, + decision_wake, queued_times, run_scheduler_until, snapshot, }; use crate::scheduler::tests::contracts::ticks::test_types::SchedulerLatencyTestCase; use crate::snapshots::main::publish_snapshot::publish_snapshot; @@ -25,16 +27,18 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks backend: Backend::Sqlite, expected_ticks: 3, expected_ticks_after_failover: 4, - expected_max_tick_to_decision: TimeDelta::milliseconds(500), - expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + expected_max_wake_after_due: TimeDelta::milliseconds(500), + expected_max_failover_wake: TimeDelta::milliseconds(1_500), + launcher_poll: Duration::from_millis(700), }, SchedulerLatencyTestCase { description: "PostgreSQL scheduler latency", backend: Backend::Postgres, expected_ticks: 3, expected_ticks_after_failover: 4, - expected_max_tick_to_decision: TimeDelta::milliseconds(500), - expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + expected_max_wake_after_due: TimeDelta::milliseconds(500), + expected_max_failover_wake: TimeDelta::milliseconds(1_500), + launcher_poll: Duration::from_millis(700), }, ]; for test_case in test_cases { @@ -68,7 +72,8 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks .await .expect("trigger")[0] .trigger_id; - let ticks = run_scheduler_until(store, trigger_id, at(6)).await; + let (ticks, cycles) = + run_scheduler_until(store, trigger_id, (at(6), test_case.launcher_poll)).await; let queued = queued_times(store, location).await; let earliest = ticks .iter() @@ -88,24 +93,26 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks "{}: later times are covered by the waiting run", test_case.description ); - let latencies: Vec = ticks - .iter() - .map(|tick| tick.evaluated_at - tick.scheduled_for) - .collect(); - println!("{} tick-to-decision: {latencies:?}", test_case.description); assert_eq!( - latencies.len(), + ticks.len(), test_case.expected_ticks, "{}", test_case.description ); - assert!( - latencies - .iter() - .all(|latency| *latency <= test_case.expected_max_tick_to_decision), - "{}: {latencies:?}", - test_case.description - ); + for tick in &ticks { + let (wake, cycle) = decision_wake(&cycles, tick.evaluated_at, tick.scheduled_for); + println!( + "{}: due {} woke {wake:?}, store {:?} to decision, cycle {cycle:?}", + test_case.description, + tick.scheduled_for, + tick.evaluated_at - cycle.timings.started_at, + ); + assert!( + wake <= test_case.expected_max_wake_after_due, + "{}: woke {wake:?} after the due time", + test_case.description + ); + } // The first leader stopped without releasing its lease while two more times fell due; // the standby catches up on the latest one only. // A standby is already polling when that lease lapses, so its own retry wait counts. @@ -116,8 +123,10 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks expire_lease(store).await; Utc::now() }; - let (ticks, available) = - tokio::join!(run_scheduler_until(&second, trigger_id, at(14)), lapse); + let ((ticks, cycles), available) = tokio::join!( + run_scheduler_until(&second, trigger_id, (at(14), Duration::ZERO)), + lapse + ); assert_eq!( ticks.len(), test_case.expected_ticks_after_failover, @@ -128,11 +137,15 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks .iter() .find(|tick| tick.scheduled_for == at(9)) .expect("the latest overdue time is decided"); - let catch_up = latest.evaluated_at - available; - println!("{} failover catch-up: {catch_up:?}", test_case.description); + let (wake, cycle) = decision_wake(&cycles, latest.evaluated_at, available); + println!( + "{} failover: woke {wake:?} after the lapse, store {:?} to decision, cycle {cycle:?}", + test_case.description, + latest.evaluated_at - cycle.timings.started_at, + ); assert!( - catch_up <= test_case.expected_max_failover_catch_up, - "{}: {catch_up:?}", + wake <= test_case.expected_max_failover_wake, + "{}: woke {wake:?} after the lapse", test_case.description ); close_store(second).await; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index f6cb23edc0..146b81d23b 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -36,6 +36,52 @@ impl crate::scheduler::types::RunLauncher for IdleLauncher { } } +/// A cycle's phase timings and when the launcher poll after it finished, freeing the loop to sleep. +#[derive(Clone, Debug)] +pub(crate) struct RecordedCycle { + pub(crate) timings: crate::scheduler::models::CycleTimings, + pub(crate) polled_at: chrono::DateTime, +} + +/// Records cycle timings; polls take `poll_delay` after a cycle that decided a trigger, as launches do. +pub(crate) struct CycleRecorder<'a> { + pub(crate) cycles: &'a mut Vec, + pub(crate) poll_delay: std::time::Duration, + pub(crate) launching: bool, +} + +impl crate::scheduler::types::RunLauncher for CycleRecorder<'_> { + async fn poll( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + if self.launching { + tokio::time::sleep(self.poll_delay).await; + } + if let Some(cycle) = self.cycles.last_mut() { + cycle.polled_at = chrono::Utc::now(); + } + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + Ok(()) + } + + fn cycled(&mut self, cycle: &crate::scheduler::models::SchedulerCycle) { + self.cycles.push(RecordedCycle { + timings: cycle.timings.clone(), + polled_at: cycle.timings.started_at, + }); + self.launching = cycle.evaluated > 0; + } +} + pub(crate) struct TickRetentionTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, @@ -88,6 +134,8 @@ pub(crate) struct SchedulerLatencyTestCase { pub(crate) backend: Backend, pub(crate) expected_ticks: usize, pub(crate) expected_ticks_after_failover: usize, - pub(crate) expected_max_tick_to_decision: chrono::TimeDelta, - pub(crate) expected_max_failover_catch_up: chrono::TimeDelta, + pub(crate) expected_max_wake_after_due: chrono::TimeDelta, + pub(crate) expected_max_failover_wake: chrono::TimeDelta, + /// Work after the cycle's next-due read, which must not delay the wake. + pub(crate) launcher_poll: std::time::Duration, } From 0b570ab429f06bc02f40dfe4666b753a82df8900 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:59:39 +0000 Subject: [PATCH 18/18] refactor: share target lookup and load-selection request reading Named-target checks reuse the compile attachment's resolved_target instead of a copy, the named-target schema binding declares its keyword arguments directly, and both load-selection bindings read their request through one helper. --- .../_helpers/compile_attachment/target.rs | 2 +- .../_helpers/compile_driver/targets.rs | 63 +++++-------------- .../bindings/_helpers/load/load_selection.rs | 59 +++++++++-------- fensu.toml | 4 ++ 4 files changed, 55 insertions(+), 73 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 34a83cb59a..a0a27b13ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -23,7 +23,7 @@ const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; /// The selected target's name and merged config, or `(None, None)`. -fn resolved_target<'py>( +pub(crate) fn resolved_target<'py>( discovered: &Bound<'py, PyAny>, selected_target: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index b67ff4e718..652b64cceb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -42,8 +42,8 @@ const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; +use crate::bindings::_helpers::compile_attachment::target::resolved_target; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::discovery::guidance::setting_help; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -148,26 +148,20 @@ fn project_setting_help( /// Require shared-warehouse named targets to choose a schema strategy. #[pyfunction] -#[pyo3(signature = (**options))] +#[pyo3(signature = (*, discovered_inputs, adapter, selected_target, cli_vars = None))] fn validate_named_target_schema_strategy<'py>( - py: Python<'py>, - options: Option>, + discovered_inputs: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, + cli_vars: Option>, ) -> Result<(), PyErr> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "validate_named_target_schema_strategy", - options, - ( - &["discovered_inputs", "adapter", "selected_target"], - &["cli_vars"], - ), - )?; + let py: Python<'py> = discovered_inputs.py(); named_target_schema_strategy( - &arguments.value("discovered_inputs")?, - &arguments.value("adapter")?, + discovered_inputs, + adapter, ( - &arguments.value("selected_target")?, - &arguments.value("cli_vars")?, + selected_target, + &cli_vars.unwrap_or_else(|| py.None().into_bound(py)), ), ) } @@ -182,10 +176,10 @@ pub(crate) fn named_target_schema_strategy( if native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? { return Ok(()); } - let Some((target_name, target_config)) = selected_target_config(discovered, selected_target)? - else { + let (target_name, target_config) = resolved_target(discovered, selected_target)?; + if target_name.is_none() { return Ok(()); - }; + } let schema: Option = text_or_none(&native_field(&target_config, "schema")?)?; if schema.is_some_and(|schema| !python_strip(&schema).is_empty()) { return Ok(()); @@ -647,9 +641,10 @@ pub(crate) fn preserved_loader_schemas( loaders: &HashSet, ) -> PyResult<()> { let py: Python<'_> = discovered.py(); - let Some((_, target_config)) = selected_target_config(discovered, selected_target)? else { + let (target_name, target_config) = resolved_target(discovered, selected_target)?; + if target_name.is_none() { return Ok(()); - }; + } if !text_or_none(&native_field(&target_config, "schema")?)? .is_some_and(|schema| python_strip(&schema) == PRESERVE_TARGET_VALUE) { @@ -676,30 +671,6 @@ pub(crate) fn preserved_loader_schemas( Ok(()) } -/// The selected target's name and configuration, or `None` when no named target is selected. -fn selected_target_config<'py>( - discovered: &Bound<'py, PyAny>, - selected_target: &Bound<'py, PyAny>, -) -> PyResult, Bound<'py, PyAny>)>> { - let py: Python<'py> = discovered.py(); - let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; - let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; - let target_name: Bound<'py, PyAny> = resolve_target_name( - py, - &project_config.clone(), - &local_config.clone(), - &selected_target.clone(), - ) - .and_then(|value| value.into_bound_py_any(py))?; - if target_name.is_none() { - return Ok(None); - } - let target_config: Bound<'py, PyAny> = - resolve_target_config(py, &project_config, &local_config, &target_name.clone()) - .and_then(|value| value.into_bound_py_any(py))?; - Ok(Some((target_name, target_config))) -} - /// Raise when neither the resource nor the project names a physical write schema. pub(crate) fn managed_write_schema( resource_label: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index df61fe16d4..88e816a2a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -459,11 +459,33 @@ fn names(value: &Bound<'_, PyAny>) -> PyResult> { value.extract() } -fn passed_defaults<'py>(arguments: &KeywordArguments<'py>) -> PyResult> { - Ok(LoaderDefaults { +/// A Python selection request's arguments, the project it names and its loader defaults. +fn selection_request<'py>( + py: Python<'py>, + (function, required): (&str, &[&str]), + options: Option>, +) -> PyResult<(KeywordArguments<'py>, LoadProject<'py>, LoaderDefaults<'py>)> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + function, + options, + ( + required, + &["loader_default_database", "loader_default_schema"], + ), + )?; + let project: LoadProject<'py> = LoadProject::new( + &arguments.value("discovered_inputs")?, + ( + &arguments.value("target_config")?, + &py.None().into_bound(py), + ), + )?; + let defaults: LoaderDefaults<'py> = LoaderDefaults { database: arguments.value("loader_default_database")?, schema: arguments.value("loader_default_schema")?, - }) + }; + Ok((arguments, project, defaults)) } /// Select source and intermediate loader execution entries for `sqb load`. @@ -473,26 +495,20 @@ fn select_load_entries_entry<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( + let (arguments, project, defaults) = selection_request( py, - "select_load_entries", - options, ( + "select_load_entries", &["discovered_inputs", "select", "exclude", "target_config"], - &["loader_default_database", "loader_default_schema"], ), - )?; - let target_config: Bound<'py, PyAny> = arguments.value("target_config")?; - let project: LoadProject<'py> = LoadProject::new( - &arguments.value("discovered_inputs")?, - (&target_config, &py.None().into_bound(py)), + options, )?; let entries: Vec> = project.selected_entries( ( &names(&arguments.value("select")?)?, &names(&arguments.value("exclude")?)?, ), - &passed_defaults(&arguments)?, + &defaults, )?; PyTuple::new(py, entries) } @@ -504,28 +520,19 @@ fn select_load_reference_entries<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( + let (arguments, project, defaults) = selection_request( py, - "select_load_reference_entries", - options, ( + "select_load_reference_entries", &["discovered_inputs", "selected_sources", "target_config"], - &["loader_default_database", "loader_default_schema"], ), - )?; - let target_config: Bound<'py, PyAny> = arguments.value("target_config")?; - let project: LoadProject<'py> = LoadProject::new( - &arguments.value("discovered_inputs")?, - (&target_config, &py.None().into_bound(py)), + options, )?; let selected: Vec> = arguments .value("selected_sources")? .try_iter()? .collect::>()?; - PyTuple::new( - py, - project.reference_entries(&selected, &passed_defaults(&arguments)?)?, - ) + PyTuple::new(py, project.reference_entries(&selected, &defaults)?) } /// The default database and schema intermediate loader tables are written to. diff --git a/fensu.toml b/fensu.toml index e7cd8dcc61..3f1d83277e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1441,6 +1441,10 @@ reason = "Each callback or worker report forwards one distinct build or check ev paths = ["crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs"] reason = "Two Python entry points with different keyword contracts, project and named target, delegate to project_connection_config." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs"] +reason = "Two Python entry points with different keyword contracts, selected and referenced loader entries, delegate to selection_request and LoadProject." + [[dupes.allowlist]] paths = ["crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs", "crates/sqlbuild-cli/src/native_commands/main/_location_work.rs"] reason = "Metadata sync keeps its pending page for retry and work polling drops the failed request; each reports its own failure."