diff --git a/benchmarks/duckshop/edge_cases/orchestration/direct_state_pruning.json b/benchmarks/duckshop/edge_cases/orchestration/direct_state_pruning.json new file mode 100644 index 0000000000..eda7dc2280 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/direct_state_pruning.json @@ -0,0 +1,5 @@ +{ + "description": "Direct state history for janitor pruning that keeps the newest two rows per key. run_001 and run_002 share an hour at the keep boundary, so the tie breaks on the newer run or result id.", + "runs": [["run_000", 10], ["run_001", 12], ["run_002", 12], ["run_003", 13]], + "audit_binding_keys": ["orders.valid_orders", "customers.unique_ids"] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json index 5be86e4798..ae0af98857 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json +++ b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json @@ -15,6 +15,15 @@ {"available": false, "source": "warehouse.db", "destination": "../retained-warehouse.db", "failure": "read", "tables": ["fingerprints"]}, {"available": true, "source": "../retained-warehouse.db", "destination": "warehouse.db", "failure": null, "tables": ["fingerprints", "source_freshness"]} ], + "held_write_lock": { + "duckdb": [ + { + "held": {"failure": "busy", "tables": ["source_freshness"]}, + "released": {"failure": null, "tables": ["fingerprints", "source_freshness"]} + } + ], + "sqlite": [] + }, "insert_sql": [ "INSERT INTO _sqlbuild_fingerprints SELECT node_type,node_name,target_database,target_schema,target_name,'{run_id}',definition_hash,version_hash,schema_fingerprint,definition_b64,metadata_json_b64,'{observed_at}' FROM _sqlbuild_fingerprints WHERE run_id='legacy-run'", "INSERT INTO _sqlbuild_source_freshness SELECT source_name,target_database,target_schema,target_name,'{run_id}',strategy,value_kind,data_version,data_version_hash,'{observed_at}' FROM _sqlbuild_source_freshness WHERE run_id='legacy-run'" diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ff3e5842bc..2548c57842 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -3277,7 +3277,7 @@ export interface components { * @description Closed failure stages keep warehouse errors and credentials out of health observations. * @enum {string} */ - WarehouseSyncFailure: "watermark" | "read" | "delivery"; + WarehouseSyncFailure: "watermark" | "read" | "delivery" | "busy"; WarehouseSyncResponse: { applied: boolean; }; diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index bc15cbc49d..97cddf35e6 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -211,6 +211,13 @@ "help": "immutable deployment image identity to pin for workers", "group": 1 }, + { + "kind": "store", + "options": ["--sync-at-least-every"], + "dest": "location_sync_at_least_every", + "help": "for serve, sync the warehouse at least this often, such as 30m; 0 or off turns it off", + "group": 1 + }, { "kind": "store", "options": ["--run-id"], diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 6fd859c0bb..6e85c29d77 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -10,6 +10,6 @@ pub mod diff_report; pub mod dispatch; pub mod launcher; pub mod native_commands; -pub(crate) mod person_login; +pub mod person_login; pub mod replay; pub(crate) mod workers; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index d6903fafd9..efec73492f 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -14,6 +14,10 @@ use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; use crate::adhoc::main::adhoc_planner::adhoc_planner; +use sqlbuild_config::project::main::load_control_config::load_control_config; +use sqlbuild_config::project::main::project_config_paths::project_config_paths; +use sqlbuild_config::project::main::sync_interval_seconds::sync_interval_seconds; +use sqlbuild_config::project::models::ControlConfig; use sqlbuild_orchestration::api::models::{ ApiAuthentication, DevelopmentScope, HumanLogin, ServerFacts, }; @@ -73,8 +77,8 @@ use crate::native_commands::_helpers::terminal::{ use crate::native_commands::constants::{ ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_OVERLAP_SECONDS, METADATA_POLL_SECONDS, METADATA_READER_CLOSE_SECONDS, METADATA_RECONCILE_SECONDS, - METADATA_REPLY_BYTES, METADATA_SYNC_CHECK_SECONDS, SCHEDULER_COMMAND, - UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, + METADATA_REPLY_BYTES, METADATA_SAFETY_SYNC_SECONDS, METADATA_SYNC_CHECK_SECONDS, + SAFETY_SYNC_ENV, SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; @@ -82,8 +86,8 @@ use crate::native_commands::main::_run_control::run_control; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, - LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, NativeCommand, - NativeInvocation, ProjectLocation, ScanToken, + LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, MetadataReply, + NativeCommand, NativeInvocation, ProjectLocation, ScanToken, }; use crate::native_commands::types::LocationPublication; use crate::workers::main::docker_image_sources::docker_image_sources; @@ -335,7 +339,7 @@ pub(crate) fn location_result( pub(crate) fn metadata_sync( (locations, generation): (Vec, i64), - reconcile_seconds: u32, + (reconcile_seconds, safety_seconds): (u32, u64), ) -> LocationMetadataSync { LocationMetadataSync { locations, @@ -355,6 +359,9 @@ pub(crate) fn metadata_sync( next_at: Instant::now(), idle: true, cycle_started_at: None, + safety_seconds, + safety_at: Instant::now() + Duration::from_secs(safety_seconds), + busy: false, change_tokens: HashMap::new(), scan_token: None, scan_delivered: false, @@ -483,15 +490,9 @@ pub(crate) async fn begin_after_run( Ok(due) => due, Err(error) => return (state, Err(error)), }; - let ran = match (state.cycle_started_at, due.runs_ended_at) { - (None, _) => true, - (Some(started_at), Some(ended_at)) => ended_at >= started_at, - (Some(_), None) => false, - }; + let ran = state.sync_due(due.runs_ended_at); if ran { - state.idle = false; - state.cycle_started_at = Some(due.now); - state.next_at = Instant::now(); + state.begin_sync(due.now); } (state, Ok(ran)) } @@ -589,6 +590,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync } state.idle = true; state.next_at = Instant::now(); + state.safety_at = Instant::now() + Duration::from_secs(state.safety_seconds); drop(write_output(&[( Stream::Stderr, "Warehouse history synchronization completed.\n", @@ -699,6 +701,28 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result Some(value), + Err(std::env::VarError::NotPresent) => None, + Err(std::env::VarError::NotUnicode(_)) => { + return Err(ControlError::Invalid(format!( + "{SAFETY_SYNC_ENV} must be text" + ))); + } + }; + let safety_sync_seconds = safety_sync_seconds( + text(namespace, "location_sync_at_least_every").as_deref(), + environment.as_deref(), + || { + load_control_config(&project_config_paths(&project_dir)).map_err(|failure| { + failure.into_config_failure().map_or_else( + || "sqlbuild_project.toml could not be read".to_owned(), + |failure| failure.render(|_| "sqlbuild_project.toml".to_owned()), + ) + }) + }, + ) + .map_err(ControlError::Invalid)?; Ok(LocationServe { project_dir, server_url: text(namespace, "location_server_url").unwrap_or_default(), @@ -706,9 +730,27 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result, + environment: Option<&str>, + project: impl FnOnce() -> Result, +) -> Result { + if let Some(text) = flag { + return sync_interval_seconds(text).map_err(|help| format!("--sync-at-least-every {help}")); + } + if let Some(text) = environment.filter(|text| !text.trim().is_empty()) { + return sync_interval_seconds(text).map_err(|help| format!("{SAFETY_SYNC_ENV} {help}")); + } + Ok(project()? + .sync_at_least_every_seconds + .unwrap_or(METADATA_SAFETY_SYNC_SECONDS)) +} + /// Reconcile from the server after restarts or lost acknowledgements; transient outages retry. pub(crate) async fn location_generation( options: &LocationServe, @@ -1152,6 +1194,7 @@ pub(crate) async fn read_metadata_page( mut state: LocationMetadataSync, stopped: impl Future>, ) -> Result)>, ControlError> { + state.busy = false; if let Some(statistics) = &state.statistics && statistics.names.is_empty() { @@ -1181,10 +1224,15 @@ pub(crate) async fn read_metadata_page( Duration::from_secs(METADATA_POLL_SECONDS), exchange_metadata_read(reader, request), ) => match result { - Ok((reader, Ok(page))) => { + Ok((reader, Ok(MetadataReply::Page(page)))) => { state.reader = Some(reader); Ok(Some((state, page))) } + Ok((reader, Ok(MetadataReply::Busy))) => { + state.busy = true; + state.reader = Some(reader); + Ok(Some((state, None))) + } Ok((reader, Err(_))) => { stop_location_runtime(Some(reader)).await?; Ok(Some((state, None))) @@ -1230,10 +1278,7 @@ pub(crate) async fn close_metadata_reader( async fn exchange_metadata_read( mut reader: LocationCompiler, request: Vec, -) -> ( - LocationCompiler, - Result, ControlError>, -) { +) -> (LocationCompiler, Result) { let result = async { reader.stdin.write_all(&request).await?; reader.stdin.write_all(b"\n").await?; @@ -1253,7 +1298,13 @@ async fn exchange_metadata_read( "location reply exceeds its byte limit".into(), )); } - serde_json::from_str(&line) + let reply: serde_json::Value = serde_json::from_str(&line) + .map_err(|_| ControlError::Invalid("invalid location reply".into()))?; + if reply.get("busy") == Some(&serde_json::Value::Bool(true)) { + return Ok(MetadataReply::Busy); + } + serde_json::from_value(reply) + .map(MetadataReply::Page) .map_err(|_| ControlError::Invalid("invalid location reply".into())) } .await; diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 4b7f7d30a8..a4ed2b6af9 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -98,6 +98,10 @@ pub(crate) const METADATA_READER_CLOSE_SECONDS: u64 = 5; pub(crate) const METADATA_SYNC_CHECK_SECONDS: u64 = 5; /// Full key-set reconciliation is independent of the incremental poll and overlap window. pub(crate) const METADATA_RECONCILE_SECONDS: u32 = 300; +/// An idle location syncs at least this often, so a lost run or request signal cannot leave drift. +pub(crate) const METADATA_SAFETY_SYNC_SECONDS: u64 = 1800; +/// Overrides `[control] sync_at_least_every`; `--sync-at-least-every` overrides both. +pub(crate) const SAFETY_SYNC_ENV: &str = "SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY"; pub(crate) const METADATA_OVERLAP_SECONDS: i64 = 300; /// A source-key cursor can repeat the last row's base64-encoded definition beside the records. pub(crate) const METADATA_REPLY_BYTES: u64 = diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index ea5ddaf6f2..d5ee1a5c47 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -78,16 +78,7 @@ pub(super) async fn sync_location_metadata( }; state = read; let Some(page) = page else { - tokio::select! { - result = &mut stopped => { result?; return Ok(None); }, - () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Read))) => {}, - } - state.retry_read(); - drop(write_output(&[( - Stream::Stderr, - "Warehouse history read failed; retaining mirror and retrying a bounded page.\n", - )])); - return Ok(Some(state)); + return failed_read(state, (client, poll), started, stopped.as_mut()).await; }; if location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed { @@ -123,6 +114,34 @@ pub(super) async fn sync_location_metadata( finish_metadata_page(acknowledge_metadata_page(state)).await } +/// A busy warehouse file waits a poll interval; any other failed read retries a smaller page. +async fn failed_read( + mut state: LocationMetadataSync, + (client, poll): (&LocationWorkClient, &LocationWorkPoll), + started: Instant, + stopped: impl Future>, +) -> Result, ControlError> { + let failure = if state.busy { + WarehouseSyncFailure::Busy + } else { + WarehouseSyncFailure::Read + }; + tokio::select! { + result = stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, Some(failure))) => {}, + } + if state.busy { + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + return Ok(Some(state)); + } + state.retry_read(); + drop(write_output(&[( + Stream::Stderr, + "Warehouse history read failed; retaining mirror and retrying a bounded page.\n", + )])); + Ok(Some(state)) +} + fn retry_metadata( mut state: LocationMetadataSync, error: ClientError, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index b97562018e..30e7528589 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -121,7 +121,10 @@ async fn watch_location( published_generation = Some(generation); metadata = Some(metadata_sync( (locations, generation), - options.metadata_reconcile_seconds, + ( + options.metadata_reconcile_seconds, + options.safety_sync_seconds, + ), )); let Some(files) = check_published_location( options, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 4123e2b9e1..f242f0d0f6 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -10,7 +10,8 @@ use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::LocationWorkRequest; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use sqlbuild_orchestration::warehouse_metadata::models::{ - MetadataBatch, StatisticsRead, WarehouseHistoryKind, WarehouseStateLocation, + MetadataBatch, StatisticsRead, WarehouseHistoryKind, WarehouseHistoryPage, + WarehouseStateLocation, }; use uuid::Uuid; @@ -149,6 +150,8 @@ pub(crate) struct LocationServe { pub(crate) target: String, pub(crate) image_id: Option, pub(crate) metadata_reconcile_seconds: u32, + /// Seconds between safety syncs of an idle location; `0` turns them off. + pub(crate) safety_sync_seconds: u64, } /// Retained source observations survive compilation; warehouse exclusions never replace them. @@ -209,6 +212,12 @@ pub(crate) struct LocationMetadataSync { pub(crate) idle: bool, /// Server time when the current or last synchronization began; `None` before the first. pub(crate) cycle_started_at: Option>, + /// Seconds between safety syncs; `0` turns them off. + pub(crate) safety_seconds: u64, + /// When an idle location syncs even though no run ended and nobody asked. + pub(crate) safety_at: tokio::time::Instant, + /// The last read found the warehouse file held by another process, such as a DuckDB build. + pub(crate) busy: bool, /// Per scan index: the table's change token after its last complete read. pub(crate) change_tokens: HashMap, /// The token read before the current scan's first page, recorded once the scan completes. @@ -221,6 +230,15 @@ pub(crate) struct LocationMetadataSync { pub(crate) reader: Option, } +/// The metadata reader's answer to one read. +#[derive(Debug)] +pub(crate) enum MetadataReply { + /// A page, or `None` when the read failed. + Page(Option), + /// Another process holds the warehouse file; nothing was read. + Busy, +} + /// A table's change token after a complete read, and what that read proves about the mirror. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct ScanToken { @@ -240,6 +258,23 @@ impl LocationMetadataSync { } } + /// Whether an idle location starts its one outstanding sync: first, requested, after a run, or safety. + pub(crate) fn sync_due(&self, runs_ended_at: Option>) -> bool { + let ran = match (self.cycle_started_at, runs_ended_at) { + (None, _) => true, + (Some(started_at), Some(ended_at)) => ended_at >= started_at, + (Some(_), None) => false, + }; + ran || (self.safety_seconds > 0 && tokio::time::Instant::now() >= self.safety_at) + } + + /// Start the sync `sync_due` asked for, at server time `now`. + pub(crate) fn begin_sync(&mut self, now: DateTime) { + self.idle = false; + self.cycle_started_at = Some(now); + self.next_at = tokio::time::Instant::now(); + } + /// Failed samples revisit catalog before retrying; no frozen delivery is discarded. pub(crate) fn retry_read(&mut self) { self.statistics = None; diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs index ed08157d2e..e8afdadad9 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -1,6 +1,6 @@ use std::path::Path; -use sqlbuild_orchestration::warehouse_metadata::models::MetadataBatch; +use sqlbuild_orchestration::warehouse_metadata::models::{MetadataBatch, WarehouseStateLocation}; use uuid::Uuid; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; @@ -60,3 +60,11 @@ pub(super) fn acknowledge_complete_pages( acknowledge_metadata_page(state) }) } + +/// One state location, for metadata sync tests. +pub(super) fn one_location() -> Vec { + vec![WarehouseStateLocation { + database: Some("analytics".into()), + schema: "state".into(), + }] +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index cb3a03c279..83282fde2c 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -3,9 +3,17 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryKind, WarehouseStateLocation, }; -use crate::native_commands::_helpers::execution::metadata_sync; -use crate::native_commands::tests::helpers::acknowledge_complete_pages; -use crate::native_commands::tests::test_types::{MetadataRetryTestCase, UnchangedScanTestCase}; +use chrono::Utc; +use sqlbuild_config::project::models::ControlConfig; +use std::time::Duration; +use tokio::time::Instant; + +use crate::native_commands::_helpers::execution::{metadata_sync, safety_sync_seconds}; +use crate::native_commands::tests::helpers::{acknowledge_complete_pages, one_location}; +use crate::native_commands::tests::test_types::{ + MetadataRetryTestCase, SafetySyncIdleTestCase, SafetySyncSettingTestCase, + SyncCoalescingTestCase, UnchangedScanTestCase, +}; #[test] fn given_failed_metadata_read_when_back_at_statistics_then_page_shrinks_until_scan_completes() { @@ -34,7 +42,7 @@ fn given_failed_metadata_read_when_back_at_statistics_then_page_shrinks_until_sc database: Some("analytics".into()), schema: "state".into(), }; - let mut state = metadata_sync((vec![location], 1), 3600); + let mut state = metadata_sync((vec![location], 1), (3600, 0)); state.index = WarehouseHistoryKind::ALL .iter() .position(|kind| *kind == test_case.failed_kind) @@ -133,7 +141,7 @@ fn given_recorded_change_token_when_scanning_again_then_only_proven_unchanged_re database: Some("analytics".into()), schema: "state".into(), }; - let mut state = metadata_sync((vec![location], 1), 3600); + let mut state = metadata_sync((vec![location], 1), (3600, 0)); state.reconcile = test_case.read_reconciled; state.scan_token = Some("t1".into()); state.scan_delivered = test_case.read_delivered; @@ -161,3 +169,192 @@ fn given_recorded_change_token_when_scanning_again_then_only_proven_unchanged_re ); } } + +#[test] +fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync_runs() { + let test_cases = [ + SyncCoalescingTestCase { + description: "four build requests and a run ending share one follow-up", + requests: 4, + run_ends_during_sync: true, + safety_seconds: 1800, + safety_due_after_seconds: 3600, + expected_follow_ups: 1, + }, + SyncCoalescingTestCase { + description: "a passed safety interval joins pending requests instead of adding a sync", + requests: 2, + run_ends_during_sync: false, + safety_seconds: 1800, + safety_due_after_seconds: 0, + expected_follow_ups: 1, + }, + SyncCoalescingTestCase { + description: "a safety interval that passes during a sync adds no extra one", + requests: 0, + run_ends_during_sync: false, + safety_seconds: 1800, + safety_due_after_seconds: 0, + expected_follow_ups: 0, + }, + SyncCoalescingTestCase { + description: "no signal leaves the location idle", + requests: 0, + run_ends_during_sync: false, + safety_seconds: 1800, + safety_due_after_seconds: 3600, + expected_follow_ups: 0, + }, + ]; + for test_case in test_cases { + let started = Utc::now(); + let mut state = metadata_sync((one_location(), 1), (3600, test_case.safety_seconds)); + assert!( + state.sync_due(None), + "{}: the first sync is due", + test_case.description + ); + state.begin_sync(started); + for _ in 0..test_case.requests { + state.request_sync(); + } + let runs_ended_at = test_case + .run_ends_during_sync + .then(|| started + chrono::Duration::seconds(1)); + state.safety_at = Instant::now() + Duration::from_secs(test_case.safety_due_after_seconds); + state = acknowledge_complete_pages(state, WarehouseHistoryKind::ALL.len()); + assert!( + state.idle, + "{}: the first sync completed", + test_case.description + ); + let mut polled = Some(state); + let follow_ups = (1..=5) + .take_while(|poll| { + let mut current = polled.take().expect("one state between polls"); + let due = current.sync_due(runs_ended_at); + current.begin_sync(started + chrono::Duration::seconds(10 * poll)); + polled = Some(acknowledge_complete_pages( + current, + WarehouseHistoryKind::ALL.len(), + )); + due + }) + .count(); + assert_eq!( + follow_ups, test_case.expected_follow_ups, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_an_idle_location_when_the_safety_interval_passes_then_it_syncs_unless_turned_off() { + let test_cases = [ + SafetySyncIdleTestCase { + description: "the interval passed while idle", + safety_seconds: 1800, + safety_due_after_seconds: 0, + expected_sync_due: true, + }, + SafetySyncIdleTestCase { + description: "the interval has not passed", + safety_seconds: 1800, + safety_due_after_seconds: 3600, + expected_sync_due: false, + }, + SafetySyncIdleTestCase { + description: "0 turns safety syncs off", + safety_seconds: 0, + safety_due_after_seconds: 0, + expected_sync_due: false, + }, + ]; + let started = Utc::now(); + for test_case in test_cases { + let mut state = metadata_sync((one_location(), 1), (3600, test_case.safety_seconds)); + state.begin_sync(started); + state = acknowledge_complete_pages(state, WarehouseHistoryKind::ALL.len()); + state.safety_at = Instant::now() + Duration::from_secs(test_case.safety_due_after_seconds); + assert_eq!( + state.sync_due(None), + test_case.expected_sync_due, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_safety_sync_settings_when_resolving_then_flag_beats_environment_beats_project() { + let test_cases = [ + SafetySyncSettingTestCase { + description: "nothing set uses thirty minutes", + flag: None, + environment: None, + project: None, + expected_seconds: Ok(1800), + }, + SafetySyncSettingTestCase { + description: "the project file sets it", + flag: None, + environment: None, + project: Some(3600), + expected_seconds: Ok(3600), + }, + SafetySyncSettingTestCase { + description: "the environment overrides the project file", + flag: None, + environment: Some("2h"), + project: Some(3600), + expected_seconds: Ok(7200), + }, + SafetySyncSettingTestCase { + description: "a blank environment value falls through to the project file", + flag: None, + environment: Some(" "), + project: Some(3600), + expected_seconds: Ok(3600), + }, + SafetySyncSettingTestCase { + description: "the flag overrides the environment", + flag: Some("off"), + environment: Some("2h"), + project: Some(3600), + expected_seconds: Ok(0), + }, + SafetySyncSettingTestCase { + description: "a flag under a minute is refused", + flag: Some("30s"), + environment: None, + project: None, + expected_seconds: Err( + "--sync-at-least-every must be a duration of at least '1m' such as '30m', or '0' or 'off' to turn safety syncs off", + ), + }, + SafetySyncSettingTestCase { + description: "an unparseable environment value is refused", + flag: None, + environment: Some("-5m"), + project: None, + expected_seconds: Err( + "SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY must be a duration of at least '1m' such as '30m', or '0' or 'off' to turn safety syncs off", + ), + }, + ]; + for test_case in test_cases { + let resolved = safety_sync_seconds(test_case.flag, test_case.environment, || { + Ok(ControlConfig { + sync_at_least_every_seconds: test_case.project, + ..ControlConfig::default() + }) + }); + assert_eq!( + resolved, + test_case.expected_seconds.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 46d3cc8389..8282a11eaf 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -111,3 +111,32 @@ pub(super) struct UnchangedScanTestCase { pub(super) mirror_holds_rows: bool, pub(super) expected_token: Option<&'static str>, } + +pub(super) struct SyncCoalescingTestCase { + pub(super) description: &'static str, + /// Sync requests that arrive while one sync is in progress, such as one per CLI build. + pub(super) requests: usize, + pub(super) run_ends_during_sync: bool, + /// Seconds between safety syncs; `0` turns them off. + pub(super) safety_seconds: u64, + /// Seconds until the safety interval passes, set while the sync runs; `0` passes during it. + pub(super) safety_due_after_seconds: u64, + pub(super) expected_follow_ups: usize, +} + +pub(super) struct SafetySyncSettingTestCase { + pub(super) description: &'static str, + pub(super) flag: Option<&'static str>, + pub(super) environment: Option<&'static str>, + pub(super) project: Option, + pub(super) expected_seconds: Result, +} + +/// An idle location after one complete sync, and whether its safety interval makes a sync due. +pub(super) struct SafetySyncIdleTestCase { + pub(super) description: &'static str, + /// Seconds between safety syncs; `0` turns them off. + pub(super) safety_seconds: u64, + pub(super) safety_due_after_seconds: u64, + pub(super) expected_sync_due: bool, +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/build_sync.rs b/crates/sqlbuild-cli/src/person_login/_helpers/build_sync.rs new file mode 100644 index 0000000000..300fe441e4 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/build_sync.rs @@ -0,0 +1,96 @@ +//! The post-build sync's phases: read the configured location, sign in, and ask for the sync. + +use std::env::VarError; +use std::path::{Path, PathBuf}; + +use sqlbuild_config::project::main::load_control_config::load_control_config; +use sqlbuild_config::project::main::project_config_paths::project_config_paths; +use sqlbuild_orchestration::api::main::parse_origin::parse_origin; + +use crate::person_login::_helpers::{credentials, transport}; +use crate::person_login::constants::{ + CONTROL_CA_CERT_ENV, CONTROL_LOCATION_ENV, CONTROL_SERVER_ENV, +}; +use crate::person_login::models::{PersonClient, PersonCredential}; + +/// The configured server and location name, from the environment or `[control]`; `None` when neither is set. +pub(crate) fn configured_location( + project_dir: &Path, +) -> Result, std::io::Error> { + let configured = load_control_config(&project_config_paths(project_dir)) + .map_err(|_| std::io::Error::other("sqlbuild_project.toml [control] could not be read."))?; + match ( + setting(CONTROL_SERVER_ENV, configured.server)?, + setting(CONTROL_LOCATION_ENV, configured.location)?, + ) { + (Some(server), Some(location)) => Ok(Some((server, location))), + (None, None) => Ok(None), + _ => Err(std::io::Error::other( + "set both [control] server and location, or neither.", + )), + } +} + +fn setting(name: &str, configured: Option) -> Result, std::io::Error> { + match std::env::var(name) { + Ok(value) if !value.trim().is_empty() => Ok(Some(value)), + Ok(_) | Err(VarError::NotPresent) => Ok(configured), + Err(VarError::NotUnicode(_)) => Err(std::io::Error::other(format!("{name} must be text."))), + } +} + +/// Signs in with the stored session and asks for the sync; `None` when nobody is signed in to the server. +pub(crate) fn sign_in_and_sync( + (server, location): (String, String), + target: &str, +) -> Result, std::io::Error> { + let origin = parse_origin(&server, "CLI").map_err(std::io::Error::other)?; + let Some(credential) = credentials::load(&origin)? else { + return Ok(None); + }; + let ca = std::env::var_os(CONTROL_CA_CERT_ENV).map(PathBuf::from); + let client = transport::connect(&origin, ca.as_deref())?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime + .block_on(sync_location_target( + (&client, &credential), + (&location, target), + &origin, + )) + .map(Some) +} + +async fn sync_location_target( + (client, credential): (&PersonClient, &PersonCredential), + (location, target): (&str, &str), + origin: &str, +) -> Result { + let targets = transport::location_targets(client, credential).await?; + let Some(found) = targets + .iter() + .find(|candidate| candidate.name == location && candidate.target == target) + else { + let named = targets.iter().any(|candidate| candidate.name == location); + return Ok(if named { + format!( + "Warehouse sync skipped: location '{location}' has no target '{target}' you can view on {origin}.\n" + ) + } else { + format!( + "Warehouse sync skipped: no location named '{location}' that you can view on {origin}.\n" + ) + }); + }; + transport::warehouse_sync( + client, + credential, + (found.location_id, target), + "Requested after sqb build", + ) + .await?; + Ok(format!( + "Warehouse sync requested for location '{location}' target '{target}'.\n" + )) +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs b/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs index 11f145944a..3cbc2b7b34 100644 --- a/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(super) mod build_sync; pub(super) mod credentials; pub(super) mod flow; pub(super) mod transport; diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs index 7703fe34ff..e6bdf19a7f 100644 --- a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs +++ b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs @@ -13,7 +13,7 @@ use uuid::Uuid; use crate::person_login::_helpers::credentials; use crate::person_login::constants::REQUEST_SECONDS; -use crate::person_login::models::{PersonClient, PersonCredential}; +use crate::person_login::models::{Bootstrap, LocationTarget, PersonClient, PersonCredential}; pub(crate) fn connect(origin: &str, ca: Option<&Path>) -> Result { let mut builder = reqwest::Client::builder() @@ -139,11 +139,12 @@ pub(crate) async fn warehouse_sync( client: &PersonClient, credential: &PersonCredential, (location, target): (Uuid, &str), + reason: &str, ) -> Result { let path = target_path(client, (location, target), &["warehouse-syncs"])?; let request = PersonWorkRequest { request_id: Uuid::now_v7(), - reason: "Requested from the CLI".into(), + reason: reason.into(), }; send_json::( authorized(client, reqwest::Method::POST, &path, credential)?.json(&request), @@ -157,6 +158,24 @@ pub(crate) async fn warehouse_sync( .map(|response| response.applied) } +/// The location targets the signed-in person can view. +pub(crate) async fn location_targets( + client: &PersonClient, + credential: &PersonCredential, +) -> Result, std::io::Error> { + send_json::( + authorized( + client, + reqwest::Method::GET, + "/api/control/bootstrap", + credential, + )?, + ("read locations from", "Location lookup", "location list"), + ) + .await + .map(|bootstrap| bootstrap.access.locations) +} + /// Each failure names the action, e.g. "could not read published SQL from the server". async fn send_json( request: reqwest::RequestBuilder, diff --git a/crates/sqlbuild-cli/src/person_login/constants.rs b/crates/sqlbuild-cli/src/person_login/constants.rs index d525dc760e..5b55c33035 100644 --- a/crates/sqlbuild-cli/src/person_login/constants.rs +++ b/crates/sqlbuild-cli/src/person_login/constants.rs @@ -5,3 +5,9 @@ pub(super) const POLL_SECONDS: u64 = 1; pub(super) const LOGIN_SECONDS: u64 = 300; pub(super) const REQUEST_SECONDS: u64 = 15; pub(super) const KEYRING_SERVICE: &str = "sqlbuild-cli"; +/// Overrides `[control] server` and `[control] location`; an extra trusted CA for the server. +pub(super) const CONTROL_SERVER_ENV: &str = "SQLBUILD_CONTROL_SERVER"; +pub(super) const CONTROL_LOCATION_ENV: &str = "SQLBUILD_CONTROL_LOCATION"; +pub(super) const CONTROL_CA_CERT_ENV: &str = "SQLBUILD_CONTROL_CA_CERT"; +/// A post-build sync request gives up after this long so the build's exit is never held. +pub(super) const BUILD_SYNC_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-cli/src/person_login/main/mod.rs b/crates/sqlbuild-cli/src/person_login/main/mod.rs index db72101ec9..ce45b29160 100644 --- a/crates/sqlbuild-cli/src/person_login/main/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/main/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod read_node_sql; pub(crate) mod request_warehouse_sync; pub(crate) mod run_login; +pub mod sync_after_build; diff --git a/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs b/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs index 28f04804a5..3f51a28e27 100644 --- a/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs +++ b/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs @@ -11,6 +11,12 @@ pub(crate) fn request_warehouse_sync( (location, target): (Uuid, &str), ) -> Result { transport::with_person_session(server, async |client, credential| { - transport::warehouse_sync(client, credential, (location, target)).await + transport::warehouse_sync( + client, + credential, + (location, target), + "Requested from the CLI", + ) + .await }) } diff --git a/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs b/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs new file mode 100644 index 0000000000..676fa0721f --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs @@ -0,0 +1,36 @@ +//! A signed-in `sqb build` asks its configured location to refresh the warehouse copy, best-effort. + +use std::path::Path; +use std::sync::mpsc::RecvTimeoutError; +use std::time::Duration; + +use crate::person_login::_helpers::build_sync::{configured_location, sign_in_and_sync}; +use crate::person_login::constants::BUILD_SYNC_SECONDS; + +/// One stderr line about the sync, or `None` when no location is configured or nobody is signed in. +pub fn sync_after_build(project_dir: &Path, target: &str) -> Option { + let (sender, receiver) = std::sync::mpsc::channel(); + let (project_dir, target) = (project_dir.to_path_buf(), target.to_owned()); + drop(std::thread::spawn(move || { + drop(sender.send(request_sync(&project_dir, &target))); + })); + match receiver.recv_timeout(Duration::from_secs(BUILD_SYNC_SECONDS)) { + Ok(line) => line, + Err(RecvTimeoutError::Timeout) => Some(format!( + "Warehouse sync skipped: the request did not finish within {BUILD_SYNC_SECONDS} seconds.\n" + )), + Err(RecvTimeoutError::Disconnected) => None, + } +} + +fn request_sync(project_dir: &Path, target: &str) -> Option { + let outcome = match configured_location(project_dir) { + Ok(Some(configured)) => sign_in_and_sync(configured, target), + Ok(None) => return None, + Err(error) => Err(error), + }; + match outcome { + Ok(line) => line, + Err(error) => Some(format!("Warehouse sync skipped: {error}\n")), + } +} diff --git a/crates/sqlbuild-cli/src/person_login/mod.rs b/crates/sqlbuild-cli/src/person_login/mod.rs index 1c79f915ed..8ab8e70044 100644 --- a/crates/sqlbuild-cli/src/person_login/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/mod.rs @@ -2,5 +2,5 @@ pub(crate) mod _helpers; pub(crate) mod constants; -pub(crate) mod main; +pub mod main; pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/person_login/models.rs b/crates/sqlbuild-cli/src/person_login/models.rs index ddcc46e8da..6b973f418b 100644 --- a/crates/sqlbuild-cli/src/person_login/models.rs +++ b/crates/sqlbuild-cli/src/person_login/models.rs @@ -55,3 +55,21 @@ pub(super) struct PersonClient { pub(super) client: reqwest::Client, pub(super) origin: String, } + +/// The parts of `GET /api/control/bootstrap` that name the location targets a person can view. +#[derive(Debug, Deserialize)] +pub(super) struct Bootstrap { + pub(super) access: BootstrapAccess, +} + +#[derive(Debug, Deserialize)] +pub(super) struct BootstrapAccess { + pub(super) locations: Vec, +} + +#[derive(Debug, Deserialize)] +pub(super) struct LocationTarget { + pub(super) location_id: uuid::Uuid, + pub(super) name: String, + pub(super) target: String, +} diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 3ddb50b887..0521263158 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -14,9 +14,9 @@ use crate::project::_helpers::help::{ }; use crate::project::_helpers::keys::Place; use crate::project::_helpers::sections::{ - load_constants, load_cost, load_cursors, load_dbt, load_diff, load_git, load_janitor, - load_local_dbt, load_microbatches, load_migrations, load_policy_flag, load_review, - load_scenario, load_settings, load_snapshots, setting_overrides, + load_constants, load_control, load_cost, load_cursors, load_dbt, load_diff, load_git, + load_janitor, load_local_dbt, load_microbatches, load_migrations, load_policy_flag, + load_review, load_scenario, load_settings, load_snapshots, setting_overrides, }; use crate::project::_helpers::sinks::load_sinks; use crate::project::_helpers::targets::{ @@ -28,7 +28,7 @@ use crate::project::constants::{ LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; -use crate::project::models::{Built, ConfigFile, ConfigPaths, ReviewConfig}; +use crate::project::models::{Built, ConfigFile, ConfigPaths, ControlConfig, ReviewConfig}; use crate::project::types::{Entries, Fields, Loaded}; use crate::recall::constants::RECALL_SECTION; use crate::recall::main::recall_section::recall_section; @@ -278,6 +278,7 @@ fn project_fields(payload: &Entries) -> Loaded { } load_git(get(payload, "git"), PROJECT_FILE)?; load_review(get(payload, "review"), PROJECT_FILE)?; + load_control(get(payload, "control"), PROJECT_FILE)?; let mut fields = identity(payload)?; fields.extend(sections(payload)?); fields.push(("targets", targets(payload)?)); @@ -426,20 +427,31 @@ pub(crate) fn local_sql_analysis_failure( ) } -/// `[git] base_ref` of `sqlbuild_project.toml`, validated as project loading validates it. -pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { +/// One section of `sqlbuild_project.toml`, validated as project loading validates it. +fn project_section( + paths: &ConfigPaths, + section: &str, + load: fn(Option<&ConfigValue>, ConfigFile) -> Loaded, +) -> Result { let file = resolve(paths, false)?; let table = load_toml_mapping(paths, file)?; let payload = table.as_map().unwrap_or_default(); - Ok(load_git(get(payload, "git"), PROJECT_FILE)?) + Ok(load(get(payload, section), PROJECT_FILE)?) +} + +/// `[git] base_ref` of `sqlbuild_project.toml`. +pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { + project_section(paths, "git", load_git) } -/// `[review]` of `sqlbuild_project.toml`, validated as project loading validates it. +/// `[control]` of `sqlbuild_project.toml`. +pub(crate) fn control_config(paths: &ConfigPaths) -> Result { + project_section(paths, "control", load_control) +} + +/// `[review]` of `sqlbuild_project.toml`. pub(crate) fn review_config(paths: &ConfigPaths) -> Result { - let file = resolve(paths, false)?; - let table = load_toml_mapping(paths, file)?; - let payload = table.as_map().unwrap_or_default(); - Ok(load_review(get(payload, "review"), PROJECT_FILE)?) + project_section(paths, "review", load_review) } /// One warning line per configuration file whose `[settings]` still sets `column_contract_mode`. diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 31016fb3df..2bceff46dc 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -22,12 +22,15 @@ use crate::project::constants::{ SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; +use crate::project::constants::{ + CONTROL_KEYS, MIN_SYNC_INTERVAL_SECONDS, SYNC_INTERVAL_HELP, SYNC_INTERVAL_OFF, +}; use crate::project::constants::{ DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, }; use crate::project::errors::ConfigFailure; -use crate::project::models::{Built, ConfigFile, ReviewConfig}; +use crate::project::models::{Built, ConfigFile, ControlConfig, ReviewConfig}; use crate::project::types::{Entries, Fields, Loaded}; use sqlbuild_core::constants::ZERO_DAY_DURATION; @@ -571,6 +574,56 @@ pub(crate) fn load_git(payload: Option<&ConfigValue>, file: ConfigFile) -> Loade }) } +/// A safety sync interval in seconds, `0` when off; the error completes " ...". +pub(crate) fn sync_interval(text: &str) -> Result { + if SYNC_INTERVAL_OFF.contains(&text) { + return Ok(0); + } + let seconds: u64 = match fixed_duration_seconds(text, false) { + Ok(Some(seconds)) => seconds.parse().map_err(|_| SYNC_INTERVAL_HELP)?, + Ok(None) | Err(_) => return Err(SYNC_INTERVAL_HELP), + }; + if seconds < MIN_SYNC_INTERVAL_SECONDS { + return Err(SYNC_INTERVAL_HELP); + } + Ok(seconds) +} + +/// `[control]`: where a signed-in build asks for a sync, and how often a location syncs anyway. +pub(crate) fn load_control( + payload: Option<&ConfigValue>, + file: ConfigFile, +) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "control"))?; + validate_allowed_keys(mapping, &CONTROL_KEYS, place(file, "control"))?; + let server = optional_str(mapping, "server").map_err(|_| { + ConfigFailure::at( + file, + "control.server must be an HTTPS server origin such as 'https://sqlbuild.example.com'", + ) + })?; + let location = optional_str(mapping, "location").map_err(|_| { + ConfigFailure::at(file, "control.location must be a non-empty location name") + })?; + let sync_at_least_every_seconds = match get(mapping, "sync_at_least_every") { + None => None, + Some(value) => Some( + match value { + ConfigValue::String(text) => sync_interval(text), + _ => Err(SYNC_INTERVAL_HELP), + } + .map_err(|help| { + ConfigFailure::at(file, format!("control.sync_at_least_every {help}")) + })?, + ), + }; + Ok(ControlConfig { + server, + location, + sync_at_least_every_seconds, + }) +} + /// `[review]`: the defaults of `sqb review`, validated so a typo fails like any other section. pub(crate) fn load_review(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { let mapping = coerce_mapping(payload, place(file, "review"))?; diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 753734ebd8..987a3563ea 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -200,6 +200,12 @@ pub(crate) const TARGET_KEYS: [&str; 21] = [ "recall", "concurrency", ]; +pub(crate) const CONTROL_KEYS: [&str; 3] = ["location", "server", "sync_at_least_every"]; +/// Shortest safety sync interval, so a typo cannot bring back aggressive warehouse polling. +pub(crate) const MIN_SYNC_INTERVAL_SECONDS: u64 = 60; +pub(crate) const SYNC_INTERVAL_OFF: [&str; 3] = ["0", "0s", "off"]; +pub(crate) const SYNC_INTERVAL_HELP: &str = + "must be a duration of at least '1m' such as '30m', or '0' or 'off' to turn safety syncs off"; pub(crate) const REVIEW_KEYS: [&str; 4] = ["against", "critical_tags", "downstream", "schema_prefix"]; pub(crate) const REVIEW_DOWNSTREAM_POLICIES: [&str; 3] = ["critical", "all", "none"]; diff --git a/crates/sqlbuild-config/src/project/main/load_control_config.rs b/crates/sqlbuild-config/src/project/main/load_control_config.rs new file mode 100644 index 0000000000..0bb195b57a --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/load_control_config.rs @@ -0,0 +1,9 @@ +//! Read `[control]` from `sqlbuild_project.toml`. + +use crate::project::errors::LoadFailure; +use crate::project::models::{ConfigPaths, ControlConfig}; + +/// The configured server, location and safety sync interval; every field is `None` when unset. +pub fn load_control_config(paths: &ConfigPaths) -> Result { + crate::project::_helpers::documents::control_config(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 8cd4f858ed..7d1a244037 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -1,5 +1,6 @@ pub mod deprecated_setting_warnings; pub mod explicit_references_help; +pub mod load_control_config; pub mod load_git_base_ref; pub mod load_local_config; pub mod load_project_config; @@ -11,4 +12,5 @@ pub mod resolve_config_file; pub mod setting_help_text; pub mod snippet_help_text; pub mod sql_analysis_off_guidance; +pub mod sync_interval_seconds; pub mod unsupported_target_warehouse; diff --git a/crates/sqlbuild-config/src/project/main/sync_interval_seconds.rs b/crates/sqlbuild-config/src/project/main/sync_interval_seconds.rs new file mode 100644 index 0000000000..e06a803f1a --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/sync_interval_seconds.rs @@ -0,0 +1,6 @@ +//! Parse a safety sync interval the way `[control] sync_at_least_every` accepts it. + +/// Whole seconds, `0` for `"0"`, `"0s"` or `"off"`; the error completes " ...". +pub fn sync_interval_seconds(text: &str) -> Result { + crate::project::_helpers::sections::sync_interval(text) +} diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index a36260035b..3c9b9b461e 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -56,6 +56,17 @@ pub struct SettingSite<'a> { pub key: &'a str, } +/// `[control]` of `sqlbuild_project.toml`: the server and location this project syncs with. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ControlConfig { + /// HTTPS origin of the SQLBuild server, as `sqb login --server` takes it. + pub server: Option, + /// Name of the location whose warehouse copy a signed-in `sqb build` refreshes. + pub location: Option, + /// Seconds between safety syncs of `sqb location serve`; `Some(0)` turns them off. + pub sync_at_least_every_seconds: Option, +} + /// `[review]` of `sqlbuild_project.toml`: the defaults `sqb review` uses when flags are omitted. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ReviewConfig { diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index e556567d29..7b57c16140 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,11 +1,13 @@ use crate::project::errors::ConfigFailure; +use crate::project::main::load_control_config::load_control_config; use crate::project::main::load_git_base_ref::load_git_base_ref; use crate::project::main::load_review_config::load_review_config; use crate::project::models::ConfigFile; -use crate::project::models::ReviewConfig; +use crate::project::models::{ControlConfig, ReviewConfig}; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; use crate::project::tests::test_types::{ - GitBaseRefTestCase, ProjectFailureTestCase, ReviewConfigErrorTestCase, ReviewConfigTestCase, + ControlConfigTestCase, GitBaseRefTestCase, ProjectFailureTestCase, ReviewConfigErrorTestCase, + ReviewConfigTestCase, }; const PROJECT: &str = "name = 'orders'\nadapter = 'duckdb'\n"; @@ -261,3 +263,114 @@ fn given_invalid_review_section_when_reading_then_it_is_rejected() { ); } } + +#[test] +fn given_control_section_when_reading_then_returns_its_values_and_interval_seconds() { + let control = |sync: &str| ControlConfig { + server: Some("https://sqlbuild.example.com".to_owned()), + location: Some("orders-warehouse".to_owned()), + sync_at_least_every_seconds: sync.parse().ok(), + }; + let test_cases = [ + ControlConfigTestCase { + description: "every control key with a compound interval", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\n\ + server = 'https://sqlbuild.example.com'\nlocation = 'orders-warehouse'\n\ + sync_at_least_every = '1h30m'\n", + expected_config: control("5400"), + }, + ControlConfigTestCase { + description: "exactly the one-minute minimum", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\n\ + server = 'https://sqlbuild.example.com'\nlocation = 'orders-warehouse'\n\ + sync_at_least_every = '60s'\n", + expected_config: control("60"), + }, + ControlConfigTestCase { + description: "off turns safety syncs off", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\n\ + server = 'https://sqlbuild.example.com'\nlocation = 'orders-warehouse'\n\ + sync_at_least_every = 'off'\n", + expected_config: control("0"), + }, + ControlConfigTestCase { + description: "a bare zero turns safety syncs off", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\n\ + server = 'https://sqlbuild.example.com'\nlocation = 'orders-warehouse'\n\ + sync_at_least_every = '0'\n", + expected_config: control("0"), + }, + ControlConfigTestCase { + description: "no control section", + project: PROJECT, + expected_config: ControlConfig::default(), + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let config: ControlConfig = load_control_config(&paths).expect(test_case.description); + + assert_eq!( + config, test_case.expected_config, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_control_section_when_loading_then_project_loading_rejects_it() { + let interval = "control.sync_at_least_every must be a duration of at least '1m'"; + let test_cases = [ + ReviewConfigErrorTestCase { + description: "an interval one second under the minimum", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = '59s'\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "a negative interval", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = '-30m'\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "an interval that is not a duration", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = 'hourly'\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "a calendar interval", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = '1mo'\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "an integer interval", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = 1800\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "a misspelt control key", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nlocaton = 'orders-warehouse'\n", + expected_fragment: "control contains unknown key(s): locaton", + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let failures: [String; 2] = [ + format!( + "{:?}", + load_control_config(&paths).expect_err(test_case.description) + ), + format!("{:?}", first_failure(&paths).expect(test_case.description)), + ]; + + for failure in failures { + assert!( + failure.contains(test_case.expected_fragment), + "{}: {failure}", + test_case.description + ); + } + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index e4dda5fb4e..66f541daa4 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::project::errors::ConfigFailure; -use crate::project::models::ReviewConfig; +use crate::project::models::{ControlConfig, ReviewConfig}; pub(super) struct ProjectFailureTestCase { pub(super) description: &'static str, @@ -21,6 +21,12 @@ pub(crate) struct GitBaseRefTestCase { pub(crate) expected_base_ref: Option<&'static str>, } +pub(crate) struct ControlConfigTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) expected_config: ControlConfig, +} + pub(crate) struct ReviewConfigTestCase { pub(crate) description: &'static str, pub(crate) project: &'static str, diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index fa1e4427f7..cfa9eb6e03 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -1,4 +1,4 @@ -//! Named work requires a trusted owner or live person authority; retries preserve the original pin. +//! Named work requires a trusted owner or live person authority; a pending sync covers a repeated one. use chrono::Duration; use uuid::Uuid; @@ -11,7 +11,8 @@ use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub async fn request_location_work( store: &ControlStore, @@ -56,6 +57,20 @@ pub async fn request_location_work( commit(transaction).await?; return Ok(false); } + let transaction = if matches!(request.work, LocationWork::SyncWarehouse {}) { + let now = transaction.now; + let (transaction, pending) = query(transaction, + "SELECT request_id FROM sqb_location_work WHERE location_id=$1 AND target=$2 \ + AND completed_at IS NULL AND expires_at>$3 AND request_json->'work'->>'kind'='sync_warehouse' LIMIT 1", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Time(now)]).await?; + if !pending.is_empty() { + commit(transaction).await?; + return Ok(false); + } + transaction + } else { + transaction + }; commit(enqueue_location_work(transaction, &context, request, Duration::minutes(5)).await?) .await?; Ok(true) diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 541216c440..77f1d5ad40 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -47,6 +47,8 @@ pub enum WarehouseSyncFailure { Watermark, Read, Delivery, + /// Another process, such as a DuckDB build, holds the warehouse file; the sync waits for it. + Busy, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs index 0e5058430d..5e6e61e5a1 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs @@ -15,6 +15,7 @@ use crate::credentials::models::{ }; use crate::credentials::tests::location_helpers::create_location; use crate::platform::main::{ + read_location_work::read_location_work, read_person_connection_test::read_person_connection_test, request_location_work::request_location_work, }; @@ -22,7 +23,10 @@ use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::runs::tests::helpers::snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; +use crate::store::models::TransactionMode; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; @@ -141,6 +145,28 @@ fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_preced .await .expect("connection-test authority requests a sync") ); + let repeated = LocationWorkRequest { + request_id: Uuid::now_v7(), + reason: "Requested after sqb build".into(), + ..sync.clone() + }; + assert!( + !request_location_work(store, (&context, Some(credentials)), &repeated) + .await + .expect("a second sync while one is pending"), + "a sync already pending for the target answers as applied" + ); + let (transaction, stored) = read_location_work( + begin(store, TransactionMode::Read).await.expect("read"), + repeated.request_id, + ) + .await + .expect("read work"); + commit(transaction).await.expect("commit"); + assert!( + stored.is_none(), + "the coalesced sync request leaves no second pending work row" + ); assert!( read_person_connection_test( store, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 15c8a81788..98cff67d73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -5,6 +5,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cli::person_login::main::sync_after_build::sync_after_build; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::boundary::python_calls::{call_with, within}; @@ -24,6 +25,7 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::observations::source_capture::{ arm_source_capture, take_source_capture, }; +use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::records::fields::field as native_field; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { @@ -116,7 +118,26 @@ fn run_within_limits( ("remediation", native_field(&limits, "remediation")?), ], )?; - within(&deadline, |_| run_with_providers(&context)) + let exit_code: i32 = within(&deadline, |_| run_with_providers(&context))?; + if context.worker.is_none() { + let _ = request_location_sync(&context); + } + Ok(exit_code) +} + +/// A signed-in person's build refreshes its configured location's warehouse copy; never fails it. +fn request_location_sync(context: &BuildContext<'_>) -> PyResult<()> { + let py: Python<'_> = context.py(); + let target: Bound<'_, PyAny> = context.get("effective_target_name")?; + if target.is_none() { + return Ok(()); + } + let target: String = target.extract()?; + let project_dir: std::path::PathBuf = context.get("effective_project_dir")?.extract()?; + if let Some(line) = py.detach(|| sync_after_build(&project_dir, &target)) { + write_stderr(py, &line)?; + } + Ok(()) } /// Run one `sqb build` request. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 9367712a19..6f4297132c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -30,6 +30,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { freshness::registration::register(module)?; crate::bindings::_helpers::native_host::consumption_state::register(module)?; crate::bindings::_helpers::native_host::observations_state::register(module)?; + crate::bindings::_helpers::native_host::audit_results_state::register(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; execution_history::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 9378dea579..36c6485896 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -26,7 +26,8 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::{ - CONSUMPTION_TABLE_NAME, OBSERVATIONS_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, + AUDIT_RESULTS_TABLE_NAME, CONSUMPTION_TABLE_NAME, OBSERVATIONS_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, }; /// The adapter and its open connection. @@ -477,7 +478,7 @@ pub(crate) fn gather_warehouse_facts<'py>( }) } -/// Fingerprint, source freshness, consumption and observation history prune candidates for target schemas. +/// Fingerprint, source freshness, consumption, observation and audit result history prune candidates for target schemas. pub(crate) fn direct_state_prune_candidates<'py>( warehouse: &Warehouse<'py>, target_schemas: &[SchemaKey], @@ -498,6 +499,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( )); locations.push((database.clone(), schema.clone(), CONSUMPTION_TABLE_NAME)); locations.push((database.clone(), schema.clone(), OBSERVATIONS_TABLE_NAME)); + locations.push((database.clone(), schema.clone(), AUDIT_RESULTS_TABLE_NAME)); } let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut candidates: Vec> = Vec::new(); @@ -519,6 +521,10 @@ pub(crate) fn direct_state_prune_candidates<'py>( OBSERVATIONS_TABLE_NAME, "render_prune_observations_history_sql", ), + ( + AUDIT_RESULTS_TABLE_NAME, + "render_prune_audit_results_history_sql", + ), ] { if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index c372c93cc4..ef6bb7fe9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -14,3 +14,6 @@ pub(crate) const COMMAND_IMPLEMENTATIONS: [(RunCommand, &str); 5] = [ (RunCommand::Audit, "run_audit_command"), (RunCommand::Freshness, "run_freshness_command"), ]; + +/// DuckDB's message when another process, such as a running build, holds the database file. +pub(crate) const DUCKDB_LOCK_CONFLICT: &str = "Conflicting lock is held"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 5df26a40db..6c4af1c5ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -12,6 +12,7 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; +use crate::bindings::_helpers::location_command::constants::DUCKDB_LOCK_CONFLICT; use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; use crate::bindings::_helpers::native_host::warehouse_metadata::{ @@ -25,6 +26,7 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul within(&redirect, |_| { let mut session = None; let mut stream = None; + let mut reported_busy = false; let mut serve = || -> PyResult { loop { let line = py.detach(|| -> io::Result> { @@ -61,15 +63,24 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul serde_json::from_value(request).map_err(|_| { PyValueError::new_err("runtime received an invalid metadata read") })?; + let mut busy = false; if session.is_none() { let arguments = location_arguments(py, Some(options.clone()))?; session = match open_history_session(py, &arguments) { Ok(opened) => Some(opened), - Err(_) => { - write_stderr( - py, - "Warehouse connection for metadata reads failed.\n", - )?; + Err(error) => { + busy = error.to_string().contains(DUCKDB_LOCK_CONFLICT); + if !busy { + write_stderr( + py, + "Warehouse connection for metadata reads failed.\n", + )?; + } else if !reported_busy { + write_stderr( + py, + "Warehouse file is in use by another process, such as a DuckDB build; the sync waits for it.\n", + )?; + } None } }; @@ -91,13 +102,21 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul { drop(close_history_session(opened, stream.take())); } - if page.is_none() { - if let Some(opened) = session.take() { - drop(close_history_session(opened, None)); + reported_busy = busy; + if busy { + serde_json::to_string(&serde_json::json!({ "busy": true })) + } else { + if page.is_none() { + if let Some(opened) = session.take() { + drop(close_history_session(opened, None)); + } + write_stderr( + py, + "Warehouse metadata read failed. No page delivered.\n", + )?; } - write_stderr(py, "Warehouse metadata read failed. No page delivered.\n")?; + serde_json::to_string(&page) } - serde_json::to_string(&page) } else { let evaluation: SensorEvaluationRequest = serde_json::from_value(request) .map_err(|_| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs new file mode 100644 index 0000000000..dbb1234c40 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs @@ -0,0 +1,30 @@ +//! Pruning of the audit results state table through the adapter. + +use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::constants::AUDIT_RESULTS_TABLE_NAME; +use sqlbuild_state::audit_results::main::prune_audit_results_sql::prune_audit_results_sql; + +use crate::bindings::_helpers::native_host::consumption_state::state_table_name; + +/// SQL deleting audit results beyond the newest `retain_versions` per audit binding. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] +fn build_audit_results_prune_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + retain_versions: i64, +) -> PyResult { + Ok(state_table_name( + render_qualified_name, + (database, schema), + AUDIT_RESULTS_TABLE_NAME, + )? + .map(|table| prune_audit_results_sql(&table, retain_versions)) + .unwrap_or_default()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_audit_results_prune_sql, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 59eca3d642..1794e8b508 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,5 +1,6 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. +pub(crate) mod audit_results_state; pub(crate) mod connections; pub(crate) mod constants; pub(crate) mod consumption_state; diff --git a/crates/sqlbuild-state/src/audit_results/main/mod.rs b/crates/sqlbuild-state/src/audit_results/main/mod.rs new file mode 100644 index 0000000000..83d8c33dff --- /dev/null +++ b/crates/sqlbuild-state/src/audit_results/main/mod.rs @@ -0,0 +1 @@ +pub mod prune_audit_results_sql; diff --git a/crates/sqlbuild-state/src/audit_results/main/prune_audit_results_sql.rs b/crates/sqlbuild-state/src/audit_results/main/prune_audit_results_sql.rs new file mode 100644 index 0000000000..d4c871b63f --- /dev/null +++ b/crates/sqlbuild-state/src/audit_results/main/prune_audit_results_sql.rs @@ -0,0 +1,11 @@ +//! Prune old audit results from one audit results table. + +/// Delete every audit result ranked beyond the newest `retain` per audit binding. +pub fn prune_audit_results_sql(qualified_name: &str, retain: i64) -> String { + format!( + "DELETE FROM {qualified_name} WHERE result_id IN (SELECT result_id FROM \ +(SELECT result_id, ROW_NUMBER() OVER (PARTITION BY binding_key ORDER BY \ +occurred_at DESC, result_id DESC) AS result_rank FROM {qualified_name}) \ +AS ranked WHERE result_rank > {retain})" + ) +} diff --git a/crates/sqlbuild-state/src/audit_results/mod.rs b/crates/sqlbuild-state/src/audit_results/mod.rs new file mode 100644 index 0000000000..2a043412b4 --- /dev/null +++ b/crates/sqlbuild-state/src/audit_results/mod.rs @@ -0,0 +1 @@ +pub mod main; diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index 3fc9db2953..72ebf3aca6 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] +pub mod audit_results; pub mod consumption; pub mod events; pub mod execution_history; diff --git a/fensu.toml b/fensu.toml index 415e477619..66dcc9a651 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1852,7 +1852,7 @@ paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/cla reason = "tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py requires every first-class adapter to define each StrictAdapter method itself instead of inheriting it from BaseAdapter." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs"] reason = "Each state table has its own Python-facing create and prune bindings. The DDL and table naming live in the shared fixed_state_table_sql and state_table_name; each prune keeps its own retention rule." [[dupes.allowlist]] @@ -2878,6 +2878,7 @@ module_layers = [ "microbatches=plan", "model_config=front", "native_host=infra", + "native_host/audit_results_state=front", "native_host/consumption_state=front", "native_host/model_lookup=project", "native_host/observations_state=front", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 7ca1ede23e..06b138a6f6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -236,8 +236,18 @@ next request, including sessions created by [`sqb login`](login.md). ## Sync from the warehouse now A code location reads what has been built, checks and freshness back from the warehouse when it -starts and after each run the server starts has ended. It does not see a `sqb build` you run on -your own machine until then. To sync now, select **Sync now** on the location's page, or run: +starts, after each run the server starts has ended, and at least every 30 minutes. To pick up a +`sqb build` you run on your own machine straight away, name the server and location in +`sqlbuild_project.toml` and sign in with [`sqb login`](login.md): + +```toml +[control] +server = "https://control.example.com" +location = "orders-warehouse" +``` + +Each `sqb build` then asks that location to sync the target the build used, and prints one line +saying whether it did. You can also select **Sync now** on the location's page, or run: ```bash sqb control sync \ @@ -246,14 +256,37 @@ sqb control sync \ --target development ``` -The command prints `{"applied": true}` when the server has queued the request. The location starts -the sync within a few seconds, and the top bar shows when it last synced. A sync reads the -warehouse and never writes to it. On Snowflake, it skips state tables that haven't changed since -the last sync, so a sync with nothing new doesn't resume the warehouse. +The command prints `{"applied": true}` when the server has queued the request, and +`{"applied": false}` when a sync for that target is already waiting, which covers this request too. +The location starts the sync within a few seconds, and the top bar shows when it last synced. + +| Setting | Environment variable | Default | Meaning | +| --- | --- | --- | --- | +| `[control] server` | `SQLBUILD_CONTROL_SERVER` | none | HTTPS origin of the server, as `sqb login --server` takes it | +| `[control] location` | `SQLBUILD_CONTROL_LOCATION` | none | Location name a signed-in `sqb build` asks to sync | +| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"` | Longest a location waits between syncs; `"0"` or `"off"` turns it off | +| none | `SQLBUILD_CONTROL_CA_CERT` | none | Extra trusted PEM CA certificate for the server | + +An environment variable overrides the project file. `sqb location serve --sync-at-least-every` +overrides both. The interval uses the project's duration syntax, such as `"45m"` or `"2h"`, and +must be at least `"1m"`. Requesting a sync needs the same access as a connection test: Developer or Admin on the location and target. Viewer and Operator cannot request one. Every request is recorded in the audit log. +Details: + +- The sync after a build never changes the build's exit code and gives up after 5 seconds. It + prints nothing when `[control]` sets neither key or nobody is signed in to that server. When the + location is not one you can view, it prints `Warehouse sync skipped:` and the reason. +- Requests that arrive while a sync is running, from builds, **Sync now**, ended runs or the + interval, add one more sync after it, not one each. +- The interval restarts after every completed sync, so a location that syncs often for other + reasons never syncs extra. +- A sync reads the warehouse and never writes to it. On Snowflake, it skips state tables that + haven't changed since the last sync, so a sync with nothing new doesn't resume the warehouse. +- `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1ffa6ed00d..7fa3ba1a37 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5510,6 +5510,13 @@ def build_observations_create_table_sql( render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], ) -> str: ... +def build_audit_results_prune_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + retain_versions: int, +) -> str: ... def build_observations_prune_sql( *, database: str | None, diff --git a/src/sqlbuild/adapter/contract/classes/audit_results.py b/src/sqlbuild/adapter/contract/classes/audit_results.py new file mode 100644 index 0000000000..592bcf8521 --- /dev/null +++ b/src/sqlbuild/adapter/contract/classes/audit_results.py @@ -0,0 +1,24 @@ +"""Default adapter pruning for the audit results state table.""" + +from __future__ import annotations + +from typing import Any, cast + + +class AuditResultsMixin: + """Provide portable audit results pruning for first-class warehouse adapters.""" + + def render_prune_audit_results_history_sql( + self, *, database: str | None, schema: str, retain_versions: int + ) -> str: + """Render SQL deleting audit results beyond the newest `retain_versions` per binding.""" + + from sqlbuild._native import build_audit_results_prune_sql + + adapter: Any = cast(Any, self) + return build_audit_results_prune_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + retain_versions=retain_versions, + ) diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 7366fe01c9..847c7a2c65 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2500,6 +2500,18 @@ def render_prune_observations_history_sql( del database, schema, retain_versions raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def render_prune_audit_results_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old audit results.""" + + del database, schema, retain_versions + raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def sql_analysis_dialect(self) -> str | None: """Return the configured SQL analysis dialect name, if any.""" diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 1e40f0a464..4f8ec2cb57 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -1163,6 +1163,17 @@ def render_prune_observations_history_sql( """Render SQL that prunes old observations.""" ... + @abstractmethod + def render_prune_audit_results_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old audit results.""" + ... + @abstractmethod def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 4658bb818c..3aa5666d9a 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -18,6 +18,7 @@ SqlValueKind, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( _COLLECTION_SQL_VALUE_KINDS, BaseAdapter, @@ -153,7 +154,12 @@ class BigQueryAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """BigQuery adapter backed by google-cloud-bigquery.""" diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index cc8d367733..e4ef093412 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -22,6 +22,7 @@ SqlValueKind, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -156,7 +157,12 @@ class ClickHouseAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """ClickHouse adapter that builds tables aside and rewrites whole partitions.""" diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index b5a1c03c8a..8914544982 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -21,6 +21,7 @@ SqlValueKind, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -146,7 +147,12 @@ class DatabricksAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """Databricks adapter backed by databricks-sql-connector.""" diff --git a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py index 820c5d6447..2a9eb06c16 100644 --- a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py +++ b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py @@ -6,6 +6,7 @@ from typing import Any, ClassVar from uuid import uuid4 +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import _quote_sql_string from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter @@ -15,7 +16,9 @@ from sqlbuild.microbatches.constants import MICROBATCH_GENERATION_COMMENT_PREFIX -class DuckDbAdapter(MicrobatchMixin, ConsumptionMixin, ObservationsMixin, DuckDbBackedAdapter): +class DuckDbAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, AuditResultsMixin, DuckDbBackedAdapter +): """DuckDB adapter allowing implicit main for targetless and named local targets.""" adapter_name: ClassVar[str] = BuiltinAdapter.DUCKDB.value diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index 6419cae9f0..d14283f6b6 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -8,6 +8,7 @@ from typing import Any, ClassVar from urllib.parse import urlencode +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin @@ -23,7 +24,9 @@ ) -class MotherDuckAdapter(MicrobatchMixin, ConsumptionMixin, ObservationsMixin, DuckDbBackedAdapter): +class MotherDuckAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, AuditResultsMixin, DuckDbBackedAdapter +): """MotherDuck adapter backed by DuckDB's MotherDuck connection support.""" adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 7907c13f32..581f912dbd 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -17,6 +17,7 @@ SqlValue, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _build_names_filter, @@ -124,7 +125,12 @@ class PostgresAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """PostgreSQL adapter backed by psycopg.""" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index a0469a709a..f5eae198f6 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -23,6 +23,7 @@ collect_snowflake_cost, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -199,7 +200,12 @@ class SnowflakeAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """Snowflake adapter backed by snowflake-connector-python.""" diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 3ae20ff28b..b933c09664 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -25,6 +25,7 @@ SqlValue, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -168,7 +169,12 @@ class SqliteAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """SQLite adapter: one database file, SQLBuild schemas stored as `schema__name` tables.""" diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index f6ecbc9cf5..fe336d2821 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -19,6 +19,7 @@ SqlValueKind, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _build_names_filter, @@ -135,7 +136,12 @@ def _render_tsql_distinct_condition(*, left: str, right: str) -> str: class SqlServerAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """Microsoft SQL Server adapter backed by pymssql.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index 5ba61a7da0..ed9c45ac8c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -65,6 +65,7 @@ class JanitorDirectStatePruningE2ETestCase: expected_source_freshness_count_after: int expected_fingerprint_run_ids_after: tuple[str, ...] expected_source_freshness_run_ids_after: tuple[str, ...] + expected_audit_results_after: tuple[tuple[str, str], ...] @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index 6c2047da66..b6ca3a8f9d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -11,6 +11,7 @@ from pathlib import Path from textwrap import dedent from types import MappingProxyType +from typing import Any from sqlbuild._native import ( Fingerprint, @@ -20,6 +21,8 @@ write_source_freshness_records, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.audit_results.main._write import write_audit_result_records +from sqlbuild.executor.audit_results.models import AuditResultRecord from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( execute_duckdb, prepare_inline_project, @@ -143,6 +146,57 @@ def create_janitor_scenario_relations(*, db_path: Path) -> None: connection.close() +DIRECT_STATE_PRUNING: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/direct_state_pruning.json").read_text() +) + + +def audit_record(*, binding_key: str, run_id: str, occurred_at: datetime) -> AuditResultRecord: + """One passing final audit result for `binding_key` in `run_id`.""" + + audit_name: str = binding_key.split(".")[-1] + return AuditResultRecord( + result_id=f"{binding_key}:{run_id}", + schema_version=1, + occurred_at=occurred_at, + invocation_id=f"invocation_{run_id}", + run_id=run_id, + audit_name=audit_name, + audit_definition_name=audit_name, + binding_key=binding_key, + definition_fingerprint="definition-fingerprint", + execution_fingerprint=f"execution_{run_id}", + evaluation_mode="violations", + run_scope_phase="final", + attachment_kind="model", + attached_target_kind="model", + attached_target_name="orders", + attached_column_name=None, + target_database=None, + target_schema="main", + target_name="orders", + severity="error", + outcome="pass", + execution_error=None, + violation_count=0, + measured_value=None, + sample_count=None, + sample_unit=None, + minimum_samples=None, + thresholds_json=None, + evidence_json=None, + evidence_count=None, + evidence_truncated=None, + evidence_error=None, + measurement_sql=None, + evidence_sql=None, + executed_sql="SELECT 1 WHERE FALSE", + sql_digest="digest", + metadata_json=None, + reused=False, + ) + + def create_direct_state_history(*, db_path: Path, schema: str = "main") -> None: import duckdb @@ -151,12 +205,7 @@ def create_direct_state_history(*, db_path: Path, schema: str = "main") -> None: try: run_id: str observed_hour: int - for run_id, observed_hour in ( - ("run_000", 10), - ("run_001", 12), - ("run_002", 12), - ("run_003", 13), - ): + for run_id, observed_hour in DIRECT_STATE_PRUNING["runs"]: _ = write_fingerprint( connection=connection, execute=lambda *, connection, sql: connection.execute(sql), @@ -178,6 +227,22 @@ def create_direct_state_history(*, db_path: Path, schema: str = "main") -> None: render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, ) + write_audit_result_records( + connection=connection, + execute=lambda *, connection, sql: connection.execute(sql), + database=None, + schema=schema, + records=tuple( + audit_record( + binding_key=binding_key, + run_id=run_id, + occurred_at=datetime(2026, 1, 15, observed_hour, 10, 0, tzinfo=UTC), + ) + for binding_key in DIRECT_STATE_PRUNING["audit_binding_keys"] + ), + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) write_source_freshness_records( connection=connection, execute=lambda *, connection, sql: connection.execute(sql), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index a7c21d7385..79694159b2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -393,12 +393,13 @@ def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_t plan_command=("--no-color", "plan"), expected_exit_code=0, expected_stdout_fragments=( - "direct state pruned 3", + "direct state pruned 4", "Eligible direct state pruning", "main._sqlbuild_fingerprints keep latest 2", "main._sqlbuild_source_freshness keep latest 2", "main._sqlbuild_consumption keep latest 2", - "Pruned 3 direct state tables.", + "main._sqlbuild_audit_results keep latest 2", + "Pruned 4 direct state tables.", ), expected_fingerprint_count_before=5, expected_fingerprint_count_after=3, @@ -406,6 +407,13 @@ def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_t expected_source_freshness_count_after=2, expected_fingerprint_run_ids_after=("run_003", "run_002"), expected_source_freshness_run_ids_after=("run_003", "run_002"), + # Newest two per audit binding. + expected_audit_results_after=( + ("customers.unique_ids", "run_003"), + ("customers.unique_ids", "run_002"), + ("orders.valid_orders", "run_003"), + ("orders.valid_orders", "run_002"), + ), ) ], ids=lambda case: case.description, @@ -477,6 +485,13 @@ def test_given_direct_state_history_when_running_janitor_then_it_prunes_history( "ORDER BY observed_at DESC, run_id DESC" ), ) == [(run_id,) for run_id in test_case.expected_source_freshness_run_ids_after] + assert query_duckdb( + db_path=db_path, + sql=( + "SELECT binding_key, run_id FROM main._sqlbuild_audit_results " + "WHERE run_id LIKE 'run_%' ORDER BY binding_key, occurred_at DESC, result_id DESC" + ), + ) == list(test_case.expected_audit_results_after) plan_result: subprocess.CompletedProcess[str] = run_sqb( command=test_case.plan_command, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py index ba85d4120e..ee4e9a2603 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py @@ -162,7 +162,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview prod", "prod.customers -> prod._sqb_archive__", - "\u2713 Janitor completed archived 1, deleted 1, pruned 3", + "\u2713 Janitor completed archived 1, deleted 1, pruned 4", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), @@ -176,7 +176,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview dev", "dev.customers -> dev._sqb_archive__", - "\u2713 Janitor completed archived 1, deleted 1, pruned 3", + "\u2713 Janitor completed archived 1, deleted 1, pruned 4", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index ea206b7b33..512b7a78dd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -2,9 +2,12 @@ import json import os +import sqlite3 import subprocess import sys import time +from contextlib import closing +from functools import partial from http.cookies import SimpleCookie from pathlib import Path from subprocess import Popen @@ -300,3 +303,59 @@ def verify_connection_test_transport( with pytest.raises(HTTPError) as denied: person_request(api, path, body=body | extra, session=administrator) assert denied.value.code == 422 + + +def build_stderr( + invoke: partial[subprocess.CompletedProcess[Any]], + project: tuple[str, Path], + environment: dict[str, str], + configuration: str, + extra: dict[str, str] | None = None, +) -> str: + """Build the project with `configuration` as its project file; the build always succeeds.""" + binary, project_file = project + _ = project_file.write_text(configuration) + built: subprocess.CompletedProcess[str] = invoke( + [binary, "--no-color", "build", "--project-dir", str(project_file.parent)] + + ["--target", "development"], + env=environment | (extra or {}), + ) + assert built.returncode == 0, built.stderr + return built.stderr + + +def quiet_builds( + invoke: partial[subprocess.CompletedProcess[Any]], + project: tuple[str, Path], + environment: dict[str, str], + origin: str, +) -> tuple[partial[str], str]: + """Builds print no sync line without a [control] location or before anyone signs in.""" + unconfigured: str = project[1].read_text() + build: partial[str] = partial(build_stderr, invoke, project, environment) + for control in ("", f"[control]\nserver = '{origin}'\nlocation = 'orders'\n"): + assert "Warehouse sync" not in build(unconfigured + control), control + return build, unconfigured + + +def assert_build_syncs_coalesce( + build: partial[str], project: tuple[str, str, str], control_store: Path +) -> None: + """An unknown location is named; the environment overrides the file; repeats queue nothing.""" + unconfigured, origin, requester = project + returns: str = unconfigured + f"[control]\nserver = '{origin}'\nlocation = 'returns'\n" + assert ( + f"Warehouse sync skipped: no location named 'returns' that you can view on {origin}." + in build(returns) + ) + for _ in range(4): + assert "Warehouse sync requested for location 'orders' target 'development'." in build( + returns, {"SQLBUILD_CONTROL_LOCATION": "orders"} + ) + with closing(sqlite3.connect(control_store)) as store: + queued: list[tuple[str]] = store.execute( + "SELECT requested_by FROM sqb_location_work " + "WHERE json_extract(request_json,'$.work.kind')='sync_warehouse'" + ).fetchall() + assert queued == [(requester,)], "build syncs join the person's pending request" + _ = build(unconfigured) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py index e6e33f0a39..d926447448 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py @@ -1,10 +1,8 @@ """An installed CLI inspects actual published SQL through a bounded live person session.""" import json -import sqlite3 import subprocess import sys -from contextlib import closing from functools import partial from pathlib import Path from typing import Any @@ -28,7 +26,9 @@ sign_in, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + assert_build_syncs_coalesce, provision_person, + quiet_builds, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( administer, @@ -124,6 +124,12 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori ) absent: subprocess.CompletedProcess[str] = invoke(command) assert absent.returncode != 0 and not absent.stdout and "not signed in" in absent.stderr + build, unconfigured = quiet_builds( + invoke, + (binary, worker.project / "sqlbuild_project.toml"), + client_environment | {"SQLBUILD_CONTROL_CA_CERT": str(certificate)}, + origin, + ) with pending_login( [binary, "login", *common, "--no-browser"], client_environment, @@ -168,12 +174,11 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori requested: subprocess.CompletedProcess[str] = invoke(sync) assert requested.returncode == 0, requested.stderr assert json.loads(requested.stdout) == {"applied": True} - with closing(sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3")) as store: - queued: list[tuple[str]] = store.execute( - "SELECT requested_by FROM sqb_location_work " - "WHERE json_extract(request_json,'$.work.kind')='sync_warehouse'" - ).fetchall() - assert queued == [("principal:" + person["principal_id"],)] + assert_build_syncs_coalesce( + build, + (unconfigured, origin, "principal:" + person["principal_id"]), + tmp_path / "control/.sqlbuild/control.sqlite3", + ) inspected: subprocess.CompletedProcess[str] = invoke(command) assert inspected.returncode == 0 and "Published SQL read successfully" in inspected.stderr sql: dict[str, Any] = json.loads(inspected.stdout) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 488c18d903..c07d61f44e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -15,6 +15,7 @@ from urllib.request import Request, urlopen from uuid import uuid4 +import duckdb import pytest from sqlbuild._native import ( @@ -60,6 +61,7 @@ append_observation_history, append_old_name_history, append_retried_fingerprints, + hold_write_lock, location_process, open_legacy_warehouse, snapshot_api, @@ -1150,6 +1152,27 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre datetime.fromisoformat(report["received_at"]) for report in reports ) assert (project / "warehouse.db").exists() == availability["available"] + held_locks: list[dict[str, Any]] = fixture["held_write_lock"][test_case.adapter_name] + for held_lock in held_locks: + lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db") + record_finished_run(control) + reports = wait_for_warehouse_sync_health( + api, held_lock["held"], location, log, after=observed_health_at + ) + observed_health_at = max( + datetime.fromisoformat(report["received_at"]) for report in reports + ) + lock.close() + record_finished_run(control) + reports = wait_for_warehouse_sync_health( + api, held_lock["released"], location, log, after=observed_health_at + ) + observed_health_at = max( + datetime.fromisoformat(report["received_at"]) for report in reports + ) + assert log.read_text().count("Warehouse file is in use by another process") == len( + held_locks + ) @pytest.mark.parametrize( diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 50e6d53950..0a4342f570 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -57,7 +57,7 @@ describe('toLocationDetail', () => { }); describe('warehouseSyncOf', () => { - const sync = (at: string, table: 'catalog' | 'statistics', failure: 'read' | 'delivery' | null) => ({ + const sync = (at: string, table: 'catalog' | 'statistics', failure: 'read' | 'delivery' | 'busy' | null) => ({ received_at: at, report: { event_id: at, session_id: 's1', target: 'development', health: { kind: 'warehouse_sync' as const, generation: 1, location: { schema: 'main' }, table, failure, records_sent: 0, elapsed_millis: 5 } } }); @@ -91,6 +91,13 @@ describe('warehouseSyncOf', () => { ]); }); + it('shows a build holding the warehouse file as a wait, not a failure', () => { + const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; + const observations = [sync('2026-11-08T12:03:00Z', 'catalog', 'busy'), sync('2026-11-08T12:01:00Z', 'catalog', null)]; + expect(toWarehouseConnection(location, observations).tables.map((t) => [t.failure, t.waiting, t.lastSyncedAt])).toEqual([['Waiting for a build that holds the warehouse file', true, '2026-11-08T12:01:00Z']]); + expect(warehouseSyncOf(observations)).toEqual({ lastSyncedAt: '2026-11-08T12:01:00Z', lastError: null, lastReportedAt: '2026-11-08T12:03:00Z' }); + }); + it('clears the error once the failing table syncs again', () => { expect(warehouseSyncOf([sync('2026-11-08T12:05:00Z', 'catalog', null), sync('2026-11-08T12:03:00Z', 'catalog', 'read')])?.lastError).toBeNull(); }); diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index 436cb4d6ad..357b298f52 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -120,12 +120,15 @@ export function toLocationDetail(location: Location, observations: Wire['Locatio }; } -const SYNC_STAGE: Record = { +const SYNC_STAGE: Record, string> = { watermark: 'reading where the last sync stopped', read: 'reading the warehouse', delivery: 'sending results to the server' }; +/** What held up a table's newest sync; a build holding the warehouse file is a wait, not a failure. */ +const syncProblemOf = (failure: Wire['WarehouseSyncFailure']) => (failure === 'busy' ? 'Waiting for a build that holds the warehouse file' : `Failed while ${SYNC_STAGE[failure]}`); + type Sync = Wire['LocationHealthObservation'] & { report: { health: Extract } }; /** Sync reports newest first, and the newest of each table. */ @@ -145,7 +148,8 @@ export function toWarehouseConnection(location: Location, observations: Wire['Lo at: received_at, recordsSent: health.records_sent, elapsedMs: health.elapsed_millis, - failure: health.failure ? `Failed while ${SYNC_STAGE[health.failure]}` : null, + failure: health.failure ? syncProblemOf(health.failure) : null, + waiting: health.failure === 'busy', lastSyncedAt: syncs.find((s) => s.report.health.table === health.table && !s.report.health.failure)?.received_at ?? null })) .sort((a, b) => a.table.localeCompare(b.table)); @@ -164,7 +168,7 @@ export function toWarehouseConnection(location: Location, observations: Wire['Lo export function warehouseSyncOf(observations: Wire['LocationHealthObservation'][]): WarehouseSync | null { const { syncs, newest } = syncsOf(observations); if (!syncs.length) return null; - const failing = [...newest.values()].flatMap((s) => (s.report.health.failure ? [`${s.report.health.table}: failed while ${SYNC_STAGE[s.report.health.failure]}`] : [])); + const failing = [...newest.values()].flatMap(({ report: { health } }) => (health.failure && health.failure !== 'busy' ? [`${health.table}: failed while ${SYNC_STAGE[health.failure]}`] : [])); return { lastSyncedAt: syncs.find((s) => !s.report.health.failure)?.received_at ?? null, lastError: failing.length ? failing.sort().join('\n') : null, diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index 944f0e0cbb..223fae92da 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -702,7 +702,7 @@ export class MockPlatformApi implements PlatformApi { target: c.target, stateLocation: { database: typeof c.values.database === 'string' ? c.values.database : null, schema: 'sqlbuild' }, lastTest: this.#connectionTests.get(`${l.id}/${c.name}`) ?? c.lastTest, - tables: ['audit_results', 'catalog', 'fingerprints', 'statistics'].map((table, i) => ({ table, at: synced, recordsSent: 12 * (i + 1), elapsedMs: 180 + 40 * i, failure: null, lastSyncedAt: synced })) + tables: ['audit_results', 'catalog', 'fingerprints', 'statistics'].map((table, i) => ({ table, at: synced, recordsSent: 12 * (i + 1), elapsedMs: 180 + 40 * i, failure: null, waiting: false, lastSyncedAt: synced })) })) ); return this.#ctx.reply(structuredClone(connections)); diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index b374e8b8b0..087d374610 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -233,8 +233,10 @@ export type WarehouseTableSync = { at: Iso; recordsSent: number; elapsedMs: number; - /** What the newest sync was doing when it failed; null when it succeeded. */ + /** What the newest sync was doing when it failed, or what it is waiting for; null when it succeeded. */ failure: string | null; + /** The newest sync is waiting for a build that holds the warehouse file, which is not a failure. */ + waiting: boolean; /** The newest sync of the table that succeeded. */ lastSyncedAt: Iso | null; }; diff --git a/ui/src/routes/connections/+page.svelte b/ui/src/routes/connections/+page.svelte index 00c18a612b..85b11f1237 100644 --- a/ui/src/routes/connections/+page.svelte +++ b/ui/src/routes/connections/+page.svelte @@ -44,7 +44,7 @@ {t.table} - {#if t.failure}{:else}{/if} + {#if t.failure}{:else}{/if} {formatNumber(t.recordsSent)} diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 9ebd2dad66..4f797e35bd 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -222,8 +222,18 @@ next request, including sessions created by [`sqb login`](/docs/cli/login/). ## Sync from the warehouse now A code location reads what has been built, checks and freshness back from the warehouse when it -starts and after each run the server starts has ended. It does not see a `sqb build` you run on -your own machine until then. To sync now, select **Sync now** on the location's page, or run: +starts, after each run the server starts has ended, and at least every 30 minutes. To pick up a +`sqb build` you run on your own machine straight away, name the server and location in +`sqlbuild_project.toml` and sign in with [`sqb login`](/docs/cli/login/): + +```toml +[control] +server = "https://control.example.com" +location = "orders-warehouse" +``` + +Each `sqb build` then asks that location to sync the target the build used, and prints one line +saying whether it did. You can also select **Sync now** on the location's page, or run: ```bash sqb control sync \ @@ -232,14 +242,37 @@ sqb control sync \ --target development ``` -The command prints `{"applied": true}` when the server has queued the request. The location starts -the sync within a few seconds, and the top bar shows when it last synced. A sync reads the -warehouse and never writes to it. On Snowflake, it skips state tables that haven't changed since -the last sync, so a sync with nothing new doesn't resume the warehouse. +The command prints `{"applied": true}` when the server has queued the request, and +`{"applied": false}` when a sync for that target is already waiting, which covers this request too. +The location starts the sync within a few seconds, and the top bar shows when it last synced. + +| Setting | Environment variable | Default | Meaning | +| --- | --- | --- | --- | +| `[control] server` | `SQLBUILD_CONTROL_SERVER` | none | HTTPS origin of the server, as `sqb login --server` takes it | +| `[control] location` | `SQLBUILD_CONTROL_LOCATION` | none | Location name a signed-in `sqb build` asks to sync | +| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"` | Longest a location waits between syncs; `"0"` or `"off"` turns it off | +| none | `SQLBUILD_CONTROL_CA_CERT` | none | Extra trusted PEM CA certificate for the server | + +An environment variable overrides the project file. `sqb location serve --sync-at-least-every` +overrides both. The interval uses the project's duration syntax, such as `"45m"` or `"2h"`, and +must be at least `"1m"`. Requesting a sync needs the same access as a connection test: Developer or Admin on the location and target. Viewer and Operator cannot request one. Every request is recorded in the audit log. +Details: + +- The sync after a build never changes the build's exit code and gives up after 5 seconds. It + prints nothing when `[control]` sets neither key or nobody is signed in to that server. When the + location is not one you can view, it prints `Warehouse sync skipped:` and the reason. +- Requests that arrive while a sync is running, from builds, **Sync now**, ended runs or the + interval, add one more sync after it, not one each. +- The interval restarts after every completed sync, so a location that syncs often for other + reasons never syncs extra. +- A sync reads the warehouse and never writes to it. On Snowflake, it skips state tables that + haven't changed since the last sync, so a sync with nothing new doesn't resume the warehouse. +- `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a