Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"description": "Direct state history for janitor pruning that keeps the newest two rows per key. run_001 and run_002 share an hour at the keep boundary, so the tie breaks on the newer run or result id.",
"runs": [["run_000", 10], ["run_001", 12], ["run_002", 12], ["run_003", 13]],
"audit_binding_keys": ["orders.valid_orders", "customers.unique_ids"]
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@
{"available": false, "source": "warehouse.db", "destination": "../retained-warehouse.db", "failure": "read", "tables": ["fingerprints"]},
{"available": true, "source": "../retained-warehouse.db", "destination": "warehouse.db", "failure": null, "tables": ["fingerprints", "source_freshness"]}
],
"held_write_lock": {
"duckdb": [
{
"held": {"failure": "busy", "tables": ["source_freshness"]},
"released": {"failure": null, "tables": ["fingerprints", "source_freshness"]}
}
],
"sqlite": []
},
"insert_sql": [
"INSERT INTO _sqlbuild_fingerprints SELECT node_type,node_name,target_database,target_schema,target_name,'{run_id}',definition_hash,version_hash,schema_fingerprint,definition_b64,metadata_json_b64,'{observed_at}' FROM _sqlbuild_fingerprints WHERE run_id='legacy-run'",
"INSERT INTO _sqlbuild_source_freshness SELECT source_name,target_database,target_schema,target_name,'{run_id}',strategy,value_kind,data_version,data_version_hash,'{observed_at}' FROM _sqlbuild_source_freshness WHERE run_id='legacy-run'"
Expand Down
2 changes: 1 addition & 1 deletion clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3277,7 +3277,7 @@ export interface components {
* @description Closed failure stages keep warehouse errors and credentials out of health observations.
* @enum {string}
*/
WarehouseSyncFailure: "watermark" | "read" | "delivery";
WarehouseSyncFailure: "watermark" | "read" | "delivery" | "busy";
WarehouseSyncResponse: {
applied: boolean;
};
Expand Down
7 changes: 7 additions & 0 deletions crates/sqlbuild-cli/src/argv/command_table.json
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,13 @@
"help": "immutable deployment image identity to pin for workers",
"group": 1
},
{
"kind": "store",
"options": ["--sync-at-least-every"],
"dest": "location_sync_at_least_every",
"help": "for serve, sync the warehouse at least this often, such as 30m; 0 or off turns it off",
"group": 1
},
{
"kind": "store",
"options": ["--run-id"],
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlbuild-cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,6 @@ pub mod diff_report;
pub mod dispatch;
pub mod launcher;
pub mod native_commands;
pub(crate) mod person_login;
pub mod person_login;
pub mod replay;
pub(crate) mod workers;
89 changes: 70 additions & 19 deletions crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ use sqlbuild_orchestration::api::errors::ClientError;
use sqlbuild_orchestration::api::main::serve_api::serve_api;

use crate::adhoc::main::adhoc_planner::adhoc_planner;
use sqlbuild_config::project::main::load_control_config::load_control_config;
use sqlbuild_config::project::main::project_config_paths::project_config_paths;
use sqlbuild_config::project::main::sync_interval_seconds::sync_interval_seconds;
use sqlbuild_config::project::models::ControlConfig;
use sqlbuild_orchestration::api::models::{
ApiAuthentication, DevelopmentScope, HumanLogin, ServerFacts,
};
Expand Down Expand Up @@ -73,17 +77,17 @@ use crate::native_commands::_helpers::terminal::{
use crate::native_commands::constants::{
ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_OVERLAP_SECONDS,
METADATA_POLL_SECONDS, METADATA_READER_CLOSE_SECONDS, METADATA_RECONCILE_SECONDS,
METADATA_REPLY_BYTES, METADATA_SYNC_CHECK_SECONDS, SCHEDULER_COMMAND,
UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND,
METADATA_REPLY_BYTES, METADATA_SAFETY_SYNC_SECONDS, METADATA_SYNC_CHECK_SECONDS,
SAFETY_SYNC_ENV, SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND,
};
use crate::native_commands::errors::PythonNeeded;
use crate::native_commands::main::_run_admin::run_admin;
use crate::native_commands::main::_run_control::run_control;
use crate::native_commands::main::_run_cost::run_cost;
use crate::native_commands::models::{
CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler,
LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, NativeCommand,
NativeInvocation, ProjectLocation, ScanToken,
LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, MetadataReply,
NativeCommand, NativeInvocation, ProjectLocation, ScanToken,
};
use crate::native_commands::types::LocationPublication;
use crate::workers::main::docker_image_sources::docker_image_sources;
Expand Down Expand Up @@ -335,7 +339,7 @@ pub(crate) fn location_result<T: serde::de::DeserializeOwned>(

pub(crate) fn metadata_sync(
(locations, generation): (Vec<WarehouseStateLocation>, i64),
reconcile_seconds: u32,
(reconcile_seconds, safety_seconds): (u32, u64),
) -> LocationMetadataSync {
LocationMetadataSync {
locations,
Expand All @@ -355,6 +359,9 @@ pub(crate) fn metadata_sync(
next_at: Instant::now(),
idle: true,
cycle_started_at: None,
safety_seconds,
safety_at: Instant::now() + Duration::from_secs(safety_seconds),
busy: false,
change_tokens: HashMap::new(),
scan_token: None,
scan_delivered: false,
Expand Down Expand Up @@ -483,15 +490,9 @@ pub(crate) async fn begin_after_run(
Ok(due) => due,
Err(error) => return (state, Err(error)),
};
let ran = match (state.cycle_started_at, due.runs_ended_at) {
(None, _) => true,
(Some(started_at), Some(ended_at)) => ended_at >= started_at,
(Some(_), None) => false,
};
let ran = state.sync_due(due.runs_ended_at);
if ran {
state.idle = false;
state.cycle_started_at = Some(due.now);
state.next_at = Instant::now();
state.begin_sync(due.now);
}
(state, Ok(ran))
}
Expand Down Expand Up @@ -589,6 +590,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync
}
state.idle = true;
state.next_at = Instant::now();
state.safety_at = Instant::now() + Duration::from_secs(state.safety_seconds);
drop(write_output(&[(
Stream::Stderr,
"Warehouse history synchronization completed.\n",
Expand Down Expand Up @@ -699,16 +701,56 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result<LocationServe, C
"SQLBUILD_METADATA_RECONCILE_SECONDS must be a positive integer".into(),
)
})?;
let environment = match std::env::var(SAFETY_SYNC_ENV) {
Ok(value) => Some(value),
Err(std::env::VarError::NotPresent) => None,
Err(std::env::VarError::NotUnicode(_)) => {
return Err(ControlError::Invalid(format!(
"{SAFETY_SYNC_ENV} must be text"
)));
}
};
let safety_sync_seconds = safety_sync_seconds(
text(namespace, "location_sync_at_least_every").as_deref(),
environment.as_deref(),
|| {
load_control_config(&project_config_paths(&project_dir)).map_err(|failure| {
failure.into_config_failure().map_or_else(
|| "sqlbuild_project.toml could not be read".to_owned(),
|failure| failure.render(|_| "sqlbuild_project.toml".to_owned()),
)
})
},
)
.map_err(ControlError::Invalid)?;
Ok(LocationServe {
project_dir,
server_url: text(namespace, "location_server_url").unwrap_or_default(),
location_id,
target,
image_id: text(namespace, "location_image_id"),
metadata_reconcile_seconds,
safety_sync_seconds,
})
}

/// The safety sync interval from the flag, then the environment, then the project, then thirty minutes.
pub(crate) fn safety_sync_seconds(
flag: Option<&str>,
environment: Option<&str>,
project: impl FnOnce() -> Result<ControlConfig, String>,
) -> Result<u64, String> {
if let Some(text) = flag {
return sync_interval_seconds(text).map_err(|help| format!("--sync-at-least-every {help}"));
}
if let Some(text) = environment.filter(|text| !text.trim().is_empty()) {
return sync_interval_seconds(text).map_err(|help| format!("{SAFETY_SYNC_ENV} {help}"));
}
Ok(project()?
.sync_at_least_every_seconds
.unwrap_or(METADATA_SAFETY_SYNC_SECONDS))
}

/// Reconcile from the server after restarts or lost acknowledgements; transient outages retry.
pub(crate) async fn location_generation(
options: &LocationServe,
Expand Down Expand Up @@ -1152,6 +1194,7 @@ pub(crate) async fn read_metadata_page(
mut state: LocationMetadataSync,
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<(LocationMetadataSync, Option<WarehouseHistoryPage>)>, ControlError> {
state.busy = false;
if let Some(statistics) = &state.statistics
&& statistics.names.is_empty()
{
Expand Down Expand Up @@ -1181,10 +1224,15 @@ pub(crate) async fn read_metadata_page(
Duration::from_secs(METADATA_POLL_SECONDS),
exchange_metadata_read(reader, request),
) => match result {
Ok((reader, Ok(page))) => {
Ok((reader, Ok(MetadataReply::Page(page)))) => {
state.reader = Some(reader);
Ok(Some((state, page)))
}
Ok((reader, Ok(MetadataReply::Busy))) => {
state.busy = true;
state.reader = Some(reader);
Ok(Some((state, None)))
}
Ok((reader, Err(_))) => {
stop_location_runtime(Some(reader)).await?;
Ok(Some((state, None)))
Expand Down Expand Up @@ -1230,10 +1278,7 @@ pub(crate) async fn close_metadata_reader(
async fn exchange_metadata_read(
mut reader: LocationCompiler,
request: Vec<u8>,
) -> (
LocationCompiler,
Result<Option<WarehouseHistoryPage>, ControlError>,
) {
) -> (LocationCompiler, Result<MetadataReply, ControlError>) {
let result = async {
reader.stdin.write_all(&request).await?;
reader.stdin.write_all(b"\n").await?;
Expand All @@ -1253,7 +1298,13 @@ async fn exchange_metadata_read(
"location reply exceeds its byte limit".into(),
));
}
serde_json::from_str(&line)
let reply: serde_json::Value = serde_json::from_str(&line)
.map_err(|_| ControlError::Invalid("invalid location reply".into()))?;
if reply.get("busy") == Some(&serde_json::Value::Bool(true)) {
return Ok(MetadataReply::Busy);
}
serde_json::from_value(reply)
.map(MetadataReply::Page)
.map_err(|_| ControlError::Invalid("invalid location reply".into()))
}
.await;
Expand Down
4 changes: 4 additions & 0 deletions crates/sqlbuild-cli/src/native_commands/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ pub(crate) const METADATA_READER_CLOSE_SECONDS: u64 = 5;
pub(crate) const METADATA_SYNC_CHECK_SECONDS: u64 = 5;
/// Full key-set reconciliation is independent of the incremental poll and overlap window.
pub(crate) const METADATA_RECONCILE_SECONDS: u32 = 300;
/// An idle location syncs at least this often, so a lost run or request signal cannot leave drift.
pub(crate) const METADATA_SAFETY_SYNC_SECONDS: u64 = 1800;
/// Overrides `[control] sync_at_least_every`; `--sync-at-least-every` overrides both.
pub(crate) const SAFETY_SYNC_ENV: &str = "SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY";
pub(crate) const METADATA_OVERLAP_SECONDS: i64 = 300;
/// A source-key cursor can repeat the last row's base64-encoded definition beside the records.
pub(crate) const METADATA_REPLY_BYTES: u64 =
Expand Down
39 changes: 29 additions & 10 deletions crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,16 +78,7 @@ pub(super) async fn sync_location_metadata(
};
state = read;
let Some(page) = page else {
tokio::select! {
result = &mut stopped => { result?; return Ok(None); },
() = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Read))) => {},
}
state.retry_read();
drop(write_output(&[(
Stream::Stderr,
"Warehouse history read failed; retaining mirror and retrying a bounded page.\n",
)]));
return Ok(Some(state));
return failed_read(state, (client, poll), started, stopped.as_mut()).await;
};
if location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed
{
Expand Down Expand Up @@ -123,6 +114,34 @@ pub(super) async fn sync_location_metadata(
finish_metadata_page(acknowledge_metadata_page(state)).await
}

/// A busy warehouse file waits a poll interval; any other failed read retries a smaller page.
async fn failed_read(
mut state: LocationMetadataSync,
(client, poll): (&LocationWorkClient, &LocationWorkPoll),
started: Instant,
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<LocationMetadataSync>, ControlError> {
let failure = if state.busy {
WarehouseSyncFailure::Busy
} else {
WarehouseSyncFailure::Read
};
tokio::select! {
result = stopped => { result?; return Ok(None); },
() = metadata_health(&state, (client, poll), (started, Some(failure))) => {},
}
if state.busy {
state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS);
return Ok(Some(state));
}
state.retry_read();
drop(write_output(&[(
Stream::Stderr,
"Warehouse history read failed; retaining mirror and retrying a bounded page.\n",
)]));
Ok(Some(state))
}

fn retry_metadata(
mut state: LocationMetadataSync,
error: ClientError,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,10 @@ async fn watch_location(
published_generation = Some(generation);
metadata = Some(metadata_sync(
(locations, generation),
options.metadata_reconcile_seconds,
(
options.metadata_reconcile_seconds,
options.safety_sync_seconds,
),
));
let Some(files) = check_published_location(
options,
Expand Down
37 changes: 36 additions & 1 deletion crates/sqlbuild-cli/src/native_commands/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ use sqlbuild_orchestration::declarations::models::TriggerChange;
use sqlbuild_orchestration::platform::models::LocationWorkRequest;
use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest};
use sqlbuild_orchestration::warehouse_metadata::models::{
MetadataBatch, StatisticsRead, WarehouseHistoryKind, WarehouseStateLocation,
MetadataBatch, StatisticsRead, WarehouseHistoryKind, WarehouseHistoryPage,
WarehouseStateLocation,
};
use uuid::Uuid;

Expand Down Expand Up @@ -149,6 +150,8 @@ pub(crate) struct LocationServe {
pub(crate) target: String,
pub(crate) image_id: Option<String>,
pub(crate) metadata_reconcile_seconds: u32,
/// Seconds between safety syncs of an idle location; `0` turns them off.
pub(crate) safety_sync_seconds: u64,
}

/// Retained source observations survive compilation; warehouse exclusions never replace them.
Expand Down Expand Up @@ -209,6 +212,12 @@ pub(crate) struct LocationMetadataSync {
pub(crate) idle: bool,
/// Server time when the current or last synchronization began; `None` before the first.
pub(crate) cycle_started_at: Option<DateTime<Utc>>,
/// Seconds between safety syncs; `0` turns them off.
pub(crate) safety_seconds: u64,
/// When an idle location syncs even though no run ended and nobody asked.
pub(crate) safety_at: tokio::time::Instant,
/// The last read found the warehouse file held by another process, such as a DuckDB build.
pub(crate) busy: bool,
/// Per scan index: the table's change token after its last complete read.
pub(crate) change_tokens: HashMap<usize, ScanToken>,
/// The token read before the current scan's first page, recorded once the scan completes.
Expand All @@ -221,6 +230,15 @@ pub(crate) struct LocationMetadataSync {
pub(crate) reader: Option<LocationCompiler>,
}

/// The metadata reader's answer to one read.
#[derive(Debug)]
pub(crate) enum MetadataReply {
/// A page, or `None` when the read failed.
Page(Option<WarehouseHistoryPage>),
/// Another process holds the warehouse file; nothing was read.
Busy,
}

/// A table's change token after a complete read, and what that read proves about the mirror.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ScanToken {
Expand All @@ -240,6 +258,23 @@ impl LocationMetadataSync {
}
}

/// Whether an idle location starts its one outstanding sync: first, requested, after a run, or safety.
pub(crate) fn sync_due(&self, runs_ended_at: Option<DateTime<Utc>>) -> bool {
let ran = match (self.cycle_started_at, runs_ended_at) {
(None, _) => true,
(Some(started_at), Some(ended_at)) => ended_at >= started_at,
(Some(_), None) => false,
};
ran || (self.safety_seconds > 0 && tokio::time::Instant::now() >= self.safety_at)
}

/// Start the sync `sync_due` asked for, at server time `now`.
pub(crate) fn begin_sync(&mut self, now: DateTime<Utc>) {
self.idle = false;
self.cycle_started_at = Some(now);
self.next_at = tokio::time::Instant::now();
}

/// Failed samples revisit catalog before retrying; no frozen delivery is discarded.
pub(crate) fn retry_read(&mut self) {
self.statistics = None;
Expand Down
Loading
Loading