diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index e2282f59ae..e5df5d26fe 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -530,22 +530,6 @@ export interface paths { patch?: never; trace?: never; }; - "/api/control/locations/{location_id}/targets/{target}/mirrored-totals": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get: operations["readPersonMirroredTotals"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/api/control/locations/{location_id}/targets/{target}/node-states": { parameters: { query?: never; @@ -1234,23 +1218,6 @@ export interface paths { patch?: never; trace?: never; }; - "/api/locations/{location_id}/targets/{target}/metadata/retention": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** How much state history a full read delivers; older servers answer 404 and the location reads it all. */ - get: operations["read_metadata_retention"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/api/locations/{location_id}/targets/{target}/metadata/statistics-due": { parameters: { query?: never; @@ -1765,7 +1732,7 @@ export interface components { ChangedBuildReport: { changed: string[]; downstream: string[]; - /** @description Why the plan failed, sent with empty lists; only newer servers accept it. */ + /** @description Why the plan failed, sent with empty lists. */ failing?: string | null; /** Format: int64 */ generation: number; @@ -2220,6 +2187,11 @@ export interface components { /** @enum {string} */ kind: "warehouse_sync"; location: components["schemas"]["WarehouseStateLocation"]; + /** + * Format: int64 + * @description On the report of a table's last delivered page, the records the server then mirrored of that table across the target's state locations. + */ + mirrored_records?: number | null; /** * Format: date-time * @description On the report that completes or stops a sync, when the next safety sync is due unless a run ends first; absent otherwise, where no timed sync follows, and from older locations. @@ -2421,6 +2393,8 @@ export interface components { /** Format: int64 */ generation: number; kind: components["schemas"]["WarehouseHistoryKind"]; + /** @description The table's read ends with this page, so the reply counts the records mirrored of it. */ + last_page?: boolean; records: Record[]; scan?: null | components["schemas"]["MetadataScan"]; /** Format: uuid */ @@ -2452,12 +2426,6 @@ export interface components { physical_generation_id: string; target_name: string; }; - /** @description How many records the server holds for one mirrored state table of a target. */ - MirroredTotal: { - /** Format: int64 */ - records: number; - table: components["schemas"]["WarehouseHistoryKind"]; - }; MutationResponse: { applied: boolean; }; @@ -2955,12 +2923,12 @@ export interface components { synced_at?: string | null; /** @description Set when the mirror is still at the read's known version: the records are left out. */ unchanged?: boolean; - /** @description The mirror version these records were read at; sent only to a versioned read. */ - version?: string | null; + /** @description The mirror version these records were read at. */ + version: string; }; /** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */ PreviewCopyRead: { - /** @description Whether the copy carries consumption history; sent only when set, as older servers reject it. */ + /** @description Whether the copy carries consumption history. */ consumption?: boolean; /** Format: int64 */ generation: number; @@ -2968,8 +2936,6 @@ export interface components { known_version?: string | null; /** Format: uuid */ session_id: string; - /** @description Whether the location holds copies by mirror version; sent only when set, as older servers reject it. */ - versioned?: boolean; }; /** @description The newest successful build of a node, which a `cursor_max` observation must come from. */ PreviewLatestRun: { @@ -4102,9 +4068,6 @@ export interface components { }; /** @description What went wrong reading one page, without the warehouse's message, which may name hosts or users. */ WarehouseReadFailure: { - /** @enum {string} */ - kind: "reader_start"; - } | { /** @enum {string} */ kind: "read_error"; } | { @@ -4150,11 +4113,6 @@ export interface components { warehouse_size: string; warehouse_type?: string | null; }; - /** @description The newest versions per node or check a full read of state history needs to deliver. */ - WarehouseRetention: { - /** Format: int32 */ - state_history_versions: number; - }; /** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */ WarehouseStateLocation: { database?: string | null; @@ -4180,6 +4138,11 @@ export interface components { now: string; /** Format: date-time */ runs_ended_at?: string | null; + /** + * Format: int32 + * @description The newest versions per node or check a full read of state history needs to deliver. + */ + state_history_versions: number; }; /** @description A registered reader asks whether a run ended since its last synchronization began. */ WarehouseSyncDueRead: { @@ -4222,6 +4185,11 @@ export interface components { }; WarehouseSyncResponse: { applied: boolean; + /** + * Format: int64 + * @description Records the server mirrors of this table for the target, counted after a read's last page. + */ + mirrored_records?: number | null; }; /** @description Derive progress from committed mirror rows, scoped to the active physical reader. */ WarehouseWatermarkRead: { @@ -5586,44 +5554,6 @@ export interface operations { }; }; }; - readPersonMirroredTotals: { - parameters: { - query?: never; - header?: never; - path: { - location_id: string; - target: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Records the server mirrors of each state table of this target, in sync order */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["MirroredTotal"][]; - }; - }; - /** @description Session rejected */ - 401: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["ApiErrorBody"]; - }; - }; - }; - }; readPersonNodeStates: { parameters: { query?: never; @@ -7599,42 +7529,6 @@ export interface operations { }; }; }; - read_metadata_retention: { - parameters: { - query?: never; - header?: never; - path: { - location_id: string; - target: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["WarehouseRetention"]; - }; - }; - /** @description Invalid credential */ - 401: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Location not authorized */ - 403: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - }; - }; statistics_due: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs index 1e64fe1398..5252ad6ac4 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs @@ -34,6 +34,7 @@ use crate::native_commands::_helpers::command::terminal::{live_screen, stderr_co use crate::native_commands::_helpers::location::sync_report::{ metadata_health, read_timeout_seconds, rows_text, scan_line, }; +use crate::native_commands::constants::PLAN_NOT_PRODUCED; use crate::native_commands::constants::SCREEN_INDENT; use crate::native_commands::constants::{ ADMIN_COMMAND, CHANGED_PATHS_SHOWN, COLUMNS_VARIABLE, FALLBACK_COLUMNS, @@ -42,7 +43,6 @@ use crate::native_commands::constants::{ METADATA_SYNC_CHECK_SECONDS, SAFETY_SYNC_ENV, SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, }; -use crate::native_commands::constants::{METADATA_PAGE_SECONDS, PLAN_NOT_PRODUCED}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; use crate::native_commands::main::_run_control::run_control; @@ -502,7 +502,6 @@ pub(crate) fn metadata_sync( server_offset: chrono::Duration::zero(), safety_seconds, safety_at: Instant::now() + Duration::from_secs(safety_seconds), - busy: false, failed_reads: 0, halted: false, change_tokens: HashMap::new(), @@ -511,10 +510,8 @@ pub(crate) fn metadata_sync( mirror_holds_rows: false, reader: None, state_history_versions: None, - page_seconds: METADATA_PAGE_SECONDS, progress: SyncProgress::new(), reads_wake_compute: false, - safety_explicit: true, } } @@ -597,6 +594,7 @@ pub(crate) fn freeze_history_page( location: state.location().clone(), complete: page.next_after.is_none(), }), + last_page: page.next_after.is_none(), }; if serde_json::to_vec(&batch) .map_err(|error| ControlError::Invalid(error.to_string()))? @@ -648,10 +646,7 @@ pub(crate) async fn begin_after_run( }; let ran = state.sync_due(due.runs_ended_at); if ran { - state.state_history_versions = match client.metadata_retention(target).await { - Ok(retention) => retention.map(|retention| retention.state_history_versions), - Err(error) => return (state, Err(error)), - }; + state.state_history_versions = Some(due.state_history_versions); state.begin_sync(due.now); } (state, Ok(ran)) @@ -670,12 +665,22 @@ pub(crate) fn acknowledge_metadata_page(mut state: LocationMetadataSync) -> Loca if state.kind() == WarehouseHistoryKind::Statistics { state.statistics_limit = MAX_HISTORY_RECORDS; } + if let Some(count) = state.progress.scan_mirrored { + let kinds = WarehouseHistoryKind::ALL.len(); + let kind = state.index % kinds; + state + .change_tokens + .iter_mut() + .filter(|(index, _)| *index % kinds == kind) + .for_each(|(_, recorded)| recorded.mirrored = Some(count)); + } match state.scan_token.take() { Some(token) => { let recorded = ScanToken { token, reconciled: state.reconciling(), delivered: state.scan_delivered || state.observed_since.is_some(), + mirrored: state.progress.scan_mirrored, }; state.change_tokens.insert(state.index, recorded); } @@ -743,6 +748,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync state.progress.sync_rows += state.progress.scan_rows; state.progress.scan_rows = 0; state.progress.scan_unchanged = false; + state.progress.scan_mirrored = None; state.progress.scan_started = Instant::now(); state.index += 1; state.scan_id = state.server_scan_id(); @@ -874,7 +880,6 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result>, ) -> Result< @@ -1375,7 +1380,6 @@ pub(crate) async fn read_metadata_page( )>, ControlError, > { - state.busy = false; if let Some(statistics) = &state.statistics && statistics.names.is_empty() { @@ -1400,7 +1404,7 @@ pub(crate) async fn read_metadata_page( } }, }; - let seconds = read_timeout_seconds(&state); + let seconds = read_timeout_seconds(&state, page_seconds); tokio::pin!(stopped); let read = tokio::select! { result = &mut stopped => { @@ -1417,7 +1421,6 @@ pub(crate) async fn read_metadata_page( Ok(page) } Ok((reader, Ok(MetadataReply::Busy))) => { - state.busy = true; state.reader = Some(reader); Err(ReadFailure::Busy) } @@ -1633,14 +1636,13 @@ pub(crate) async fn settle_publication( stopped: impl Future>, ) -> Result, ControlError> { let (mut metadata, deferred) = if connect { - let mut sync: LocationMetadataSync = metadata_sync( + let sync: LocationMetadataSync = metadata_sync( (locations, generation), ( options.metadata_reconcile_seconds, options.safety_sync_seconds, ), ); - sync.safety_explicit = options.safety_sync_explicit; (Some(sync), None) } else { (metadata, Some((locations, generation))) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/held_copy.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/held_copy.rs index 2bdca9d425..ee73c74a36 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/held_copy.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/held_copy.rs @@ -3,10 +3,8 @@ use std::path::PathBuf; use std::sync::{Mutex, PoisonError}; -use sqlbuild_orchestration::platform::models::LocationWorkClient; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; -use uuid::Uuid; +use sqlbuild_orchestration::warehouse_metadata::models::PreviewCopy; use crate::native_commands::models::{HeldCopy, LocationServe}; @@ -29,28 +27,6 @@ pub(crate) fn held_copy( }) } -/// A versioned read; a server without mirror versions rejects the fields, so it is read unversioned. -pub(crate) async fn read_copy( - client: &LocationWorkClient, - target: &str, - read: PreviewCopyRead, -) -> Result { - match client.preview_copy(target, &read).await { - Ok(copy) => Ok(copy), - Err(_) => client - .preview_copy( - target, - &PreviewCopyRead { - versioned: false, - known_version: None, - ..read - }, - ) - .await - .map_err(|error| error.to_string()), - } -} - /// The location's one held copy file for its target, which each newer copy overwrites. fn held_copy_path(options: &LocationServe) -> PathBuf { let target: String = options @@ -70,7 +46,7 @@ fn held_copy_path(options: &LocationServe) -> PathBuf { )) } -/// Atomically replace the held copy file; a copy without a version gets a fresh one. +/// Atomically replace the held copy file. pub(crate) fn hold_copy( held_copies: &Mutex>, options: &LocationServe, @@ -79,10 +55,7 @@ pub(crate) fn hold_copy( ) -> Result { let path: PathBuf = held_copy_path(options); let written: PathBuf = path.with_extension(format!("{}.tmp", std::process::id())); - let version: String = copy - .version - .clone() - .unwrap_or_else(|| Uuid::now_v7().to_string()); + let version: String = copy.version.clone(); std::fs::write( &written, serde_json::to_vec(©).map_err(std::io::Error::other)?, diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs index 98c1a4a737..e488f2b107 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs @@ -50,17 +50,18 @@ pub(crate) async fn failed_read( (started, failure): (Instant, ReadFailure), stopped: impl Future>, ) -> Result, ControlError> { - let stage = if state.busy { + let busy: bool = failure == ReadFailure::Busy; + let stage = if busy { WarehouseSyncFailure::Busy } else { WarehouseSyncFailure::Read }; - let halts: bool = !state.busy && state.halts(failure.page_size()); + let halts: bool = !busy && state.halts(failure.page_size()); tokio::select! { result = stopped => { result?; return Ok(None); }, () = report_metadata_health(&state, (client, poll), (started, Some(stage), Some((&failure, halts)))) => {}, } - if state.busy { + if busy { state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); return Ok(Some(state)); } @@ -135,6 +136,7 @@ pub(crate) async fn deliver_metadata_page( } }; state.applied |= response.applied; + state.progress.scan_mirrored = response.mirrored_records; tokio::select! { result = &mut stopped => { result?; return Ok(None); }, () = metadata_health(&state, (client, poll), (started, None)) => {}, diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_report.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_report.rs index 4332b36cc1..45cc7ab425 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_report.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_report.rs @@ -131,6 +131,7 @@ pub(crate) fn metadata_health_of( ..sync_progress(state, failure.is_none()) }), next_sync_at, + mirrored_records: state.progress.scan_mirrored, } } @@ -167,12 +168,12 @@ pub(crate) async fn metadata_health( report_metadata_health(state, (client, poll), (started, failure, None)).await; } -/// How long one page may take: statistics scale with their relations, other pages get a fixed allowance. -pub(crate) fn read_timeout_seconds(state: &LocationMetadataSync) -> u64 { +/// How long one page may take: statistics scale with their relations, other pages get `page_seconds`. +pub(crate) fn read_timeout_seconds(state: &LocationMetadataSync, page_seconds: u64) -> u64 { if state.kind() == WarehouseHistoryKind::Statistics { METADATA_POLL_SECONDS + STATISTICS_RELATION_SECONDS * state.statistics_limit as u64 } else { - state.page_seconds + page_seconds } } @@ -185,20 +186,15 @@ pub(crate) fn page_limit(state: &LocationMetadataSync) -> usize { } } -/// Say a stopped sync resumes; `one_attempt` when one more failure stops it again. -pub(crate) fn resumed_line(one_attempt: bool) { +/// Say a stopped sync resumes, with failed reads backing off again before they stop it. +pub(crate) fn resumed_line() { drop(write_output(&[( Stream::Stderr, "Retrying the stopped warehouse history synchronization.\n", )])); location_warning( "warehouse sync retrying", - if one_attempt { - "one attempt; another failed read stops it again" - } else { - "on request; failed reads back off again before they stop it" - } - .to_owned(), + "failed reads back off again before they stop it".to_owned(), ); } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index f3c4867748..5c4216cbe4 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -17,6 +17,7 @@ use crate::native_commands::_helpers::location::sync_outcomes::{ deliver_metadata_page, failed_read, retry_metadata, }; use crate::native_commands::_helpers::location::sync_report::{metadata_health, resumed_line}; +use crate::native_commands::constants::METADATA_PAGE_SECONDS; use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch}; /// One bounded source or mirror page leaves named work polling a turn between pages. @@ -43,7 +44,7 @@ pub(crate) async fn sync_location_metadata( }; state = next; match due { - Ok(true) if halted => resumed_line(state.failed_reads > 0), + Ok(true) if halted => resumed_line(), Ok(true) => {} Ok(false) => return Ok(Some(state)), Err(error) => return retry_metadata(state, error), @@ -70,8 +71,12 @@ pub(crate) async fn sync_location_metadata( return retry_metadata(state, error); } } - let Some((read, page)) = - read_metadata_page((options, script), state, stopped.as_mut()).await? + let Some((read, page)) = read_metadata_page( + (options, script, METADATA_PAGE_SECONDS), + state, + stopped.as_mut(), + ) + .await? else { return Ok(None); }; @@ -89,11 +94,11 @@ pub(crate) async fn sync_location_metadata( return Ok(Some(state)); } if page.unchanged { + state.skip_unchanged(); tokio::select! { result = &mut stopped => { result?; return Ok(None); }, () = metadata_health(&state, (client, poll), (started, None)) => {}, } - state.progress.scan_unchanged = true; return finish_metadata_page(next_scan(state)).await; } state = freeze_history_page(state, page, poll)?; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs index d6cdcde0da..0ba6251feb 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -11,9 +11,7 @@ use tokio::sync::mpsc; use uuid::Uuid; use crate::native_commands::_helpers::location::execution::{plan_failure, without_diffs}; -use crate::native_commands::_helpers::location::held_copy::{ - HELD_COPY, held_copy, hold_copy, read_copy, -}; +use crate::native_commands::_helpers::location::held_copy::{HELD_COPY, held_copy, hold_copy}; use crate::native_commands::_helpers::location::plan_request::{ plan_arguments, run_plan, selected_names, }; @@ -38,19 +36,13 @@ pub(super) async fn plan_location( .is_some_and(|changed| !matches!(changed, Value::Null | Value::Bool(false))); let held: Option = held_copy(&HELD_COPY, (&options.target, generation), consumption); let read_consumption: bool = consumption || held.as_ref().is_some_and(|held| held.consumption); - let copy = match read_copy( - client, - &options.target, - PreviewCopyRead { - session_id, - generation, - consumption: read_consumption, - versioned: true, - known_version: held.as_ref().map(|held| held.version.clone()), - }, - ) - .await - { + let read = PreviewCopyRead { + session_id, + generation, + consumption: read_consumption, + known_version: held.as_ref().map(|held| held.version.clone()), + }; + let copy = match client.preview_copy(&options.target, &read).await { Ok(copy) => copy, Err(error) => { return Ok(Some(without_diffs(plan_failure(&format!( diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 3967c069de..b7ef802894 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -162,8 +162,6 @@ pub(crate) struct LocationServe { pub(crate) metadata_reconcile_seconds: u32, /// Seconds between safety syncs of an idle location; `0` turns them off. pub(crate) safety_sync_seconds: u64, - /// A flag, the environment or the project set the interval, rather than the default. - pub(crate) safety_sync_explicit: bool, /// `sqb dev` reads events on stdout and sends control lines on stdin. pub(crate) dev_events: bool, } @@ -265,8 +263,6 @@ pub(crate) struct LocationMetadataSync { pub(crate) safety_seconds: u64, /// When an idle location syncs even though no run ended and nobody asked. pub(crate) safety_at: tokio::time::Instant, - /// The last read found the warehouse file held by another process, such as a DuckDB build. - pub(crate) busy: bool, /// Failed reads in a row that did not shrink the page; a delivered page resets it. pub(crate) failed_reads: u32, /// Failed reads in a row stopped the sync; it rests idle until a run, a request, a publish or a safety sync. @@ -281,16 +277,12 @@ pub(crate) struct LocationMetadataSync { pub(crate) mirror_holds_rows: bool, /// One warm runtime interpreter and warehouse session serve every page of a synchronization. pub(crate) reader: Option, - /// The server's state-history limit, read as each synchronization begins; `None` from older servers. + /// The server's state-history limit, read as each synchronization begins; `None` before the first. pub(crate) state_history_versions: Option, - /// Seconds a state-table or catalog page may take before the read counts as failed. - pub(crate) page_seconds: u64, /// Progress of the current synchronization, for its per-table lines and summary. pub(crate) progress: SyncProgress, /// The adapter's history reads start or wake billed compute, as the publication's connection check reports. pub(crate) reads_wake_compute: bool, - /// Someone set the safety interval, so it applies even where reads wake compute. - pub(crate) safety_explicit: bool, /// The server applied a delivered page of the current synchronization, so the synced copy changed. pub(crate) applied: bool, /// The generation was published after the last plan of what changed, which waits for this sync. @@ -306,6 +298,8 @@ pub(crate) struct SyncProgress { pub(crate) scan_rows: u64, /// The current table's change token showed it unchanged, so nothing was read. pub(crate) scan_unchanged: bool, + /// Records the server mirrors of the current table, as it counted after the table's last page. + pub(crate) scan_mirrored: Option, } impl SyncProgress { @@ -317,6 +311,7 @@ impl SyncProgress { sync_rows: 0, scan_rows: 0, scan_unchanged: false, + scan_mirrored: None, } } } @@ -358,8 +353,7 @@ impl ReadFailure { pub(crate) fn cause(&self) -> Option { match self { Self::Busy => None, - Self::ReaderStart(_) => Some(WarehouseReadFailure::ReaderStart {}), - Self::Read(_) => Some(WarehouseReadFailure::ReadError {}), + Self::ReaderStart(_) | Self::Read(_) => Some(WarehouseReadFailure::ReadError {}), Self::TimedOut(seconds) => Some(WarehouseReadFailure::TimedOut { seconds: *seconds }), } } @@ -383,6 +377,8 @@ pub(crate) struct ScanToken { pub(crate) reconciled: bool, /// The read delivered rows or read on top of mirrored rows, so an empty mirror lost rows. pub(crate) delivered: bool, + /// The server's newest count of this kind's mirror over every state location, which a skipped read repeats. + pub(crate) mirrored: Option, } impl LocationMetadataSync { @@ -422,7 +418,7 @@ impl LocationMetadataSync { /// Whether an idle location skips interval syncs because history reads wake billed compute. pub(crate) fn defers_safety_sync(&self) -> bool { - self.reads_wake_compute && !self.safety_explicit + self.reads_wake_compute } /// A scan id timed by the server's clock: a synced plan trusts a record as of its scan's start, never after a read. @@ -436,11 +432,12 @@ impl LocationMetadataSync { )) } - /// Start the sync `sync_due` asked for at server time `now`; a stopped sync restarts its pass. + /// Start the sync `sync_due` asked for at server time `now`; a stopped sync restarts its pass with a fresh failure count. pub(crate) fn begin_sync(&mut self, now: DateTime) { self.server_offset = now - Utc::now(); self.scan_id = self.server_scan_id(); if std::mem::take(&mut self.halted) { + self.failed_reads = 0; self.index = 0; self.source_after = None; self.observed_since = None; @@ -520,6 +517,15 @@ impl LocationMetadataSync { && (self.reconcile || self.kind() == WarehouseHistoryKind::Catalog) } + /// Skip a table unchanged since its recorded read, reporting the newest mirrored count of its kind again. + pub(crate) fn skip_unchanged(&mut self) { + self.progress.scan_unchanged = true; + self.progress.scan_mirrored = self + .change_tokens + .get(&self.index) + .and_then(|recorded| recorded.mirrored); + } + /// The recorded token a scan may skip on, given whether it reconciles and the mirror holds rows. pub(crate) fn unchanged_token(&self) -> Option<&str> { let recorded = self.change_tokens.get(&self.index)?; diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs index ec80fc150e..ec83c94d0a 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -61,6 +61,7 @@ pub(super) fn acknowledge_complete_pages( generation: 1, records: Vec::new(), scan: None, + last_page: false, }, next_source_after: None, }); @@ -106,7 +107,6 @@ pub(super) fn published_location( image_id: None, metadata_reconcile_seconds: 3600, safety_sync_seconds: 1800, - safety_sync_explicit: false, dev_events: false, }; let client = connect_location_work((&options.server_url, "token"), options.location_id) @@ -130,6 +130,5 @@ pub(super) fn published_location( let mut state = metadata.expect("a connected publication starts a sync"); state.begin_sync(chrono::Utc::now()); state.idle = false; - state.page_seconds = 5; ((options, script), (client, poll), state) } diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index 71c88d2577..a5c34766e6 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -20,7 +20,8 @@ use crate::native_commands::tests::test_types::{ }; use crate::native_commands::tests::test_types::{ MetadataBackoffTestCase, MetadataRetryTestCase, SafetyResumeTestCase, SafetySyncIdleTestCase, - SafetySyncSettingTestCase, StoppedPassTestCase, SyncCoalescingTestCase, UnchangedScanTestCase, + SafetySyncSettingTestCase, SkippedCountTestCase, StoppedPassTestCase, SyncCoalescingTestCase, + UnchangedScanTestCase, }; use crate::native_commands::tests::test_types::{ RowsTextTestCase, ScanLineTestCase, SyncProgressTestCase, WakingIdleTestCase, @@ -178,7 +179,8 @@ fn given_repeated_failed_reads_when_retrying_then_only_page_size_errors_shrink_a } #[test] -fn given_a_stopped_sync_when_a_run_or_its_safety_sync_is_due_then_it_makes_one_attempt() { +fn given_a_stopped_sync_when_a_run_or_its_safety_sync_resumes_it_then_failed_reads_back_off_again() +{ let test_cases = [ SafetyResumeTestCase { description: "before the safety sync is due the sync stays stopped", @@ -189,12 +191,12 @@ fn given_a_stopped_sync_when_a_run_or_its_safety_sync_is_due_then_it_makes_one_a expected_outcome: (false, false, None, true), }, SafetyResumeTestCase { - description: "a due safety sync resumes it, and one more failure stops it for another interval", + description: "a due safety sync resumes it with a fresh count, so one more failure backs off", safety_seconds: 1800, due_in: Duration::ZERO, billed: false, ran: false, - expected_outcome: (true, true, None, true), + expected_outcome: (true, true, Some(Duration::from_secs(30)), false), }, SafetyResumeTestCase { description: "with safety syncs off only a run, a request or a publication resumes it", @@ -213,20 +215,20 @@ fn given_a_stopped_sync_when_a_run_or_its_safety_sync_is_due_then_it_makes_one_a expected_outcome: (false, false, None, true), }, SafetyResumeTestCase { - description: "on a billed warehouse a run that ends after the stop resumes it", + description: "on a billed warehouse a run that ends after the stop resumes it with a fresh count", safety_seconds: 1800, due_in: Duration::from_secs(600), billed: true, ran: true, - expected_outcome: (true, true, None, true), + expected_outcome: (true, true, Some(Duration::from_secs(30)), false), }, ]; for test_case in test_cases { let mut state = metadata_sync((one_location(), 1), (3600, test_case.safety_seconds)); state.reads_wake_compute = test_case.billed; - state.safety_explicit = !test_case.billed; state.begin_sync(Utc::now()); let stopped: Vec> = (0..5).map(|_| state.fail_read(false)).collect(); + let stopped_for: Duration = state.safety_at.saturating_duration_since(Instant::now()); state.safety_at = Instant::now() + test_case.due_in; let runs_ended_at = test_case .ran @@ -239,10 +241,7 @@ fn given_a_stopped_sync_when_a_run_or_its_safety_sync_is_due_then_it_makes_one_a ( stopped[4], (resumed, paging, wait, state.halted), - (state.safety_at.saturating_duration_since(Instant::now()) - + Duration::from_secs(1)) - .as_secs() - >= test_case.safety_seconds, + (stopped_for + Duration::from_secs(1)).as_secs() >= test_case.safety_seconds, ), (None, test_case.expected_outcome, true), "{}", @@ -408,6 +407,70 @@ fn given_recorded_change_token_when_scanning_again_then_only_proven_unchanged_re } } +#[test] +fn given_a_table_skipped_as_unchanged_when_reporting_health_then_the_newest_mirrored_count_of_its_kind_repeats() + { + let test_cases = [ + SkippedCountTestCase { + description: "one state location repeats its own read's count", + locations: 1, + later_read: None, + expected_mirrored: Some(120), + }, + SkippedCountTestCase { + description: "a later read of the same kind in another location replaces the repeated count", + locations: 2, + later_read: Some(130), + expected_mirrored: Some(130), + }, + ]; + for test_case in test_cases { + let locations: Vec = (0..test_case.locations) + .map(|position| WarehouseStateLocation { + database: Some("analytics".into()), + schema: format!("state_{position}"), + }) + .collect(); + let kinds = WarehouseHistoryKind::ALL.len(); + let skipped = (test_case.locations - 1) * kinds; + let mut state = metadata_sync((locations, 1), (3600, 0)); + state.index = skipped; + state.scan_token = Some("t1".into()); + state.scan_delivered = true; + state.progress.scan_mirrored = Some(120); + let state = acknowledge_complete_pages(state, 1); + assert_eq!( + state.progress.scan_mirrored, None, + "{}: the next table starts without a count", + test_case.description + ); + let mut state = test_case + .later_read + .into_iter() + .fold(state, |mut state, count| { + state.index = 0; + state.scan_token = Some("t2".into()); + state.scan_delivered = true; + state.progress.scan_mirrored = Some(count); + acknowledge_complete_pages(state, 1) + }); + state.index = skipped; + state.skip_unchanged(); + let health = metadata_health_of(&state, (Instant::now(), None, None), Utc::now()); + let LocationHealth::WarehouseSync { + mirrored_records, .. + } = health + else { + panic!("{}: not a warehouse sync report", test_case.description); + }; + assert_eq!( + mirrored_records, test_case.expected_mirrored, + "{}", + test_case.description + ); + } +} + #[test] fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync_runs() { let test_cases = [ @@ -606,35 +669,28 @@ fn given_a_state_history_limit_when_requesting_history_then_only_full_reads_of_b description: "a first read of builds stops at the limit", kind: WarehouseHistoryKind::Fingerprints, incremental: false, - server_limit: Some(5), + server_limit: 5, expected_newest_versions: Some(5), }, NewestVersionsTestCase { description: "a first read of check results stops at the limit", kind: WarehouseHistoryKind::AuditResults, incremental: false, - server_limit: Some(20), + server_limit: 20, expected_newest_versions: Some(20), }, NewestVersionsTestCase { description: "an incremental read delivers every new build; pruning bounds the store", kind: WarehouseHistoryKind::Fingerprints, incremental: true, - server_limit: Some(5), + server_limit: 5, expected_newest_versions: None, }, NewestVersionsTestCase { description: "other tables are never limited", kind: WarehouseHistoryKind::NodeResults, incremental: false, - server_limit: Some(5), - expected_newest_versions: None, - }, - NewestVersionsTestCase { - description: "a server without the setting gets the full history", - kind: WarehouseHistoryKind::Fingerprints, - incremental: false, - server_limit: None, + server_limit: 5, expected_newest_versions: None, }, ]; @@ -644,7 +700,7 @@ fn given_a_state_history_limit_when_requesting_history_then_only_full_reads_of_b .iter() .position(|kind| *kind == test_case.kind) .expect("kind is listed"); - state.state_history_versions = test_case.server_limit; + state.state_history_versions = Some(test_case.server_limit); state.observed_since = test_case.incremental.then(Utc::now); let request: serde_json::Value = serde_json::from_slice(&history_request(&state).expect("request")).expect("JSON"); @@ -712,7 +768,6 @@ fn given_a_failing_metadata_read_when_the_page_fails_then_the_cause_kind_and_pag image_id: None, metadata_reconcile_seconds: 3600, safety_sync_seconds: 0, - safety_sync_explicit: false, dev_events: false, }; let mut state = metadata_sync((one_location(), 1), (3600, 0)); @@ -720,13 +775,12 @@ fn given_a_failing_metadata_read_when_the_page_fails_then_the_cause_kind_and_pag .iter() .position(|kind| *kind == test_case.kind) .expect("kind is listed"); - state.page_seconds = 1; let (state, page) = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .expect("runtime") .block_on(read_metadata_page( - (&options, &test_case.script), + (&options, &test_case.script, 1), state, std::future::pending(), )) @@ -884,13 +938,11 @@ fn given_a_delivered_page_when_reporting_health_then_progress_completes_only_on_ } #[test] -fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_or_a_set_interval_sync() - { +fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_or_requests_sync() { let test_cases = [ WakingIdleTestCase { - description: "the default interval never wakes compute on a timer", + description: "the interval, set or default, never wakes compute on a timer", wakes: true, - explicit: false, run_ended: false, requested: false, expected_due: false, @@ -898,15 +950,6 @@ fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_o WakingIdleTestCase { description: "metadata-only reads keep the default interval", wakes: false, - explicit: false, - run_ended: false, - requested: false, - expected_due: true, - }, - WakingIdleTestCase { - description: "an interval someone set still applies", - wakes: true, - explicit: true, run_ended: false, requested: false, expected_due: true, @@ -914,7 +957,6 @@ fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_o WakingIdleTestCase { description: "a run that ended still syncs", wakes: true, - explicit: false, run_ended: true, requested: false, expected_due: true, @@ -922,7 +964,6 @@ fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_o WakingIdleTestCase { description: "a request still syncs", wakes: true, - explicit: false, run_ended: false, requested: true, expected_due: true, @@ -933,7 +974,6 @@ fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_o state.cycle_started_at = Some(Utc::now()); state.safety_at = Instant::now() - Duration::from_secs(1); state.reads_wake_compute = test_case.wakes; - state.safety_explicit = test_case.explicit; test_case.requested.then(|| state.request_sync()); let runs_ended_at = test_case .run_ended @@ -1009,13 +1049,13 @@ fn given_a_failed_read_when_reporting_health_then_it_carries_cause_error_and_whe ), }, FailedReadHealthTestCase { - description: "a reader that did not start reports its own cause", + description: "a reader that did not start reports a read error that names it", failure: ReadFailure::ReaderStart("no such file".into()), failures_before: 0, billed: false, expected_report: ( false, - Some(WarehouseReadFailure::ReaderStart {}), + Some(WarehouseReadFailure::ReadError {}), Some("the metadata reader did not start: no such file"), false, None, @@ -1029,7 +1069,6 @@ fn given_a_failed_read_when_reporting_health_then_it_carries_cause_error_and_whe .position(|kind| *kind == WarehouseHistoryKind::Fingerprints) .expect("kind is listed"); state.reads_wake_compute = test_case.billed; - state.safety_explicit = false; state.begin_sync(Utc::now()); (0..test_case.failures_before).for_each(|_| { let _ = state.fail_read(false); @@ -1155,7 +1194,7 @@ fn given_a_published_location_when_its_sync_stops_before_any_page_succeeds_then_ let mut state = state; for _ in 0..METADATA_FAILURE_LIMIT { let (read, page) = - read_metadata_page((&options, &script), state, std::future::pending()) + read_metadata_page((&options, &script, 5), state, std::future::pending()) .await .expect("no control error") .expect("not stopped"); diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_sync_schedule.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_sync_schedule.rs index 76e437ac8c..603b926421 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_sync_schedule.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_sync_schedule.rs @@ -20,7 +20,6 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: None, safety_seconds: 1800, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: Some(1800), }, SyncCompletionTestCase { @@ -30,7 +29,6 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: None, safety_seconds: 1800, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: None, }, SyncCompletionTestCase { @@ -40,7 +38,6 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: None, safety_seconds: 1800, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: None, }, SyncCompletionTestCase { @@ -50,7 +47,6 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: Some(WarehouseSyncFailure::Delivery), safety_seconds: 1800, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: None, }, SyncCompletionTestCase { @@ -60,35 +56,22 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: None, safety_seconds: 0, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: None, }, SyncCompletionTestCase { - description: "where reads wake compute the default interval names no next sync", + description: "where reads wake compute no next sync is named, whatever the interval", acknowledged_scans: last, more_pages: false, failure: None, safety_seconds: 1800, reads_wake_compute: true, - safety_explicit: false, expected_next_in_seconds: None, }, - SyncCompletionTestCase { - description: "where reads wake compute an interval someone set still names the next sync", - acknowledged_scans: last, - more_pages: false, - failure: None, - safety_seconds: 1800, - reads_wake_compute: true, - safety_explicit: true, - expected_next_in_seconds: Some(1800), - }, ]; let now = Utc::now(); for test_case in test_cases { let mut state = metadata_sync((one_location(), 1), (3600, test_case.safety_seconds)); state.reads_wake_compute = test_case.reads_wake_compute; - state.safety_explicit = test_case.safety_explicit; state.begin_sync(now); state = acknowledge_complete_pages(state, test_case.acknowledged_scans); state.pending = Some(LocationMetadataPage { @@ -98,6 +81,7 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa generation: 1, records: Vec::new(), scan: None, + last_page: false, }, next_source_after: test_case.more_pages.then(|| vec![Some("orders".into())]), }); diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 076eb5dfaa..2bf84f524f 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -188,7 +188,7 @@ pub(crate) struct NewestVersionsTestCase { pub(crate) description: &'static str, pub(crate) kind: WarehouseHistoryKind, pub(crate) incremental: bool, - pub(crate) server_limit: Option, + pub(crate) server_limit: u16, pub(crate) expected_newest_versions: Option, } @@ -216,7 +216,6 @@ pub(super) struct SyncCompletionTestCase { pub(super) failure: Option, pub(super) safety_seconds: u64, pub(super) reads_wake_compute: bool, - pub(super) safety_explicit: bool, pub(super) expected_next_in_seconds: Option, } @@ -288,7 +287,6 @@ pub(crate) struct SyncProgressTestCase { pub(crate) struct WakingIdleTestCase { pub(crate) description: &'static str, pub(crate) wakes: bool, - pub(crate) explicit: bool, pub(crate) run_ended: bool, pub(crate) requested: bool, pub(crate) expected_due: bool, @@ -307,6 +305,14 @@ pub(super) struct ReplanTestCase { pub(super) expected_replans: Vec, } +pub(crate) struct SkippedCountTestCase { + pub(crate) description: &'static str, + pub(crate) locations: usize, + /// The count a read of the same kind in the first location reports after the skipped one's read. + pub(crate) later_read: Option, + pub(crate) expected_mirrored: Option, +} + /// A scan whose location clock runs ten minutes ahead of the server's. pub(super) struct FastClockScanTestCase { pub(super) description: &'static str, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index fd5bbbf923..f1e5d9fe70 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -195,10 +195,6 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/health", get(crate::api::_helpers::resources::person_reads::health), ) - .route( - "/api/control/locations/{location_id}/targets/{target}/mirrored-totals", - get(crate::api::_helpers::resources::person_reads::mirrored_totals), - ) .route( "/api/control/platform/problems", get(crate::api::_helpers::resources::person_reads::problems), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs index 123eb3a3ac..032cc5426a 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs @@ -26,10 +26,9 @@ use crate::node_states::main::read_person_node_states::read_person_node_states; use crate::node_states::models::{CachedJson, NodeStateCache, NodeStatesRecord}; use crate::platform::main::health::read_person_location_health::read_person_location_health; use crate::platform::main::person_reads::list_person_platform_events::list_person_platform_events; -use crate::platform::main::person_reads::read_person_mirrored_totals::read_person_mirrored_totals; use crate::platform::main::person_reads::read_person_problems::read_person_problems; use crate::platform::models::{ - LocationHealthObservation, MirroredTotal, PersonPlatformEventPage, PlatformProblems, + LocationHealthObservation, PersonPlatformEventPage, PlatformProblems, }; use crate::runs::main::listings::list_person_run_nodes::list_person_run_nodes; use crate::runs::models::{ @@ -656,28 +655,6 @@ pub(crate) async fn health( )) } -#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/mirrored-totals", operation_id = "readPersonMirroredTotals", - params(("location_id" = Uuid, Path), ("target" = String, Path)), - responses((status = 200, body = Vec, description = "Records the server mirrors of each state table of this target, in sync order"), - (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] -pub(crate) async fn mirrored_totals( - (State(store), Extension(context)): (State, Extension), - Extension(default_role): Extension, - Path((location_id, target)): Path<(Uuid, String)>, - Extension(AuthenticatedPerson(principal, _)): Extension, -) -> Result>, ApiError> { - Ok(Json( - read_person_mirrored_totals( - &store, - (&context, principal), - (location_id, &target), - default_role, - ) - .await?, - )) -} - #[utoipa::path(get, path = "/api/control/platform/problems", operation_id = "readPersonPlatformProblems", responses((status = 200, body = PlatformProblems, description = "Live scheduler state, and broken locations and late ticks of targets the person lists"), (status = 401, description = "Session rejected")), security(("session" = []), ("cliSession" = [])))] diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index f50f64741d..73ead0ebc8 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -47,10 +47,6 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, Extension), - Extension(identity): Extension, - Path((location_id, _target)): Path<(Uuid, String)>, -) -> Result, ApiError> { - require_location(&identity, location_id)?; - Ok(Json(WarehouseRetention { - state_history_versions: facts.state_history.kept_versions(&store), - })) -} - #[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/sync-due", params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = WarehouseSyncDueRead, responses((status = 200, body = WarehouseSyncDue), (status = 400, body = crate::api::models::ApiErrorBody), @@ -129,14 +112,15 @@ pub(crate) async fn retention( (status = 409, description = "Reader generation fenced")), security(("bearer" = [])))] pub(crate) async fn sync_due( - State(store): State, + (State(store), Extension(facts)): (State, Extension), Extension(identity): Extension, Path((location_id, target)): Path<(Uuid, String)>, Json(request): Json, ) -> Result, ApiError> { require_location(&identity, location_id)?; + let versions: u16 = facts.state_history.kept_versions(&store); Ok(Json( - read_sync_due(&store, (location_id, &target), &request).await?, + read_sync_due(&store, (location_id, &target), (&request, versions)).await?, )) } diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 8e3ab58dcc..31c54c5e53 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -51,7 +51,6 @@ use crate::api::_helpers::resources::{ person_reads::sensor_state, person_reads::snapshots, person_reads::health, - person_reads::mirrored_totals, person_reads::problems, person_reads::platform_events, person_operations::catalog, @@ -112,7 +111,6 @@ use crate::api::_helpers::resources::{ warehouse_metadata::preview_copy, changed_builds::report, warehouse_metadata::statistics_due, - warehouse_metadata::retention, warehouse_metadata::sync_due, warehouse_metadata::watermark ))] diff --git a/crates/sqlbuild-orchestration/src/node_states/models.rs b/crates/sqlbuild-orchestration/src/node_states/models.rs index 0ee77410a5..aab7d14fd7 100644 --- a/crates/sqlbuild-orchestration/src/node_states/models.rs +++ b/crates/sqlbuild-orchestration/src/node_states/models.rs @@ -68,7 +68,7 @@ pub struct ChangedBuildReport { pub generation: i64, pub changed: Vec, pub downstream: Vec, - /// Why the plan failed, sent with empty lists; only newer servers accept it. + /// Why the plan failed, sent with empty lists. #[serde(default, skip_serializing_if = "Option::is_none")] pub failing: Option, } diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location/work_client.rs index 41271cbc42..d4f39af723 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/location/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location/work_client.rs @@ -12,8 +12,7 @@ use crate::platform::models::{ }; use crate::warehouse_metadata::models::{ MetadataBatch, PreviewCopy, PreviewCopyRead, StatisticsDueRead, StatisticsRead, - WarehouseRetention, WarehouseSyncDue, WarehouseSyncDueRead, WarehouseSyncResponse, - WarehouseWatermarkRead, + WarehouseSyncDue, WarehouseSyncDueRead, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -54,33 +53,6 @@ impl LocationWorkClient { .await } - /// The server's state-history limit; `None` from a server too old to have one. - pub async fn metadata_retention( - &self, - target: &str, - ) -> Result, ClientError> { - let location_id = self.location_id.to_string(); - let endpoint = self.connection.endpoint(&[ - "api", - "locations", - &location_id, - "targets", - target, - "metadata", - "retention", - ])?; - match client_response(self.connection.client.get(endpoint)).await { - Ok(retention) => Ok(Some(retention)), - Err(ClientError::Rejected(status)) - if status == reqwest::StatusCode::NOT_FOUND - || status == reqwest::StatusCode::METHOD_NOT_ALLOWED => - { - Ok(None) - } - Err(error) => Err(error), - } - } - pub async fn metadata_sync_due( &self, target: &str, diff --git a/crates/sqlbuild-orchestration/src/platform/main/person_reads/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/person_reads/mod.rs index 9f7dfb6b1e..beaa841041 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/person_reads/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/person_reads/mod.rs @@ -1,5 +1,4 @@ pub mod list_person_platform_events; -pub mod read_person_mirrored_totals; pub mod read_person_plan; pub mod read_person_plan_diff; pub mod read_person_plan_rows; diff --git a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_mirrored_totals.rs b/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_mirrored_totals.rs deleted file mode 100644 index f870f486b5..0000000000 --- a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_mirrored_totals.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! People read how many records the server mirrors of each state table, with the same view as the target's health. - -use std::collections::BTreeMap; - -use serde_json::Value; -use uuid::Uuid; - -use crate::authorization::main::decisions::authorize_listing::authorize_listing; -use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::platform::models::MirroredTotal; -use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, query::query}; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; -use crate::warehouse_metadata::models::WarehouseHistoryKind; - -/// Every mirrored state table of the target, in sync order, with the records held for it across its state locations. -pub async fn read_person_mirrored_totals( - store: &ControlStore, - (context, credentials): (&AuditContext, Uuid), - (location_id, target): (Uuid, &str), - default_role: DefaultRole, -) -> Result, ControlError> { - let (transaction, _) = authorize_listing( - begin(store, TransactionMode::Write).await?, - (context, credentials), - (location_id, target), - default_role, - ) - .await?; - let counts = WarehouseHistoryKind::ALL - .iter() - .map(|kind| { - format!( - "SELECT '{}' AS kind, COUNT(*) AS records FROM {} WHERE location_id=$1 AND target=$2", - kind.as_str(), - kind.mirror_table() - ) - }) - .collect::>() - .join(" UNION ALL "); - let (transaction, rows) = query( - transaction, - &counts, - vec![Parameter::Id(location_id), Parameter::Text(target.into())], - ) - .await?; - commit(transaction).await?; - Ok(WarehouseHistoryKind::ALL - .iter() - .map(|kind| MirroredTotal { - table: *kind, - records: recorded(&rows, *kind), - }) - .collect()) -} - -/// The records mirrored of `kind`, or none when the count names no row of it. -fn recorded(rows: &[BTreeMap], kind: WarehouseHistoryKind) -> u64 { - rows.iter() - .find(|row| row["kind"].as_str() == Some(kind.as_str())) - .and_then(|row| row["records"].as_u64()) - .unwrap_or_default() -} diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 30b9addc6f..5e50a95ed3 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -85,6 +85,9 @@ pub enum LocationHealth { /// On the report that completes or stops a sync, when the next safety sync is due unless a run ends first; absent otherwise, where no timed sync follows, and from older locations. #[serde(default, skip_serializing_if = "Option::is_none")] next_sync_at: Option>, + /// On the report of a table's last delivered page, the records the server then mirrored of that table across the target's state locations. + #[serde(default, skip_serializing_if = "Option::is_none")] + mirrored_records: Option, }, } @@ -610,21 +613,12 @@ pub struct WarehouseSyncProgress { #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum WarehouseReadFailure { - /// The metadata reader process did not start. - ReaderStart {}, - /// The reader could not connect to the warehouse, or the read raised an error. + /// The reader did not start or connect to the warehouse, or the read raised an error. ReadError {}, /// The reader did not answer within this many seconds. TimedOut { seconds: u64 }, } -/// How many records the server holds for one mirrored state table of a target. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -pub struct MirroredTotal { - pub table: WarehouseHistoryKind, - pub records: u64, -} - /// Configured rules that failed a location's load: how many findings, and the first of them. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs index 0ef8ba206d..8715161921 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs @@ -114,7 +114,7 @@ pub(crate) async fn ingest_events( commit_ingestion( transaction, (table, location_id, target), - (batch.scan.as_ref(), &keys), + (batch.scan.as_ref(), &keys, batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs index 55a7aa1b29..505bd69f60 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs @@ -80,11 +80,11 @@ pub(crate) async fn next_ingestion_chunk( .await } -/// Delivered rows take the scan's id, and catalog rows the time they were received; a complete scan deletes every row it did not deliver. +/// Delivered rows take the scan's id and catalog rows their receipt time; a complete scan deletes the rest, and a last page counts `table`'s mirror. pub(crate) async fn commit_ingestion( mut transaction: ControlTransaction, (table, location_id, target): (&str, Uuid, &str), - (scan, keys): (Option<&MetadataScan>, &[Uuid]), + (scan, keys, last_page): (Option<&MetadataScan>, &[Uuid], bool), mut applied: bool, ) -> Result { if let Some(scan) = scan { @@ -135,13 +135,27 @@ pub(crate) async fn commit_ingestion( applied |= !removed.is_empty(); } } + let mut mirrored_records: Option = None; + if last_page { + let counted; + (transaction, counted) = query( + transaction, + &format!("SELECT COUNT(*) AS records FROM {table} WHERE location_id=$1 AND target=$2"), + vec![Parameter::Id(location_id), Parameter::Text(target.into())], + ) + .await?; + mirrored_records = counted[0]["records"].as_u64(); + } if applied { transaction .changes .insert(Change::NodeStatus(location_id, target.into())); } commit(transaction).await?; - Ok(WarehouseSyncResponse { applied }) + Ok(WarehouseSyncResponse { + applied, + mirrored_records, + }) } /// Every record of a scan page must come from the scanned physical namespace. diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs index 90a1876e53..ac04c00b76 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs @@ -62,7 +62,7 @@ pub async fn ingest_catalog( commit_ingestion( transaction, (CATALOG_MIRROR, scope.0, scope.1), - (batch.scan.as_ref(), &keys), + (batch.scan.as_ref(), &keys, batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs index 22483f3b53..424b2581e3 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs @@ -43,7 +43,7 @@ pub async fn ingest_fingerprints( commit_ingestion( transaction, ("sqb_warehouse_fingerprints", location_id, target), - (batch.scan.as_ref(), &keys), + (batch.scan.as_ref(), &keys, batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs index e60a131609..745eb341c1 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs @@ -78,7 +78,7 @@ pub async fn ingest_source_freshness( commit_ingestion( transaction, ("sqb_warehouse_freshness", location_id, target), - (batch.scan.as_ref(), &keys), + (batch.scan.as_ref(), &keys, batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs index bf0046c67e..857d0fa150 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs @@ -70,7 +70,7 @@ pub async fn ingest_statistics( commit_ingestion( transaction, ("sqb_warehouse_statistics", scope.0, scope.1), - (None, &[]), + (None, &[], batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs index 32990874fb..1869c91ef6 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs @@ -13,7 +13,7 @@ pub fn preview_state_parts(copy: PreviewCopy, has_consumption: bool) -> Warehous WarehouseStateParts { provenance: SyncedProvenance { synced_at: copy.synced_at, - version: copy.version, + version: Some(copy.version), }, relations: copy.relations.into_iter().map(relation).collect(), fingerprints: copy diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs index b0ac7457cb..7483857d0b 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs @@ -19,13 +19,8 @@ pub async fn read_preview_copy( ) -> Result { let transaction = begin_ingestion(store, scope, (request.session_id, request.generation)).await?; - let (transaction, version) = if request.versioned { - let (transaction, version) = preview_version(transaction, scope).await?; - (transaction, Some(version)) - } else { - (transaction, None) - }; - if version.is_some() && version == request.known_version { + let (transaction, version) = preview_version(transaction, scope).await?; + if request.known_version.as_ref() == Some(&version) { commit(transaction).await?; return Ok(PreviewCopy { synced_at: None, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs index 573db4f6a5..3d9f9e3c62 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs @@ -11,7 +11,7 @@ use crate::warehouse_metadata::models::{WarehouseSyncDue, WarehouseSyncDueRead}; pub async fn read_sync_due( store: &ControlStore, scope: (Uuid, &str), - request: &WarehouseSyncDueRead, + (request, state_history_versions): (&WarehouseSyncDueRead, u16), ) -> Result { let transaction = begin_ingestion(store, scope, (request.session_id, request.generation)).await?; @@ -25,5 +25,9 @@ pub async fn read_sync_due( let runs_ended_at = serde_json::from_value(rows[0]["ended_at"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; commit(transaction).await?; - Ok(WarehouseSyncDue { runs_ended_at, now }) + Ok(WarehouseSyncDue { + runs_ended_at, + now, + state_history_versions, + }) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index fc24cb88a2..f5691eadb0 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -475,6 +475,8 @@ pub struct WarehouseSyncDueRead { pub struct WarehouseSyncDue { pub runs_ended_at: Option>, pub now: DateTime, + /// The newest versions per node or check a full read of state history needs to deliver. + pub state_history_versions: u16, } /// A location asks for the mirrored copy a plan preview plans from, as its fenced reader. @@ -483,12 +485,9 @@ pub struct WarehouseSyncDue { pub struct PreviewCopyRead { pub session_id: Uuid, pub generation: i64, - /// Whether the copy carries consumption history; sent only when set, as older servers reject it. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] + /// Whether the copy carries consumption history. + #[serde(default)] pub consumption: bool, - /// Whether the location holds copies by mirror version; sent only when set, as older servers reject it. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub versioned: bool, /// The mirror version of the copy the location holds; an unchanged mirror returns no records. #[serde(default, skip_serializing_if = "Option::is_none")] pub known_version: Option, @@ -526,9 +525,8 @@ pub struct PreviewCopy { /// Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub consumption: Vec, - /// The mirror version these records were read at; sent only to a versioned read. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub version: Option, + /// The mirror version these records were read at. + pub version: String, /// Set when the mirror is still at the read's known version: the records are left out. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub unchanged: bool, @@ -579,6 +577,9 @@ pub struct WarehouseMirrorCursor { #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] pub struct WarehouseSyncResponse { pub applied: bool, + /// Records the server mirrors of this table for the target, counted after a read's last page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mirrored_records: Option, } /// One bounded page of one state table from the registered reader of a published generation. @@ -592,6 +593,9 @@ pub struct MetadataBatch { pub records: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub scan: Option, + /// The table's read ends with this page, so the reply counts the records mirrored of it. + #[serde(default)] + pub last_page: bool, } /// A full scan's complete page removes the namespace's rows that the scan did not deliver. @@ -978,13 +982,6 @@ impl StateHistoryRetention { } } -/// The newest versions per node or check a full read of state history needs to deliver. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -#[serde(deny_unknown_fields)] -pub struct WarehouseRetention { - pub state_history_versions: u16, -} - /// A current snapshot node's declared metadata and latest build, without SQL, source or config. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct CatalogNodeRecord { diff --git a/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs b/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs index efb9785c55..8c1fac69a7 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs @@ -88,6 +88,26 @@ fn given_moves_views_and_cleanups_when_people_read_maintenance_then_each_model_s ) .await .expect("another target's view"); + let mut last_page = batch( + session_id, + "old_name_views", + records("old_name_history.json", &data["old_name_views"]), + ); + last_page.last_page = true; + let replies = [ + ingest_old_name_views(store, scope, &batch(session_id, "old_name_views", vec![])) + .await + .expect("an earlier page"), + ingest_old_name_views(store, scope, &last_page) + .await + .expect("the read's last page"), + ]; + assert_eq!( + replies.map(|reply| reply.mirrored_records), + [None, Some(4)], + "{}: only a read's last page counts the mirror, and only this target's records", + test_case.description + ); let (api, client) = start_password_api(store).await; let viewer = login(&client, &api.url, "orders-viewer").await; @@ -181,46 +201,6 @@ fn given_moves_views_and_cleanups_when_people_read_maintenance_then_each_model_s .0, StatusCode::FORBIDDEN ); - let totals_path = - format!("/api/control/locations/{location}/targets/development/mirrored-totals"); - let (status, totals) = get(&client, (&api.url, &viewer), &totals_path).await; - assert_eq!(status, StatusCode::OK, "{}", test_case.description); - let counted: Vec<(&str, u64)> = totals - .as_array() - .expect("totals") - .iter() - .filter(|total| total["records"] != 0) - .map(|total| { - ( - total["table"].as_str().expect("table"), - total["records"].as_u64().expect("records"), - ) - }) - .collect(); - assert_eq!( - (totals.as_array().expect("totals").len(), counted), - ( - 14, - vec![ - ("janitor_events", 2), - ("migrations", 3), - ("old_name_views", 4) - ] - ), - "{}: every state table, counting only this target's mirrored records", - test_case.description - ); - assert_eq!( - get( - &client, - (&api.url, &viewer), - &totals_path.replace("development", "preview") - ) - .await - .0, - StatusCode::FORBIDDEN, - "mirrored totals need the same view as the target's health" - ); stop_api(api).await; close_test_store(fixture_store).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index eeb54e9281..c92ce5e9f2 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -554,7 +554,7 @@ pub(crate) async fn sync_due_scope( let fields = reply.as_object().expect("object"); let mut keys: Vec<_> = fields.keys().collect(); keys.sort(); - assert_eq!(keys, ["now", "runs_ended_at"]); + assert_eq!(keys, ["now", "runs_ended_at", "state_history_versions"]); let ended = fields["runs_ended_at"] .as_str() .map(|time| time.parse::>().expect("time")); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs index e936923cc5..81dedef654 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs @@ -159,7 +159,6 @@ pub(crate) async fn preview_copy_scope( session_id: preview_session, generation: 1, consumption: false, - versioned: false, known_version: None, }; let copy = || async { @@ -193,25 +192,25 @@ pub(crate) async fn preview_copy_scope( ) }; let unsynced = copy().await; - let older_location: Value = client + let plain: Value = client .post(format!("{metadata}/preview-copy")) .bearer_auth(bearer) .json(&json!({"session_id": preview_session, "generation": 1})) .send() .await - .expect("copy for an older location") + .expect("copy without consumption or a held version") .json() .await .expect("copy body"); assert!( unsynced.consumption.is_empty() - && ["consumption", "version", "unchanged"] + && ["consumption", "unchanged"] .iter() - .all(|field| older_location.get(field).is_none()), - "a read that asks for neither consumption nor versions never carries the fields, which older locations reject" + .all(|field| plain.get(field).is_none()) + && plain["version"] == json!(unsynced.version), + "a read without consumption or a held version carries neither, and always names its version" ); let versioned = |known_version: Option| PreviewCopyRead { - versioned: true, known_version, ..read.clone() }; @@ -220,12 +219,12 @@ pub(crate) async fn preview_copy_scope( .await .expect("versioned copy"); let held = outbound - .preview_copy("preview", &versioned(current.version.clone())) + .preview_copy("preview", &versioned(Some(current.version.clone()))) .await .expect("copy at the held version"); assert_eq!( ( - current.version.is_some() && !current.unchanged, + !current.version.is_empty() && !current.unchanged, ours(¤t) == ours(&unsynced), held.unchanged, held.version == current.version, @@ -248,7 +247,7 @@ pub(crate) async fn preview_copy_scope( ) .await; let changed = outbound - .preview_copy("preview", &versioned(current.version.clone())) + .preview_copy("preview", &versioned(Some(current.version.clone()))) .await .expect("copy after a mirrored write"); assert_eq!( @@ -264,7 +263,7 @@ pub(crate) async fn preview_copy_scope( ), "a mirrored write moves the version, so the held copy is read again" ); - let before_sync: Option = changed.version.clone(); + let before_sync: String = changed.version.clone(); let with_consumption = outbound .preview_copy( "preview", @@ -327,7 +326,7 @@ pub(crate) async fn preview_copy_scope( ); assert_ne!( outbound - .preview_copy("preview", &versioned(before_sync.clone())) + .preview_copy("preview", &versioned(Some(before_sync.clone()))) .await .expect("copy after a clean sync") .version, @@ -394,7 +393,6 @@ pub(crate) async fn preview_copy_scope( session_id, generation: 1, consumption: false, - versioned: false, known_version: None, }, ) diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index 704ff052d5..1708192f26 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -135,6 +135,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ generation: 1, records: records[..1].iter().map(|record| json!(record)).collect(), scan: None, + last_page: false, } ) .await @@ -159,6 +160,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ generation: 1, records: page.iter().map(|record| json!(record)).collect(), scan: None, + last_page: false, }; assert!( ingest_source_freshness(store, (location, "development"), &batch) @@ -252,6 +254,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ generation: 1, records: vec![json!(late)], scan: None, + last_page: false, }; ingest_source_freshness(store, (location, "development"), &batch) .await @@ -289,6 +292,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ generation: 1, records: page.iter().map(|record| json!(record)).collect(), scan: None, + last_page: false, }, ) .await @@ -422,6 +426,7 @@ fn given_a_source_version_that_returns_after_a_complete_scan_prunes_history_when generation: 1, records: vec![json!(record)], scan, + last_page: false, }, ) .await @@ -591,6 +596,7 @@ fn given_timestamp_observations_when_age_crosses_the_boundary_then_scoped_pages_ generation: 1, records: records.iter().map(|record| json!(record)).collect(), scan: None, + last_page: false, }; ingest_source_freshness(store, (location, "development"), &batch) .await diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs index ae8134f59d..43486986c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs @@ -48,12 +48,12 @@ fn requested_state( let text: String = std::fs::read_to_string(path)?; let copy: PreviewCopy = serde_json::from_str(&text) .map_err(|error| PyValueError::new_err(format!("invalid synced copy: {error}")))?; - let version: Option = copy.version.clone(); + let version: String = copy.version.clone(); let state: Arc = Arc::new(warehouse_state(preview_state_parts( copy, request["consumption"].as_bool().unwrap_or(false), ))); - *held = version.map(|version| HeldState { + *held = Some(HeldState { version, state: Arc::clone(&state), }); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 67054f7e52..3c958c3dc7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -513,7 +513,7 @@ The location starts the sync within a few seconds, and the top bar shows when it | --- | --- | --- | --- | | `[control] server` | `SQLBUILD_CONTROL_SERVER` | none | HTTPS origin of the server, as `sqb login --server` takes it | | `[control] location` | `SQLBUILD_CONTROL_LOCATION` | none | Location name a signed-in `sqb build` asks to sync | -| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"`, or off on Databricks, ClickHouse and MotherDuck | Longest a location waits between syncs; `"0"` or `"off"` turns it off. See [warehouse cost](#warehouse-cost-of-an-always-on-location) | +| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"` | Longest a location waits between syncs; `"0"` or `"off"` turns it off. Databricks, ClickHouse and MotherDuck never sync on the interval. See [warehouse cost](#warehouse-cost-of-an-always-on-location) | | none | `SQLBUILD_CONTROL_CA_CERT` | none | Extra trusted PEM CA certificate for the server | An environment variable overrides the project file. `sqb location serve --sync-at-least-every` @@ -538,8 +538,8 @@ Details: or time retries a smaller page. Any other error, such as a missing permission or table, retries after 30 seconds, then waits twice as long after each failure in a row. After 5 failures in a row the sync stops until a run ends, you request one, the location publishes again, or the next safety - sync is due (`sync_at_least_every`). It then starts again from the first kind of history. A sync - resumed by a run or the safety sync tries once, and another failure stops it again. + sync is due (`sync_at_least_every`). It then starts again from the first kind of history, and + failed reads back off again before they stop it. - `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. - When a state table holds two rows with the same ID, such as one check result written twice, the server keeps the row with the earlier time. Estimated query costs are the exception: when two @@ -565,7 +565,7 @@ A location syncs when it starts, after each run ends, when someone asks, and on Where the interval is off, an idle location syncs only after a run ends or when someone asks, with **Sync now**, `sqb control sync` or a signed-in `sqb build`. A run has just used the warehouse, so -the sync after it rarely resumes anything. Set `sync_at_least_every` to turn the interval back on. +the sync after it rarely resumes anything. Setting `sync_at_least_every` doesn't change this. ### Follow a sync diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 38ea88ac47..9b700e1400 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -752,13 +752,9 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, const locations = (await bootstrapLocations()).filter((l) => !healthRefused.has(l.id)); const read = await Promise.all( locations.map(async (location) => { - const { locationId, target } = parseLocationKey(location.id); - const [[{ data, response }, snapshot], totals] = await Promise.all([ - locationHealth(location.id), - http.GET('/api/control/locations/{location_id}/targets/{target}/mirrored-totals', { params: { path: { location_id: locationId, target } } }) - ]); + const [{ data, response }, snapshot] = await locationHealth(location.id); if (response.status === 403) healthRefused.add(location.id); - return data ? [toWarehouseConnection(location, data, snapshot, totals.data ?? null)] : []; + return data ? [toWarehouseConnection(location, data, snapshot)] : []; }) ); return read.flat(); diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 2ead32cb87..9d0d471f1e 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -128,21 +128,27 @@ describe('warehouseSyncOf', () => { ]); }); - it('counts the records mirrored of each table, and names the next sync from the report that completed one', () => { + it('counts the records mirrored of each table from its newest counted read, and names the next sync from the report that completed one', () => { const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; - const completed = sync('2026-11-08T12:03:00Z', 'statistics', null); - const observations = [{ ...completed, report: { ...completed.report, health: { ...completed.report.health, next_sync_at: '2026-11-08T12:33:00Z' } } }, sync('2026-11-08T12:02:00Z', 'catalog', null)]; - const connection = toWarehouseConnection(location, observations, null, [ - { table: 'catalog', records: 120 }, - { table: 'statistics', records: 4_800 } - ]); + const counted = (at: string, table: 'catalog' | 'statistics', records: number) => { + const report = sync(at, table, null); + return { ...report, report: { ...report.report, health: { ...report.report.health, mirrored_records: records } } }; + }; + const completed = counted('2026-11-08T12:03:00Z', 'statistics', 4_800); + const observations = [ + { ...completed, report: { ...completed.report, health: { ...completed.report.health, next_sync_at: '2026-11-08T12:33:00Z' } } }, + sync('2026-11-08T12:02:30Z', 'catalog', null), + counted('2026-11-08T12:02:00Z', 'catalog', 120), + counted('2026-11-08T12:01:00Z', 'catalog', 90) + ]; + const connection = toWarehouseConnection(location, observations); expect(connection.tables.map((t) => [t.table, t.mirroredTotal])).toEqual([ ['catalog', 120], ['statistics', 4_800] ]); expect(connection.nextSyncAt).toBe('2026-11-08T12:33:00Z'); - const running = toWarehouseConnection(location, [sync('2026-11-08T12:40:00Z', 'catalog', null), ...observations]); - expect([running.nextSyncAt, running.tables[0].mirroredTotal]).toEqual([null, null]); + const uncounted = toWarehouseConnection(location, [sync('2026-11-08T12:40:00Z', 'catalog', null)]); + expect([uncounted.nextSyncAt, uncounted.tables[0].mirroredTotal]).toEqual([null, null]); }); it('marks a sync that failed reads stopped, from the newest report only', () => { @@ -197,7 +203,7 @@ describe('warehouseSyncOf', () => { expect(warehouseSyncOf([{ ...timedOut, report: { ...timedOut.report, health } }])?.lastError).toBe('statistics: failed while reading the warehouse: no reply within 130 s, on a page of 100 relations'); const erroring = sync('2026-11-08T12:04:00Z', 'catalog', 'read'); const error = { ...erroring.report.health, read_failure: { kind: 'read_error' as const }, page_limit: 2000 }; - expect(warehouseSyncOf([{ ...erroring, report: { ...erroring.report, health: error } }])?.lastError).toBe('catalog: failed while reading the warehouse: the warehouse returned an error; the location’s log has it, on a page of 2,000 rows'); + expect(warehouseSyncOf([{ ...erroring, report: { ...erroring.report, health: error } }])?.lastError).toBe('catalog: failed while reading the warehouse: the read failed; the location’s log has why, on a page of 2,000 rows'); }); it('shows a running sync and the newest completed one, and a first sync as pending until it completes', () => { diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index 7e017abd87..96dc6e3be2 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -169,11 +169,9 @@ function readCauseOf(health: SyncHealth): string { const reason = cause.kind === 'timed_out' ? `no reply within ${cause.seconds} s` - : cause.kind === 'reader_start' - ? 'the metadata reader didn’t start' - : health.error - ? 'the warehouse returned an error' - : 'the warehouse returned an error; the location’s log has it'; + : health.error + ? 'the read failed' + : 'the read failed; the location’s log has why'; const page = health.page_limit ? `, on a page of ${health.page_limit.toLocaleString('en-US')} ${health.table === 'statistics' ? 'relations' : 'rows'}` : ''; return `: ${reason}${page}`; } @@ -199,16 +197,15 @@ export function connectionDetailsOf(published: Wire['PublishedSnapshot'] | null) } /** - * A location target's connection from its health reports, newest snapshot and mirrored totals: the - * settings it published, the newest connection check, where its state tables are read from, each - * table's newest sync and how many records the server mirrors of it, and when the next sync is due. - * `totals` is null when the server doesn't report them. + * A location target's connection from its health reports and newest snapshot: the settings it + * published, the newest connection check, where its state tables are read from, each table's newest + * sync and how many records the server mirrored of it as its last read finished, and when the next + * sync is due. */ export function toWarehouseConnection( location: Location, observations: Wire['LocationHealthObservation'][], - published: Wire['PublishedSnapshot'] | null = null, - totals: Wire['MirroredTotal'][] | null = null + published: Wire['PublishedSnapshot'] | null = null ): WarehouseConnection { const { syncs, newest } = syncsOf(observations); const tables: WarehouseTableSync[] = [...newest.values()] @@ -221,7 +218,7 @@ export function toWarehouseConnection( error: health.error ?? null, waiting: health.failure === 'busy', lastSyncedAt: syncs.find((s) => s.report.health.table === health.table && !s.report.health.failure)?.received_at ?? null, - mirroredTotal: totals ? (totals.find((t) => t.table === health.table)?.records ?? 0) : null + mirroredTotal: syncs.find((s) => s.report.health.table === health.table && s.report.health.mirrored_records != null)?.report.health.mirrored_records ?? null })) .sort((a, b) => a.table.localeCompare(b.table)); const state = syncs[0]?.report.health.location; diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 845cd2ec22..cd0ee84172 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -493,7 +493,7 @@ The location starts the sync within a few seconds, and the top bar shows when it | --- | --- | --- | --- | | `[control] server` | `SQLBUILD_CONTROL_SERVER` | none | HTTPS origin of the server, as `sqb login --server` takes it | | `[control] location` | `SQLBUILD_CONTROL_LOCATION` | none | Location name a signed-in `sqb build` asks to sync | -| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"`, or off on Databricks, ClickHouse and MotherDuck | Longest a location waits between syncs; `"0"` or `"off"` turns it off. See [warehouse cost](#warehouse-cost-of-an-always-on-location) | +| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"` | Longest a location waits between syncs; `"0"` or `"off"` turns it off. Databricks, ClickHouse and MotherDuck never sync on the interval. See [warehouse cost](#warehouse-cost-of-an-always-on-location) | | none | `SQLBUILD_CONTROL_CA_CERT` | none | Extra trusted PEM CA certificate for the server | An environment variable overrides the project file. `sqb location serve --sync-at-least-every` @@ -518,8 +518,8 @@ Details: or time retries a smaller page. Any other error, such as a missing permission or table, retries after 30 seconds, then waits twice as long after each failure in a row. After 5 failures in a row the sync stops until a run ends, you request one, the location publishes again, or the next safety - sync is due (`sync_at_least_every`). It then starts again from the first kind of history. A sync - resumed by a run or the safety sync tries once, and another failure stops it again. + sync is due (`sync_at_least_every`). It then starts again from the first kind of history, and + failed reads back off again before they stop it. - `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. - When a state table holds two rows with the same ID, such as one check result written twice, the server keeps the row with the earlier time. Estimated query costs are the exception: when two @@ -545,7 +545,7 @@ A location syncs when it starts, after each run ends, when someone asks, and on Where the interval is off, an idle location syncs only after a run ends or when someone asks, with **Sync now**, `sqb control sync` or a signed-in `sqb build`. A run has just used the warehouse, so -the sync after it rarely resumes anything. Set `sync_at_least_every` to turn the interval back on. +the sync after it rarely resumes anything. Setting `sync_at_least_every` doesn't change this. ### Follow a sync