From a69df6c5d2f3b01331087e79880856b02bd7dd85 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:18:13 +0000 Subject: [PATCH 1/4] refactor(location): drop old-server fallbacks and duplicate sync state - Sync-due replies carry the server's state-history limit; the retention endpoint and its client call are gone. - Preview copies are always versioned: no `versioned` flag, no unversioned retry, and `version` is always present. - A stopped sync resumed by a run, the timed sync or a request backs off again from a fresh failure count. - A busy warehouse file is read from the failure itself, the page timeout is a constant, and a reader that did not start reports a read error. - The timed sync never wakes billed compute, whether or not the interval was set. - Mirrored record counts come from the sync: the server counts a table after its read's last page and the location reports it, replacing the per-page-load count endpoint. --- clients/typescript/src/schema.d.ts | 148 +++--------------- .../_helpers/location/execution.rs | 23 +-- .../_helpers/location/held_copy.rs | 33 +--- .../_helpers/location/sync_outcomes.rs | 8 +- .../_helpers/location/sync_report.rs | 18 +-- .../main/_location_metadata.rs | 11 +- .../native_commands/main/_location_plan.rs | 24 +-- .../src/native_commands/models.rs | 21 +-- .../src/native_commands/tests/helpers.rs | 3 +- .../tests/test_location_metadata.rs | 64 +++----- .../tests/test_sync_schedule.rs | 20 +-- .../src/native_commands/tests/test_types.rs | 4 +- .../authentication/password_sessions.rs | 4 - .../api/_helpers/resources/person_reads.rs | 25 +-- .../src/api/_helpers/resources/router.rs | 4 - .../_helpers/resources/warehouse_metadata.rs | 26 +-- .../sqlbuild-orchestration/src/api/models.rs | 2 - .../src/node_states/models.rs | 2 +- .../platform/_helpers/location/work_client.rs | 30 +--- .../src/platform/main/person_reads/mod.rs | 1 - .../read_person_mirrored_totals.rs | 63 -------- .../src/platform/models.rs | 14 +- .../src/warehouse_metadata/_helpers/events.rs | 2 +- .../warehouse_metadata/_helpers/ingestion.rs | 19 ++- .../main/ingest/ingest_catalog.rs | 2 +- .../main/ingest/ingest_fingerprints.rs | 2 +- .../main/ingest/ingest_source_freshness.rs | 2 +- .../main/ingest/ingest_statistics.rs | 2 +- .../location_reads/preview_state_parts.rs | 2 +- .../main/location_reads/read_preview_copy.rs | 9 +- .../main/location_reads/read_sync_due.rs | 8 +- .../src/warehouse_metadata/models.rs | 27 ++-- .../tests/api/test_maintenance.rs | 60 +++---- .../tests/api/warehouse_metadata/helpers.rs | 2 +- .../preview_copy/helpers.rs | 24 ++- .../tests/sensors/test_sources.rs | 6 + .../_helpers/location_command/runtime_plan.rs | 4 +- ui/src/lib/api/http-client.ts | 8 +- ui/src/lib/api/http-locations.test.ts | 26 +-- ui/src/lib/api/http-locations.ts | 21 ++- website/src/content/docs/docs/cli/server.mdx | 8 +- 41 files changed, 217 insertions(+), 565 deletions(-) delete mode 100644 crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_mirrored_totals.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index e2282f59ae..e5df5d26fe 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -530,22 +530,6 @@ export interface paths { patch?: never; trace?: never; }; - "/api/control/locations/{location_id}/targets/{target}/mirrored-totals": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get: operations["readPersonMirroredTotals"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/api/control/locations/{location_id}/targets/{target}/node-states": { parameters: { query?: never; @@ -1234,23 +1218,6 @@ export interface paths { patch?: never; trace?: never; }; - "/api/locations/{location_id}/targets/{target}/metadata/retention": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** How much state history a full read delivers; older servers answer 404 and the location reads it all. */ - get: operations["read_metadata_retention"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/api/locations/{location_id}/targets/{target}/metadata/statistics-due": { parameters: { query?: never; @@ -1765,7 +1732,7 @@ export interface components { ChangedBuildReport: { changed: string[]; downstream: string[]; - /** @description Why the plan failed, sent with empty lists; only newer servers accept it. */ + /** @description Why the plan failed, sent with empty lists. */ failing?: string | null; /** Format: int64 */ generation: number; @@ -2220,6 +2187,11 @@ export interface components { /** @enum {string} */ kind: "warehouse_sync"; location: components["schemas"]["WarehouseStateLocation"]; + /** + * Format: int64 + * @description On the report of a table's last delivered page, the records the server then mirrored of that table across the target's state locations. + */ + mirrored_records?: number | null; /** * Format: date-time * @description On the report that completes or stops a sync, when the next safety sync is due unless a run ends first; absent otherwise, where no timed sync follows, and from older locations. @@ -2421,6 +2393,8 @@ export interface components { /** Format: int64 */ generation: number; kind: components["schemas"]["WarehouseHistoryKind"]; + /** @description The table's read ends with this page, so the reply counts the records mirrored of it. */ + last_page?: boolean; records: Record[]; scan?: null | components["schemas"]["MetadataScan"]; /** Format: uuid */ @@ -2452,12 +2426,6 @@ export interface components { physical_generation_id: string; target_name: string; }; - /** @description How many records the server holds for one mirrored state table of a target. */ - MirroredTotal: { - /** Format: int64 */ - records: number; - table: components["schemas"]["WarehouseHistoryKind"]; - }; MutationResponse: { applied: boolean; }; @@ -2955,12 +2923,12 @@ export interface components { synced_at?: string | null; /** @description Set when the mirror is still at the read's known version: the records are left out. */ unchanged?: boolean; - /** @description The mirror version these records were read at; sent only to a versioned read. */ - version?: string | null; + /** @description The mirror version these records were read at. */ + version: string; }; /** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */ PreviewCopyRead: { - /** @description Whether the copy carries consumption history; sent only when set, as older servers reject it. */ + /** @description Whether the copy carries consumption history. */ consumption?: boolean; /** Format: int64 */ generation: number; @@ -2968,8 +2936,6 @@ export interface components { known_version?: string | null; /** Format: uuid */ session_id: string; - /** @description Whether the location holds copies by mirror version; sent only when set, as older servers reject it. */ - versioned?: boolean; }; /** @description The newest successful build of a node, which a `cursor_max` observation must come from. */ PreviewLatestRun: { @@ -4102,9 +4068,6 @@ export interface components { }; /** @description What went wrong reading one page, without the warehouse's message, which may name hosts or users. */ WarehouseReadFailure: { - /** @enum {string} */ - kind: "reader_start"; - } | { /** @enum {string} */ kind: "read_error"; } | { @@ -4150,11 +4113,6 @@ export interface components { warehouse_size: string; warehouse_type?: string | null; }; - /** @description The newest versions per node or check a full read of state history needs to deliver. */ - WarehouseRetention: { - /** Format: int32 */ - state_history_versions: number; - }; /** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */ WarehouseStateLocation: { database?: string | null; @@ -4180,6 +4138,11 @@ export interface components { now: string; /** Format: date-time */ runs_ended_at?: string | null; + /** + * Format: int32 + * @description The newest versions per node or check a full read of state history needs to deliver. + */ + state_history_versions: number; }; /** @description A registered reader asks whether a run ended since its last synchronization began. */ WarehouseSyncDueRead: { @@ -4222,6 +4185,11 @@ export interface components { }; WarehouseSyncResponse: { applied: boolean; + /** + * Format: int64 + * @description Records the server mirrors of this table for the target, counted after a read's last page. + */ + mirrored_records?: number | null; }; /** @description Derive progress from committed mirror rows, scoped to the active physical reader. */ WarehouseWatermarkRead: { @@ -5586,44 +5554,6 @@ export interface operations { }; }; }; - readPersonMirroredTotals: { - parameters: { - query?: never; - header?: never; - path: { - location_id: string; - target: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Records the server mirrors of each state table of this target, in sync order */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["MirroredTotal"][]; - }; - }; - /** @description Session rejected */ - 401: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["ApiErrorBody"]; - }; - }; - }; - }; readPersonNodeStates: { parameters: { query?: never; @@ -7599,42 +7529,6 @@ export interface operations { }; }; }; - read_metadata_retention: { - parameters: { - query?: never; - header?: never; - path: { - location_id: string; - target: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["WarehouseRetention"]; - }; - }; - /** @description Invalid credential */ - 401: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Location not authorized */ - 403: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - }; - }; statistics_due: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs index f41fb37c34..f495ee0637 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs @@ -34,6 +34,7 @@ use crate::native_commands::_helpers::command::terminal::{live_screen, stderr_co use crate::native_commands::_helpers::location::sync_report::{ metadata_health, read_timeout_seconds, rows_text, scan_line, }; +use crate::native_commands::constants::PLAN_NOT_PRODUCED; use crate::native_commands::constants::SCREEN_INDENT; use crate::native_commands::constants::{ ADMIN_COMMAND, CHANGED_PATHS_SHOWN, COLUMNS_VARIABLE, FALLBACK_COLUMNS, @@ -42,7 +43,6 @@ use crate::native_commands::constants::{ METADATA_SYNC_CHECK_SECONDS, SAFETY_SYNC_ENV, SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, }; -use crate::native_commands::constants::{METADATA_PAGE_SECONDS, PLAN_NOT_PRODUCED}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; use crate::native_commands::main::_run_control::run_control; @@ -502,7 +502,6 @@ pub(crate) fn metadata_sync( server_offset: chrono::Duration::zero(), safety_seconds, safety_at: Instant::now() + Duration::from_secs(safety_seconds), - busy: false, failed_reads: 0, halted: false, change_tokens: HashMap::new(), @@ -511,10 +510,8 @@ pub(crate) fn metadata_sync( mirror_holds_rows: false, reader: None, state_history_versions: None, - page_seconds: METADATA_PAGE_SECONDS, progress: SyncProgress::new(), reads_wake_compute: false, - safety_explicit: true, } } @@ -597,6 +594,7 @@ pub(crate) fn freeze_history_page( location: state.location().clone(), complete: page.next_after.is_none(), }), + last_page: page.next_after.is_none(), }; if serde_json::to_vec(&batch) .map_err(|error| ControlError::Invalid(error.to_string()))? @@ -648,10 +646,7 @@ pub(crate) async fn begin_after_run( }; let ran = state.sync_due(due.runs_ended_at); if ran { - state.state_history_versions = match client.metadata_retention(target).await { - Ok(retention) => retention.map(|retention| retention.state_history_versions), - Err(error) => return (state, Err(error)), - }; + state.state_history_versions = Some(due.state_history_versions); state.begin_sync(due.now); } (state, Ok(ran)) @@ -743,6 +738,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync state.progress.sync_rows += state.progress.scan_rows; state.progress.scan_rows = 0; state.progress.scan_unchanged = false; + state.progress.scan_mirrored = None; state.progress.scan_started = Instant::now(); state.index += 1; state.scan_id = Uuid::now_v7(); @@ -874,7 +870,6 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result>, ) -> Result< @@ -1375,7 +1371,6 @@ pub(crate) async fn read_metadata_page( )>, ControlError, > { - state.busy = false; if let Some(statistics) = &state.statistics && statistics.names.is_empty() { @@ -1400,7 +1395,7 @@ pub(crate) async fn read_metadata_page( } }, }; - let seconds = read_timeout_seconds(&state); + let seconds = read_timeout_seconds(&state, page_seconds); tokio::pin!(stopped); let read = tokio::select! { result = &mut stopped => { @@ -1417,7 +1412,6 @@ pub(crate) async fn read_metadata_page( Ok(page) } Ok((reader, Ok(MetadataReply::Busy))) => { - state.busy = true; state.reader = Some(reader); Err(ReadFailure::Busy) } @@ -1633,14 +1627,13 @@ pub(crate) async fn settle_publication( stopped: impl Future>, ) -> Result, ControlError> { let (mut metadata, deferred) = if connect { - let mut sync: LocationMetadataSync = metadata_sync( + let sync: LocationMetadataSync = metadata_sync( (locations, generation), ( options.metadata_reconcile_seconds, options.safety_sync_seconds, ), ); - sync.safety_explicit = options.safety_sync_explicit; (Some(sync), None) } else { (metadata, Some((locations, generation))) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/held_copy.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/held_copy.rs index 2bdca9d425..ee73c74a36 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/held_copy.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/held_copy.rs @@ -3,10 +3,8 @@ use std::path::PathBuf; use std::sync::{Mutex, PoisonError}; -use sqlbuild_orchestration::platform::models::LocationWorkClient; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; -use uuid::Uuid; +use sqlbuild_orchestration::warehouse_metadata::models::PreviewCopy; use crate::native_commands::models::{HeldCopy, LocationServe}; @@ -29,28 +27,6 @@ pub(crate) fn held_copy( }) } -/// A versioned read; a server without mirror versions rejects the fields, so it is read unversioned. -pub(crate) async fn read_copy( - client: &LocationWorkClient, - target: &str, - read: PreviewCopyRead, -) -> Result { - match client.preview_copy(target, &read).await { - Ok(copy) => Ok(copy), - Err(_) => client - .preview_copy( - target, - &PreviewCopyRead { - versioned: false, - known_version: None, - ..read - }, - ) - .await - .map_err(|error| error.to_string()), - } -} - /// The location's one held copy file for its target, which each newer copy overwrites. fn held_copy_path(options: &LocationServe) -> PathBuf { let target: String = options @@ -70,7 +46,7 @@ fn held_copy_path(options: &LocationServe) -> PathBuf { )) } -/// Atomically replace the held copy file; a copy without a version gets a fresh one. +/// Atomically replace the held copy file. pub(crate) fn hold_copy( held_copies: &Mutex>, options: &LocationServe, @@ -79,10 +55,7 @@ pub(crate) fn hold_copy( ) -> Result { let path: PathBuf = held_copy_path(options); let written: PathBuf = path.with_extension(format!("{}.tmp", std::process::id())); - let version: String = copy - .version - .clone() - .unwrap_or_else(|| Uuid::now_v7().to_string()); + let version: String = copy.version.clone(); std::fs::write( &written, serde_json::to_vec(©).map_err(std::io::Error::other)?, diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs index 98c1a4a737..e488f2b107 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs @@ -50,17 +50,18 @@ pub(crate) async fn failed_read( (started, failure): (Instant, ReadFailure), stopped: impl Future>, ) -> Result, ControlError> { - let stage = if state.busy { + let busy: bool = failure == ReadFailure::Busy; + let stage = if busy { WarehouseSyncFailure::Busy } else { WarehouseSyncFailure::Read }; - let halts: bool = !state.busy && state.halts(failure.page_size()); + let halts: bool = !busy && state.halts(failure.page_size()); tokio::select! { result = stopped => { result?; return Ok(None); }, () = report_metadata_health(&state, (client, poll), (started, Some(stage), Some((&failure, halts)))) => {}, } - if state.busy { + if busy { state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); return Ok(Some(state)); } @@ -135,6 +136,7 @@ pub(crate) async fn deliver_metadata_page( } }; state.applied |= response.applied; + state.progress.scan_mirrored = response.mirrored_records; tokio::select! { result = &mut stopped => { result?; return Ok(None); }, () = metadata_health(&state, (client, poll), (started, None)) => {}, diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_report.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_report.rs index 4332b36cc1..45cc7ab425 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_report.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_report.rs @@ -131,6 +131,7 @@ pub(crate) fn metadata_health_of( ..sync_progress(state, failure.is_none()) }), next_sync_at, + mirrored_records: state.progress.scan_mirrored, } } @@ -167,12 +168,12 @@ pub(crate) async fn metadata_health( report_metadata_health(state, (client, poll), (started, failure, None)).await; } -/// How long one page may take: statistics scale with their relations, other pages get a fixed allowance. -pub(crate) fn read_timeout_seconds(state: &LocationMetadataSync) -> u64 { +/// How long one page may take: statistics scale with their relations, other pages get `page_seconds`. +pub(crate) fn read_timeout_seconds(state: &LocationMetadataSync, page_seconds: u64) -> u64 { if state.kind() == WarehouseHistoryKind::Statistics { METADATA_POLL_SECONDS + STATISTICS_RELATION_SECONDS * state.statistics_limit as u64 } else { - state.page_seconds + page_seconds } } @@ -185,20 +186,15 @@ pub(crate) fn page_limit(state: &LocationMetadataSync) -> usize { } } -/// Say a stopped sync resumes; `one_attempt` when one more failure stops it again. -pub(crate) fn resumed_line(one_attempt: bool) { +/// Say a stopped sync resumes, with failed reads backing off again before they stop it. +pub(crate) fn resumed_line() { drop(write_output(&[( Stream::Stderr, "Retrying the stopped warehouse history synchronization.\n", )])); location_warning( "warehouse sync retrying", - if one_attempt { - "one attempt; another failed read stops it again" - } else { - "on request; failed reads back off again before they stop it" - } - .to_owned(), + "failed reads back off again before they stop it".to_owned(), ); } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index f3c4867748..bc154fbe7e 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -17,6 +17,7 @@ use crate::native_commands::_helpers::location::sync_outcomes::{ deliver_metadata_page, failed_read, retry_metadata, }; use crate::native_commands::_helpers::location::sync_report::{metadata_health, resumed_line}; +use crate::native_commands::constants::METADATA_PAGE_SECONDS; use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch}; /// One bounded source or mirror page leaves named work polling a turn between pages. @@ -43,7 +44,7 @@ pub(crate) async fn sync_location_metadata( }; state = next; match due { - Ok(true) if halted => resumed_line(state.failed_reads > 0), + Ok(true) if halted => resumed_line(), Ok(true) => {} Ok(false) => return Ok(Some(state)), Err(error) => return retry_metadata(state, error), @@ -70,8 +71,12 @@ pub(crate) async fn sync_location_metadata( return retry_metadata(state, error); } } - let Some((read, page)) = - read_metadata_page((options, script), state, stopped.as_mut()).await? + let Some((read, page)) = read_metadata_page( + (options, script, METADATA_PAGE_SECONDS), + state, + stopped.as_mut(), + ) + .await? else { return Ok(None); }; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs index d6cdcde0da..0ba6251feb 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -11,9 +11,7 @@ use tokio::sync::mpsc; use uuid::Uuid; use crate::native_commands::_helpers::location::execution::{plan_failure, without_diffs}; -use crate::native_commands::_helpers::location::held_copy::{ - HELD_COPY, held_copy, hold_copy, read_copy, -}; +use crate::native_commands::_helpers::location::held_copy::{HELD_COPY, held_copy, hold_copy}; use crate::native_commands::_helpers::location::plan_request::{ plan_arguments, run_plan, selected_names, }; @@ -38,19 +36,13 @@ pub(super) async fn plan_location( .is_some_and(|changed| !matches!(changed, Value::Null | Value::Bool(false))); let held: Option = held_copy(&HELD_COPY, (&options.target, generation), consumption); let read_consumption: bool = consumption || held.as_ref().is_some_and(|held| held.consumption); - let copy = match read_copy( - client, - &options.target, - PreviewCopyRead { - session_id, - generation, - consumption: read_consumption, - versioned: true, - known_version: held.as_ref().map(|held| held.version.clone()), - }, - ) - .await - { + let read = PreviewCopyRead { + session_id, + generation, + consumption: read_consumption, + known_version: held.as_ref().map(|held| held.version.clone()), + }; + let copy = match client.preview_copy(&options.target, &read).await { Ok(copy) => copy, Err(error) => { return Ok(Some(without_diffs(plan_failure(&format!( diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 65c69f86f1..6f9bc4e296 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -162,8 +162,6 @@ pub(crate) struct LocationServe { pub(crate) metadata_reconcile_seconds: u32, /// Seconds between safety syncs of an idle location; `0` turns them off. pub(crate) safety_sync_seconds: u64, - /// A flag, the environment or the project set the interval, rather than the default. - pub(crate) safety_sync_explicit: bool, /// `sqb dev` reads events on stdout and sends control lines on stdin. pub(crate) dev_events: bool, } @@ -265,8 +263,6 @@ pub(crate) struct LocationMetadataSync { pub(crate) safety_seconds: u64, /// When an idle location syncs even though no run ended and nobody asked. pub(crate) safety_at: tokio::time::Instant, - /// The last read found the warehouse file held by another process, such as a DuckDB build. - pub(crate) busy: bool, /// Failed reads in a row that did not shrink the page; a delivered page resets it. pub(crate) failed_reads: u32, /// Failed reads in a row stopped the sync; it rests idle until a run, a request, a publish or a safety sync. @@ -281,16 +277,12 @@ pub(crate) struct LocationMetadataSync { pub(crate) mirror_holds_rows: bool, /// One warm runtime interpreter and warehouse session serve every page of a synchronization. pub(crate) reader: Option, - /// The server's state-history limit, read as each synchronization begins; `None` from older servers. + /// The server's state-history limit, read as each synchronization begins; `None` before the first. pub(crate) state_history_versions: Option, - /// Seconds a state-table or catalog page may take before the read counts as failed. - pub(crate) page_seconds: u64, /// Progress of the current synchronization, for its per-table lines and summary. pub(crate) progress: SyncProgress, /// The adapter's history reads start or wake billed compute, as the publication's connection check reports. pub(crate) reads_wake_compute: bool, - /// Someone set the safety interval, so it applies even where reads wake compute. - pub(crate) safety_explicit: bool, /// The server applied a delivered page of the current synchronization, so the synced copy changed. pub(crate) applied: bool, /// The generation was published after the last plan of what changed, which waits for this sync. @@ -306,6 +298,8 @@ pub(crate) struct SyncProgress { pub(crate) scan_rows: u64, /// The current table's change token showed it unchanged, so nothing was read. pub(crate) scan_unchanged: bool, + /// Records the server mirrors of the current table, as it counted after the table's last page. + pub(crate) scan_mirrored: Option, } impl SyncProgress { @@ -317,6 +311,7 @@ impl SyncProgress { sync_rows: 0, scan_rows: 0, scan_unchanged: false, + scan_mirrored: None, } } } @@ -358,8 +353,7 @@ impl ReadFailure { pub(crate) fn cause(&self) -> Option { match self { Self::Busy => None, - Self::ReaderStart(_) => Some(WarehouseReadFailure::ReaderStart {}), - Self::Read(_) => Some(WarehouseReadFailure::ReadError {}), + Self::ReaderStart(_) | Self::Read(_) => Some(WarehouseReadFailure::ReadError {}), Self::TimedOut(seconds) => Some(WarehouseReadFailure::TimedOut { seconds: *seconds }), } } @@ -422,12 +416,13 @@ impl LocationMetadataSync { /// Whether an idle location skips interval syncs because history reads wake billed compute. pub(crate) fn defers_safety_sync(&self) -> bool { - self.reads_wake_compute && !self.safety_explicit + self.reads_wake_compute } - /// Start the sync `sync_due` asked for at server time `now`; a stopped sync restarts its pass. + /// Start the sync `sync_due` asked for at server time `now`; a stopped sync restarts its pass with a fresh failure count. pub(crate) fn begin_sync(&mut self, now: DateTime) { if std::mem::take(&mut self.halted) { + self.failed_reads = 0; self.index = 0; self.source_after = None; self.scan_id = Uuid::now_v7(); diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs index ec80fc150e..ec83c94d0a 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -61,6 +61,7 @@ pub(super) fn acknowledge_complete_pages( generation: 1, records: Vec::new(), scan: None, + last_page: false, }, next_source_after: None, }); @@ -106,7 +107,6 @@ pub(super) fn published_location( image_id: None, metadata_reconcile_seconds: 3600, safety_sync_seconds: 1800, - safety_sync_explicit: false, dev_events: false, }; let client = connect_location_work((&options.server_url, "token"), options.location_id) @@ -130,6 +130,5 @@ pub(super) fn published_location( let mut state = metadata.expect("a connected publication starts a sync"); state.begin_sync(chrono::Utc::now()); state.idle = false; - state.page_seconds = 5; ((options, script), (client, poll), state) } diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index f1a6e68c04..ac6fb7505c 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -177,7 +177,8 @@ fn given_repeated_failed_reads_when_retrying_then_only_page_size_errors_shrink_a } #[test] -fn given_a_stopped_sync_when_a_run_or_its_safety_sync_is_due_then_it_makes_one_attempt() { +fn given_a_stopped_sync_when_a_run_or_its_safety_sync_resumes_it_then_failed_reads_back_off_again() +{ let test_cases = [ SafetyResumeTestCase { description: "before the safety sync is due the sync stays stopped", @@ -188,12 +189,12 @@ fn given_a_stopped_sync_when_a_run_or_its_safety_sync_is_due_then_it_makes_one_a expected_outcome: (false, false, None, true), }, SafetyResumeTestCase { - description: "a due safety sync resumes it, and one more failure stops it for another interval", + description: "a due safety sync resumes it with a fresh count, so one more failure backs off", safety_seconds: 1800, due_in: Duration::ZERO, billed: false, ran: false, - expected_outcome: (true, true, None, true), + expected_outcome: (true, true, Some(Duration::from_secs(30)), false), }, SafetyResumeTestCase { description: "with safety syncs off only a run, a request or a publication resumes it", @@ -212,20 +213,20 @@ fn given_a_stopped_sync_when_a_run_or_its_safety_sync_is_due_then_it_makes_one_a expected_outcome: (false, false, None, true), }, SafetyResumeTestCase { - description: "on a billed warehouse a run that ends after the stop resumes it", + description: "on a billed warehouse a run that ends after the stop resumes it with a fresh count", safety_seconds: 1800, due_in: Duration::from_secs(600), billed: true, ran: true, - expected_outcome: (true, true, None, true), + expected_outcome: (true, true, Some(Duration::from_secs(30)), false), }, ]; for test_case in test_cases { let mut state = metadata_sync((one_location(), 1), (3600, test_case.safety_seconds)); state.reads_wake_compute = test_case.billed; - state.safety_explicit = !test_case.billed; state.begin_sync(Utc::now()); let stopped: Vec> = (0..5).map(|_| state.fail_read(false)).collect(); + let stopped_for: Duration = state.safety_at.saturating_duration_since(Instant::now()); state.safety_at = Instant::now() + test_case.due_in; let runs_ended_at = test_case .ran @@ -238,10 +239,7 @@ fn given_a_stopped_sync_when_a_run_or_its_safety_sync_is_due_then_it_makes_one_a ( stopped[4], (resumed, paging, wait, state.halted), - (state.safety_at.saturating_duration_since(Instant::now()) - + Duration::from_secs(1)) - .as_secs() - >= test_case.safety_seconds, + (stopped_for + Duration::from_secs(1)).as_secs() >= test_case.safety_seconds, ), (None, test_case.expected_outcome, true), "{}", @@ -605,35 +603,28 @@ fn given_a_state_history_limit_when_requesting_history_then_only_full_reads_of_b description: "a first read of builds stops at the limit", kind: WarehouseHistoryKind::Fingerprints, incremental: false, - server_limit: Some(5), + server_limit: 5, expected_newest_versions: Some(5), }, NewestVersionsTestCase { description: "a first read of check results stops at the limit", kind: WarehouseHistoryKind::AuditResults, incremental: false, - server_limit: Some(20), + server_limit: 20, expected_newest_versions: Some(20), }, NewestVersionsTestCase { description: "an incremental read delivers every new build; pruning bounds the store", kind: WarehouseHistoryKind::Fingerprints, incremental: true, - server_limit: Some(5), + server_limit: 5, expected_newest_versions: None, }, NewestVersionsTestCase { description: "other tables are never limited", kind: WarehouseHistoryKind::NodeResults, incremental: false, - server_limit: Some(5), - expected_newest_versions: None, - }, - NewestVersionsTestCase { - description: "a server without the setting gets the full history", - kind: WarehouseHistoryKind::Fingerprints, - incremental: false, - server_limit: None, + server_limit: 5, expected_newest_versions: None, }, ]; @@ -643,7 +634,7 @@ fn given_a_state_history_limit_when_requesting_history_then_only_full_reads_of_b .iter() .position(|kind| *kind == test_case.kind) .expect("kind is listed"); - state.state_history_versions = test_case.server_limit; + state.state_history_versions = Some(test_case.server_limit); state.observed_since = test_case.incremental.then(Utc::now); let request: serde_json::Value = serde_json::from_slice(&history_request(&state).expect("request")).expect("JSON"); @@ -711,7 +702,6 @@ fn given_a_failing_metadata_read_when_the_page_fails_then_the_cause_kind_and_pag image_id: None, metadata_reconcile_seconds: 3600, safety_sync_seconds: 0, - safety_sync_explicit: false, dev_events: false, }; let mut state = metadata_sync((one_location(), 1), (3600, 0)); @@ -719,13 +709,12 @@ fn given_a_failing_metadata_read_when_the_page_fails_then_the_cause_kind_and_pag .iter() .position(|kind| *kind == test_case.kind) .expect("kind is listed"); - state.page_seconds = 1; let (state, page) = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .expect("runtime") .block_on(read_metadata_page( - (&options, &test_case.script), + (&options, &test_case.script, 1), state, std::future::pending(), )) @@ -883,13 +872,11 @@ fn given_a_delivered_page_when_reporting_health_then_progress_completes_only_on_ } #[test] -fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_or_a_set_interval_sync() - { +fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_or_requests_sync() { let test_cases = [ WakingIdleTestCase { - description: "the default interval never wakes compute on a timer", + description: "the interval, set or default, never wakes compute on a timer", wakes: true, - explicit: false, run_ended: false, requested: false, expected_due: false, @@ -897,15 +884,6 @@ fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_o WakingIdleTestCase { description: "metadata-only reads keep the default interval", wakes: false, - explicit: false, - run_ended: false, - requested: false, - expected_due: true, - }, - WakingIdleTestCase { - description: "an interval someone set still applies", - wakes: true, - explicit: true, run_ended: false, requested: false, expected_due: true, @@ -913,7 +891,6 @@ fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_o WakingIdleTestCase { description: "a run that ended still syncs", wakes: true, - explicit: false, run_ended: true, requested: false, expected_due: true, @@ -921,7 +898,6 @@ fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_o WakingIdleTestCase { description: "a request still syncs", wakes: true, - explicit: false, run_ended: false, requested: true, expected_due: true, @@ -932,7 +908,6 @@ fn given_an_adapter_whose_reads_wake_compute_when_idle_then_only_runs_requests_o state.cycle_started_at = Some(Utc::now()); state.safety_at = Instant::now() - Duration::from_secs(1); state.reads_wake_compute = test_case.wakes; - state.safety_explicit = test_case.explicit; test_case.requested.then(|| state.request_sync()); let runs_ended_at = test_case .run_ended @@ -1008,13 +983,13 @@ fn given_a_failed_read_when_reporting_health_then_it_carries_cause_error_and_whe ), }, FailedReadHealthTestCase { - description: "a reader that did not start reports its own cause", + description: "a reader that did not start reports a read error that names it", failure: ReadFailure::ReaderStart("no such file".into()), failures_before: 0, billed: false, expected_report: ( false, - Some(WarehouseReadFailure::ReaderStart {}), + Some(WarehouseReadFailure::ReadError {}), Some("the metadata reader did not start: no such file"), false, None, @@ -1028,7 +1003,6 @@ fn given_a_failed_read_when_reporting_health_then_it_carries_cause_error_and_whe .position(|kind| *kind == WarehouseHistoryKind::Fingerprints) .expect("kind is listed"); state.reads_wake_compute = test_case.billed; - state.safety_explicit = false; state.begin_sync(Utc::now()); (0..test_case.failures_before).for_each(|_| { let _ = state.fail_read(false); @@ -1154,7 +1128,7 @@ fn given_a_published_location_when_its_sync_stops_before_any_page_succeeds_then_ let mut state = state; for _ in 0..METADATA_FAILURE_LIMIT { let (read, page) = - read_metadata_page((&options, &script), state, std::future::pending()) + read_metadata_page((&options, &script, 5), state, std::future::pending()) .await .expect("no control error") .expect("not stopped"); diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_sync_schedule.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_sync_schedule.rs index 76e437ac8c..603b926421 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_sync_schedule.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_sync_schedule.rs @@ -20,7 +20,6 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: None, safety_seconds: 1800, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: Some(1800), }, SyncCompletionTestCase { @@ -30,7 +29,6 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: None, safety_seconds: 1800, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: None, }, SyncCompletionTestCase { @@ -40,7 +38,6 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: None, safety_seconds: 1800, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: None, }, SyncCompletionTestCase { @@ -50,7 +47,6 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: Some(WarehouseSyncFailure::Delivery), safety_seconds: 1800, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: None, }, SyncCompletionTestCase { @@ -60,35 +56,22 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa failure: None, safety_seconds: 0, reads_wake_compute: false, - safety_explicit: false, expected_next_in_seconds: None, }, SyncCompletionTestCase { - description: "where reads wake compute the default interval names no next sync", + description: "where reads wake compute no next sync is named, whatever the interval", acknowledged_scans: last, more_pages: false, failure: None, safety_seconds: 1800, reads_wake_compute: true, - safety_explicit: false, expected_next_in_seconds: None, }, - SyncCompletionTestCase { - description: "where reads wake compute an interval someone set still names the next sync", - acknowledged_scans: last, - more_pages: false, - failure: None, - safety_seconds: 1800, - reads_wake_compute: true, - safety_explicit: true, - expected_next_in_seconds: Some(1800), - }, ]; let now = Utc::now(); for test_case in test_cases { let mut state = metadata_sync((one_location(), 1), (3600, test_case.safety_seconds)); state.reads_wake_compute = test_case.reads_wake_compute; - state.safety_explicit = test_case.safety_explicit; state.begin_sync(now); state = acknowledge_complete_pages(state, test_case.acknowledged_scans); state.pending = Some(LocationMetadataPage { @@ -98,6 +81,7 @@ fn given_a_sync_page_report_when_it_completes_the_sync_then_it_names_the_next_sa generation: 1, records: Vec::new(), scan: None, + last_page: false, }, next_source_after: test_case.more_pages.then(|| vec![Some("orders".into())]), }); diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 83ce6e7039..41cc028ef7 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -188,7 +188,7 @@ pub(crate) struct NewestVersionsTestCase { pub(crate) description: &'static str, pub(crate) kind: WarehouseHistoryKind, pub(crate) incremental: bool, - pub(crate) server_limit: Option, + pub(crate) server_limit: u16, pub(crate) expected_newest_versions: Option, } @@ -216,7 +216,6 @@ pub(super) struct SyncCompletionTestCase { pub(super) failure: Option, pub(super) safety_seconds: u64, pub(super) reads_wake_compute: bool, - pub(super) safety_explicit: bool, pub(super) expected_next_in_seconds: Option, } @@ -288,7 +287,6 @@ pub(crate) struct SyncProgressTestCase { pub(crate) struct WakingIdleTestCase { pub(crate) description: &'static str, pub(crate) wakes: bool, - pub(crate) explicit: bool, pub(crate) run_ended: bool, pub(crate) requested: bool, pub(crate) expected_due: bool, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index fd5bbbf923..f1e5d9fe70 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -195,10 +195,6 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/health", get(crate::api::_helpers::resources::person_reads::health), ) - .route( - "/api/control/locations/{location_id}/targets/{target}/mirrored-totals", - get(crate::api::_helpers::resources::person_reads::mirrored_totals), - ) .route( "/api/control/platform/problems", get(crate::api::_helpers::resources::person_reads::problems), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs index 123eb3a3ac..032cc5426a 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs @@ -26,10 +26,9 @@ use crate::node_states::main::read_person_node_states::read_person_node_states; use crate::node_states::models::{CachedJson, NodeStateCache, NodeStatesRecord}; use crate::platform::main::health::read_person_location_health::read_person_location_health; use crate::platform::main::person_reads::list_person_platform_events::list_person_platform_events; -use crate::platform::main::person_reads::read_person_mirrored_totals::read_person_mirrored_totals; use crate::platform::main::person_reads::read_person_problems::read_person_problems; use crate::platform::models::{ - LocationHealthObservation, MirroredTotal, PersonPlatformEventPage, PlatformProblems, + LocationHealthObservation, PersonPlatformEventPage, PlatformProblems, }; use crate::runs::main::listings::list_person_run_nodes::list_person_run_nodes; use crate::runs::models::{ @@ -656,28 +655,6 @@ pub(crate) async fn health( )) } -#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/mirrored-totals", operation_id = "readPersonMirroredTotals", - params(("location_id" = Uuid, Path), ("target" = String, Path)), - responses((status = 200, body = Vec, description = "Records the server mirrors of each state table of this target, in sync order"), - (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] -pub(crate) async fn mirrored_totals( - (State(store), Extension(context)): (State, Extension), - Extension(default_role): Extension, - Path((location_id, target)): Path<(Uuid, String)>, - Extension(AuthenticatedPerson(principal, _)): Extension, -) -> Result>, ApiError> { - Ok(Json( - read_person_mirrored_totals( - &store, - (&context, principal), - (location_id, &target), - default_role, - ) - .await?, - )) -} - #[utoipa::path(get, path = "/api/control/platform/problems", operation_id = "readPersonPlatformProblems", responses((status = 200, body = PlatformProblems, description = "Live scheduler state, and broken locations and late ticks of targets the person lists"), (status = 401, description = "Session rejected")), security(("session" = []), ("cliSession" = [])))] diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index f50f64741d..73ead0ebc8 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -47,10 +47,6 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, Extension), - Extension(identity): Extension, - Path((location_id, _target)): Path<(Uuid, String)>, -) -> Result, ApiError> { - require_location(&identity, location_id)?; - Ok(Json(WarehouseRetention { - state_history_versions: facts.state_history.kept_versions(&store), - })) -} - #[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/sync-due", params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = WarehouseSyncDueRead, responses((status = 200, body = WarehouseSyncDue), (status = 400, body = crate::api::models::ApiErrorBody), @@ -129,14 +112,15 @@ pub(crate) async fn retention( (status = 409, description = "Reader generation fenced")), security(("bearer" = [])))] pub(crate) async fn sync_due( - State(store): State, + (State(store), Extension(facts)): (State, Extension), Extension(identity): Extension, Path((location_id, target)): Path<(Uuid, String)>, Json(request): Json, ) -> Result, ApiError> { require_location(&identity, location_id)?; + let versions: u16 = facts.state_history.kept_versions(&store); Ok(Json( - read_sync_due(&store, (location_id, &target), &request).await?, + read_sync_due(&store, (location_id, &target), (&request, versions)).await?, )) } diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 8e3ab58dcc..31c54c5e53 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -51,7 +51,6 @@ use crate::api::_helpers::resources::{ person_reads::sensor_state, person_reads::snapshots, person_reads::health, - person_reads::mirrored_totals, person_reads::problems, person_reads::platform_events, person_operations::catalog, @@ -112,7 +111,6 @@ use crate::api::_helpers::resources::{ warehouse_metadata::preview_copy, changed_builds::report, warehouse_metadata::statistics_due, - warehouse_metadata::retention, warehouse_metadata::sync_due, warehouse_metadata::watermark ))] diff --git a/crates/sqlbuild-orchestration/src/node_states/models.rs b/crates/sqlbuild-orchestration/src/node_states/models.rs index 0ee77410a5..aab7d14fd7 100644 --- a/crates/sqlbuild-orchestration/src/node_states/models.rs +++ b/crates/sqlbuild-orchestration/src/node_states/models.rs @@ -68,7 +68,7 @@ pub struct ChangedBuildReport { pub generation: i64, pub changed: Vec, pub downstream: Vec, - /// Why the plan failed, sent with empty lists; only newer servers accept it. + /// Why the plan failed, sent with empty lists. #[serde(default, skip_serializing_if = "Option::is_none")] pub failing: Option, } diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location/work_client.rs index 41271cbc42..d4f39af723 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/location/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location/work_client.rs @@ -12,8 +12,7 @@ use crate::platform::models::{ }; use crate::warehouse_metadata::models::{ MetadataBatch, PreviewCopy, PreviewCopyRead, StatisticsDueRead, StatisticsRead, - WarehouseRetention, WarehouseSyncDue, WarehouseSyncDueRead, WarehouseSyncResponse, - WarehouseWatermarkRead, + WarehouseSyncDue, WarehouseSyncDueRead, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -54,33 +53,6 @@ impl LocationWorkClient { .await } - /// The server's state-history limit; `None` from a server too old to have one. - pub async fn metadata_retention( - &self, - target: &str, - ) -> Result, ClientError> { - let location_id = self.location_id.to_string(); - let endpoint = self.connection.endpoint(&[ - "api", - "locations", - &location_id, - "targets", - target, - "metadata", - "retention", - ])?; - match client_response(self.connection.client.get(endpoint)).await { - Ok(retention) => Ok(Some(retention)), - Err(ClientError::Rejected(status)) - if status == reqwest::StatusCode::NOT_FOUND - || status == reqwest::StatusCode::METHOD_NOT_ALLOWED => - { - Ok(None) - } - Err(error) => Err(error), - } - } - pub async fn metadata_sync_due( &self, target: &str, diff --git a/crates/sqlbuild-orchestration/src/platform/main/person_reads/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/person_reads/mod.rs index 9f7dfb6b1e..beaa841041 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/person_reads/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/person_reads/mod.rs @@ -1,5 +1,4 @@ pub mod list_person_platform_events; -pub mod read_person_mirrored_totals; pub mod read_person_plan; pub mod read_person_plan_diff; pub mod read_person_plan_rows; diff --git a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_mirrored_totals.rs b/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_mirrored_totals.rs deleted file mode 100644 index f870f486b5..0000000000 --- a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_mirrored_totals.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! People read how many records the server mirrors of each state table, with the same view as the target's health. - -use std::collections::BTreeMap; - -use serde_json::Value; -use uuid::Uuid; - -use crate::authorization::main::decisions::authorize_listing::authorize_listing; -use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::platform::models::MirroredTotal; -use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, query::query}; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; -use crate::warehouse_metadata::models::WarehouseHistoryKind; - -/// Every mirrored state table of the target, in sync order, with the records held for it across its state locations. -pub async fn read_person_mirrored_totals( - store: &ControlStore, - (context, credentials): (&AuditContext, Uuid), - (location_id, target): (Uuid, &str), - default_role: DefaultRole, -) -> Result, ControlError> { - let (transaction, _) = authorize_listing( - begin(store, TransactionMode::Write).await?, - (context, credentials), - (location_id, target), - default_role, - ) - .await?; - let counts = WarehouseHistoryKind::ALL - .iter() - .map(|kind| { - format!( - "SELECT '{}' AS kind, COUNT(*) AS records FROM {} WHERE location_id=$1 AND target=$2", - kind.as_str(), - kind.mirror_table() - ) - }) - .collect::>() - .join(" UNION ALL "); - let (transaction, rows) = query( - transaction, - &counts, - vec![Parameter::Id(location_id), Parameter::Text(target.into())], - ) - .await?; - commit(transaction).await?; - Ok(WarehouseHistoryKind::ALL - .iter() - .map(|kind| MirroredTotal { - table: *kind, - records: recorded(&rows, *kind), - }) - .collect()) -} - -/// The records mirrored of `kind`, or none when the count names no row of it. -fn recorded(rows: &[BTreeMap], kind: WarehouseHistoryKind) -> u64 { - rows.iter() - .find(|row| row["kind"].as_str() == Some(kind.as_str())) - .and_then(|row| row["records"].as_u64()) - .unwrap_or_default() -} diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 30b9addc6f..5e50a95ed3 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -85,6 +85,9 @@ pub enum LocationHealth { /// On the report that completes or stops a sync, when the next safety sync is due unless a run ends first; absent otherwise, where no timed sync follows, and from older locations. #[serde(default, skip_serializing_if = "Option::is_none")] next_sync_at: Option>, + /// On the report of a table's last delivered page, the records the server then mirrored of that table across the target's state locations. + #[serde(default, skip_serializing_if = "Option::is_none")] + mirrored_records: Option, }, } @@ -610,21 +613,12 @@ pub struct WarehouseSyncProgress { #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum WarehouseReadFailure { - /// The metadata reader process did not start. - ReaderStart {}, - /// The reader could not connect to the warehouse, or the read raised an error. + /// The reader did not start or connect to the warehouse, or the read raised an error. ReadError {}, /// The reader did not answer within this many seconds. TimedOut { seconds: u64 }, } -/// How many records the server holds for one mirrored state table of a target. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -pub struct MirroredTotal { - pub table: WarehouseHistoryKind, - pub records: u64, -} - /// Configured rules that failed a location's load: how many findings, and the first of them. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs index 0ef8ba206d..8715161921 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs @@ -114,7 +114,7 @@ pub(crate) async fn ingest_events( commit_ingestion( transaction, (table, location_id, target), - (batch.scan.as_ref(), &keys), + (batch.scan.as_ref(), &keys, batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs index d8fde4e13f..b6b3db46ba 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs @@ -81,10 +81,11 @@ pub(crate) async fn next_ingestion_chunk( } /// Delivered rows take the scan's id; a complete scan deletes every row it did not deliver. +/// A read's last page also counts the records the target mirrors of `table`. pub(crate) async fn commit_ingestion( mut transaction: ControlTransaction, (table, location_id, target): (&str, Uuid, &str), - (scan, keys): (Option<&MetadataScan>, &[Uuid]), + (scan, keys, last_page): (Option<&MetadataScan>, &[Uuid], bool), mut applied: bool, ) -> Result { if let Some(scan) = scan { @@ -126,13 +127,27 @@ pub(crate) async fn commit_ingestion( applied |= !removed.is_empty(); } } + let mut mirrored_records: Option = None; + if last_page { + let counted; + (transaction, counted) = query( + transaction, + &format!("SELECT COUNT(*) AS records FROM {table} WHERE location_id=$1 AND target=$2"), + vec![Parameter::Id(location_id), Parameter::Text(target.into())], + ) + .await?; + mirrored_records = counted[0]["records"].as_u64(); + } if applied { transaction .changes .insert(Change::NodeStatus(location_id, target.into())); } commit(transaction).await?; - Ok(WarehouseSyncResponse { applied }) + Ok(WarehouseSyncResponse { + applied, + mirrored_records, + }) } /// Every record of a scan page must come from the scanned physical namespace. diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs index 90a1876e53..ac04c00b76 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs @@ -62,7 +62,7 @@ pub async fn ingest_catalog( commit_ingestion( transaction, (CATALOG_MIRROR, scope.0, scope.1), - (batch.scan.as_ref(), &keys), + (batch.scan.as_ref(), &keys, batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs index 22483f3b53..424b2581e3 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs @@ -43,7 +43,7 @@ pub async fn ingest_fingerprints( commit_ingestion( transaction, ("sqb_warehouse_fingerprints", location_id, target), - (batch.scan.as_ref(), &keys), + (batch.scan.as_ref(), &keys, batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs index e60a131609..745eb341c1 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs @@ -78,7 +78,7 @@ pub async fn ingest_source_freshness( commit_ingestion( transaction, ("sqb_warehouse_freshness", location_id, target), - (batch.scan.as_ref(), &keys), + (batch.scan.as_ref(), &keys, batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs index bf0046c67e..857d0fa150 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs @@ -70,7 +70,7 @@ pub async fn ingest_statistics( commit_ingestion( transaction, ("sqb_warehouse_statistics", scope.0, scope.1), - (None, &[]), + (None, &[], batch.last_page), applied, ) .await diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs index 32990874fb..1869c91ef6 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs @@ -13,7 +13,7 @@ pub fn preview_state_parts(copy: PreviewCopy, has_consumption: bool) -> Warehous WarehouseStateParts { provenance: SyncedProvenance { synced_at: copy.synced_at, - version: copy.version, + version: Some(copy.version), }, relations: copy.relations.into_iter().map(relation).collect(), fingerprints: copy diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs index b0ac7457cb..7483857d0b 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs @@ -19,13 +19,8 @@ pub async fn read_preview_copy( ) -> Result { let transaction = begin_ingestion(store, scope, (request.session_id, request.generation)).await?; - let (transaction, version) = if request.versioned { - let (transaction, version) = preview_version(transaction, scope).await?; - (transaction, Some(version)) - } else { - (transaction, None) - }; - if version.is_some() && version == request.known_version { + let (transaction, version) = preview_version(transaction, scope).await?; + if request.known_version.as_ref() == Some(&version) { commit(transaction).await?; return Ok(PreviewCopy { synced_at: None, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs index 573db4f6a5..3d9f9e3c62 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs @@ -11,7 +11,7 @@ use crate::warehouse_metadata::models::{WarehouseSyncDue, WarehouseSyncDueRead}; pub async fn read_sync_due( store: &ControlStore, scope: (Uuid, &str), - request: &WarehouseSyncDueRead, + (request, state_history_versions): (&WarehouseSyncDueRead, u16), ) -> Result { let transaction = begin_ingestion(store, scope, (request.session_id, request.generation)).await?; @@ -25,5 +25,9 @@ pub async fn read_sync_due( let runs_ended_at = serde_json::from_value(rows[0]["ended_at"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; commit(transaction).await?; - Ok(WarehouseSyncDue { runs_ended_at, now }) + Ok(WarehouseSyncDue { + runs_ended_at, + now, + state_history_versions, + }) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index fc24cb88a2..f5691eadb0 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -475,6 +475,8 @@ pub struct WarehouseSyncDueRead { pub struct WarehouseSyncDue { pub runs_ended_at: Option>, pub now: DateTime, + /// The newest versions per node or check a full read of state history needs to deliver. + pub state_history_versions: u16, } /// A location asks for the mirrored copy a plan preview plans from, as its fenced reader. @@ -483,12 +485,9 @@ pub struct WarehouseSyncDue { pub struct PreviewCopyRead { pub session_id: Uuid, pub generation: i64, - /// Whether the copy carries consumption history; sent only when set, as older servers reject it. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] + /// Whether the copy carries consumption history. + #[serde(default)] pub consumption: bool, - /// Whether the location holds copies by mirror version; sent only when set, as older servers reject it. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub versioned: bool, /// The mirror version of the copy the location holds; an unchanged mirror returns no records. #[serde(default, skip_serializing_if = "Option::is_none")] pub known_version: Option, @@ -526,9 +525,8 @@ pub struct PreviewCopy { /// Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub consumption: Vec, - /// The mirror version these records were read at; sent only to a versioned read. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub version: Option, + /// The mirror version these records were read at. + pub version: String, /// Set when the mirror is still at the read's known version: the records are left out. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub unchanged: bool, @@ -579,6 +577,9 @@ pub struct WarehouseMirrorCursor { #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] pub struct WarehouseSyncResponse { pub applied: bool, + /// Records the server mirrors of this table for the target, counted after a read's last page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mirrored_records: Option, } /// One bounded page of one state table from the registered reader of a published generation. @@ -592,6 +593,9 @@ pub struct MetadataBatch { pub records: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub scan: Option, + /// The table's read ends with this page, so the reply counts the records mirrored of it. + #[serde(default)] + pub last_page: bool, } /// A full scan's complete page removes the namespace's rows that the scan did not deliver. @@ -978,13 +982,6 @@ impl StateHistoryRetention { } } -/// The newest versions per node or check a full read of state history needs to deliver. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -#[serde(deny_unknown_fields)] -pub struct WarehouseRetention { - pub state_history_versions: u16, -} - /// A current snapshot node's declared metadata and latest build, without SQL, source or config. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct CatalogNodeRecord { diff --git a/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs b/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs index efb9785c55..8c1fac69a7 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs @@ -88,6 +88,26 @@ fn given_moves_views_and_cleanups_when_people_read_maintenance_then_each_model_s ) .await .expect("another target's view"); + let mut last_page = batch( + session_id, + "old_name_views", + records("old_name_history.json", &data["old_name_views"]), + ); + last_page.last_page = true; + let replies = [ + ingest_old_name_views(store, scope, &batch(session_id, "old_name_views", vec![])) + .await + .expect("an earlier page"), + ingest_old_name_views(store, scope, &last_page) + .await + .expect("the read's last page"), + ]; + assert_eq!( + replies.map(|reply| reply.mirrored_records), + [None, Some(4)], + "{}: only a read's last page counts the mirror, and only this target's records", + test_case.description + ); let (api, client) = start_password_api(store).await; let viewer = login(&client, &api.url, "orders-viewer").await; @@ -181,46 +201,6 @@ fn given_moves_views_and_cleanups_when_people_read_maintenance_then_each_model_s .0, StatusCode::FORBIDDEN ); - let totals_path = - format!("/api/control/locations/{location}/targets/development/mirrored-totals"); - let (status, totals) = get(&client, (&api.url, &viewer), &totals_path).await; - assert_eq!(status, StatusCode::OK, "{}", test_case.description); - let counted: Vec<(&str, u64)> = totals - .as_array() - .expect("totals") - .iter() - .filter(|total| total["records"] != 0) - .map(|total| { - ( - total["table"].as_str().expect("table"), - total["records"].as_u64().expect("records"), - ) - }) - .collect(); - assert_eq!( - (totals.as_array().expect("totals").len(), counted), - ( - 14, - vec![ - ("janitor_events", 2), - ("migrations", 3), - ("old_name_views", 4) - ] - ), - "{}: every state table, counting only this target's mirrored records", - test_case.description - ); - assert_eq!( - get( - &client, - (&api.url, &viewer), - &totals_path.replace("development", "preview") - ) - .await - .0, - StatusCode::FORBIDDEN, - "mirrored totals need the same view as the target's health" - ); stop_api(api).await; close_test_store(fixture_store).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index eeb54e9281..c92ce5e9f2 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -554,7 +554,7 @@ pub(crate) async fn sync_due_scope( let fields = reply.as_object().expect("object"); let mut keys: Vec<_> = fields.keys().collect(); keys.sort(); - assert_eq!(keys, ["now", "runs_ended_at"]); + assert_eq!(keys, ["now", "runs_ended_at", "state_history_versions"]); let ended = fields["runs_ended_at"] .as_str() .map(|time| time.parse::>().expect("time")); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs index e936923cc5..81dedef654 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs @@ -159,7 +159,6 @@ pub(crate) async fn preview_copy_scope( session_id: preview_session, generation: 1, consumption: false, - versioned: false, known_version: None, }; let copy = || async { @@ -193,25 +192,25 @@ pub(crate) async fn preview_copy_scope( ) }; let unsynced = copy().await; - let older_location: Value = client + let plain: Value = client .post(format!("{metadata}/preview-copy")) .bearer_auth(bearer) .json(&json!({"session_id": preview_session, "generation": 1})) .send() .await - .expect("copy for an older location") + .expect("copy without consumption or a held version") .json() .await .expect("copy body"); assert!( unsynced.consumption.is_empty() - && ["consumption", "version", "unchanged"] + && ["consumption", "unchanged"] .iter() - .all(|field| older_location.get(field).is_none()), - "a read that asks for neither consumption nor versions never carries the fields, which older locations reject" + .all(|field| plain.get(field).is_none()) + && plain["version"] == json!(unsynced.version), + "a read without consumption or a held version carries neither, and always names its version" ); let versioned = |known_version: Option| PreviewCopyRead { - versioned: true, known_version, ..read.clone() }; @@ -220,12 +219,12 @@ pub(crate) async fn preview_copy_scope( .await .expect("versioned copy"); let held = outbound - .preview_copy("preview", &versioned(current.version.clone())) + .preview_copy("preview", &versioned(Some(current.version.clone()))) .await .expect("copy at the held version"); assert_eq!( ( - current.version.is_some() && !current.unchanged, + !current.version.is_empty() && !current.unchanged, ours(¤t) == ours(&unsynced), held.unchanged, held.version == current.version, @@ -248,7 +247,7 @@ pub(crate) async fn preview_copy_scope( ) .await; let changed = outbound - .preview_copy("preview", &versioned(current.version.clone())) + .preview_copy("preview", &versioned(Some(current.version.clone()))) .await .expect("copy after a mirrored write"); assert_eq!( @@ -264,7 +263,7 @@ pub(crate) async fn preview_copy_scope( ), "a mirrored write moves the version, so the held copy is read again" ); - let before_sync: Option = changed.version.clone(); + let before_sync: String = changed.version.clone(); let with_consumption = outbound .preview_copy( "preview", @@ -327,7 +326,7 @@ pub(crate) async fn preview_copy_scope( ); assert_ne!( outbound - .preview_copy("preview", &versioned(before_sync.clone())) + .preview_copy("preview", &versioned(Some(before_sync.clone()))) .await .expect("copy after a clean sync") .version, @@ -394,7 +393,6 @@ pub(crate) async fn preview_copy_scope( session_id, generation: 1, consumption: false, - versioned: false, known_version: None, }, ) diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index 704ff052d5..1708192f26 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -135,6 +135,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ generation: 1, records: records[..1].iter().map(|record| json!(record)).collect(), scan: None, + last_page: false, } ) .await @@ -159,6 +160,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ generation: 1, records: page.iter().map(|record| json!(record)).collect(), scan: None, + last_page: false, }; assert!( ingest_source_freshness(store, (location, "development"), &batch) @@ -252,6 +254,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ generation: 1, records: vec![json!(late)], scan: None, + last_page: false, }; ingest_source_freshness(store, (location, "development"), &batch) .await @@ -289,6 +292,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ generation: 1, records: page.iter().map(|record| json!(record)).collect(), scan: None, + last_page: false, }, ) .await @@ -422,6 +426,7 @@ fn given_a_source_version_that_returns_after_a_complete_scan_prunes_history_when generation: 1, records: vec![json!(record)], scan, + last_page: false, }, ) .await @@ -591,6 +596,7 @@ fn given_timestamp_observations_when_age_crosses_the_boundary_then_scoped_pages_ generation: 1, records: records.iter().map(|record| json!(record)).collect(), scan: None, + last_page: false, }; ingest_source_freshness(store, (location, "development"), &batch) .await diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs index ae8134f59d..43486986c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs @@ -48,12 +48,12 @@ fn requested_state( let text: String = std::fs::read_to_string(path)?; let copy: PreviewCopy = serde_json::from_str(&text) .map_err(|error| PyValueError::new_err(format!("invalid synced copy: {error}")))?; - let version: Option = copy.version.clone(); + let version: String = copy.version.clone(); let state: Arc = Arc::new(warehouse_state(preview_state_parts( copy, request["consumption"].as_bool().unwrap_or(false), ))); - *held = version.map(|version| HeldState { + *held = Some(HeldState { version, state: Arc::clone(&state), }); diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index a75a3a54af..8c8709e14d 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -751,13 +751,9 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, const locations = (await bootstrapLocations()).filter((l) => !healthRefused.has(l.id)); const read = await Promise.all( locations.map(async (location) => { - const { locationId, target } = parseLocationKey(location.id); - const [[{ data, response }, snapshot], totals] = await Promise.all([ - locationHealth(location.id), - http.GET('/api/control/locations/{location_id}/targets/{target}/mirrored-totals', { params: { path: { location_id: locationId, target } } }) - ]); + const [{ data, response }, snapshot] = await locationHealth(location.id); if (response.status === 403) healthRefused.add(location.id); - return data ? [toWarehouseConnection(location, data, snapshot, totals.data ?? null)] : []; + return data ? [toWarehouseConnection(location, data, snapshot)] : []; }) ); return read.flat(); diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 2ead32cb87..9d0d471f1e 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -128,21 +128,27 @@ describe('warehouseSyncOf', () => { ]); }); - it('counts the records mirrored of each table, and names the next sync from the report that completed one', () => { + it('counts the records mirrored of each table from its newest counted read, and names the next sync from the report that completed one', () => { const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; - const completed = sync('2026-11-08T12:03:00Z', 'statistics', null); - const observations = [{ ...completed, report: { ...completed.report, health: { ...completed.report.health, next_sync_at: '2026-11-08T12:33:00Z' } } }, sync('2026-11-08T12:02:00Z', 'catalog', null)]; - const connection = toWarehouseConnection(location, observations, null, [ - { table: 'catalog', records: 120 }, - { table: 'statistics', records: 4_800 } - ]); + const counted = (at: string, table: 'catalog' | 'statistics', records: number) => { + const report = sync(at, table, null); + return { ...report, report: { ...report.report, health: { ...report.report.health, mirrored_records: records } } }; + }; + const completed = counted('2026-11-08T12:03:00Z', 'statistics', 4_800); + const observations = [ + { ...completed, report: { ...completed.report, health: { ...completed.report.health, next_sync_at: '2026-11-08T12:33:00Z' } } }, + sync('2026-11-08T12:02:30Z', 'catalog', null), + counted('2026-11-08T12:02:00Z', 'catalog', 120), + counted('2026-11-08T12:01:00Z', 'catalog', 90) + ]; + const connection = toWarehouseConnection(location, observations); expect(connection.tables.map((t) => [t.table, t.mirroredTotal])).toEqual([ ['catalog', 120], ['statistics', 4_800] ]); expect(connection.nextSyncAt).toBe('2026-11-08T12:33:00Z'); - const running = toWarehouseConnection(location, [sync('2026-11-08T12:40:00Z', 'catalog', null), ...observations]); - expect([running.nextSyncAt, running.tables[0].mirroredTotal]).toEqual([null, null]); + const uncounted = toWarehouseConnection(location, [sync('2026-11-08T12:40:00Z', 'catalog', null)]); + expect([uncounted.nextSyncAt, uncounted.tables[0].mirroredTotal]).toEqual([null, null]); }); it('marks a sync that failed reads stopped, from the newest report only', () => { @@ -197,7 +203,7 @@ describe('warehouseSyncOf', () => { expect(warehouseSyncOf([{ ...timedOut, report: { ...timedOut.report, health } }])?.lastError).toBe('statistics: failed while reading the warehouse: no reply within 130 s, on a page of 100 relations'); const erroring = sync('2026-11-08T12:04:00Z', 'catalog', 'read'); const error = { ...erroring.report.health, read_failure: { kind: 'read_error' as const }, page_limit: 2000 }; - expect(warehouseSyncOf([{ ...erroring, report: { ...erroring.report, health: error } }])?.lastError).toBe('catalog: failed while reading the warehouse: the warehouse returned an error; the location’s log has it, on a page of 2,000 rows'); + expect(warehouseSyncOf([{ ...erroring, report: { ...erroring.report, health: error } }])?.lastError).toBe('catalog: failed while reading the warehouse: the read failed; the location’s log has why, on a page of 2,000 rows'); }); it('shows a running sync and the newest completed one, and a first sync as pending until it completes', () => { diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index 7e017abd87..96dc6e3be2 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -169,11 +169,9 @@ function readCauseOf(health: SyncHealth): string { const reason = cause.kind === 'timed_out' ? `no reply within ${cause.seconds} s` - : cause.kind === 'reader_start' - ? 'the metadata reader didn’t start' - : health.error - ? 'the warehouse returned an error' - : 'the warehouse returned an error; the location’s log has it'; + : health.error + ? 'the read failed' + : 'the read failed; the location’s log has why'; const page = health.page_limit ? `, on a page of ${health.page_limit.toLocaleString('en-US')} ${health.table === 'statistics' ? 'relations' : 'rows'}` : ''; return `: ${reason}${page}`; } @@ -199,16 +197,15 @@ export function connectionDetailsOf(published: Wire['PublishedSnapshot'] | null) } /** - * A location target's connection from its health reports, newest snapshot and mirrored totals: the - * settings it published, the newest connection check, where its state tables are read from, each - * table's newest sync and how many records the server mirrors of it, and when the next sync is due. - * `totals` is null when the server doesn't report them. + * A location target's connection from its health reports and newest snapshot: the settings it + * published, the newest connection check, where its state tables are read from, each table's newest + * sync and how many records the server mirrored of it as its last read finished, and when the next + * sync is due. */ export function toWarehouseConnection( location: Location, observations: Wire['LocationHealthObservation'][], - published: Wire['PublishedSnapshot'] | null = null, - totals: Wire['MirroredTotal'][] | null = null + published: Wire['PublishedSnapshot'] | null = null ): WarehouseConnection { const { syncs, newest } = syncsOf(observations); const tables: WarehouseTableSync[] = [...newest.values()] @@ -221,7 +218,7 @@ export function toWarehouseConnection( error: health.error ?? null, waiting: health.failure === 'busy', lastSyncedAt: syncs.find((s) => s.report.health.table === health.table && !s.report.health.failure)?.received_at ?? null, - mirroredTotal: totals ? (totals.find((t) => t.table === health.table)?.records ?? 0) : null + mirroredTotal: syncs.find((s) => s.report.health.table === health.table && s.report.health.mirrored_records != null)?.report.health.mirrored_records ?? null })) .sort((a, b) => a.table.localeCompare(b.table)); const state = syncs[0]?.report.health.location; diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 845cd2ec22..cd0ee84172 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -493,7 +493,7 @@ The location starts the sync within a few seconds, and the top bar shows when it | --- | --- | --- | --- | | `[control] server` | `SQLBUILD_CONTROL_SERVER` | none | HTTPS origin of the server, as `sqb login --server` takes it | | `[control] location` | `SQLBUILD_CONTROL_LOCATION` | none | Location name a signed-in `sqb build` asks to sync | -| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"`, or off on Databricks, ClickHouse and MotherDuck | Longest a location waits between syncs; `"0"` or `"off"` turns it off. See [warehouse cost](#warehouse-cost-of-an-always-on-location) | +| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"` | Longest a location waits between syncs; `"0"` or `"off"` turns it off. Databricks, ClickHouse and MotherDuck never sync on the interval. See [warehouse cost](#warehouse-cost-of-an-always-on-location) | | none | `SQLBUILD_CONTROL_CA_CERT` | none | Extra trusted PEM CA certificate for the server | An environment variable overrides the project file. `sqb location serve --sync-at-least-every` @@ -518,8 +518,8 @@ Details: or time retries a smaller page. Any other error, such as a missing permission or table, retries after 30 seconds, then waits twice as long after each failure in a row. After 5 failures in a row the sync stops until a run ends, you request one, the location publishes again, or the next safety - sync is due (`sync_at_least_every`). It then starts again from the first kind of history. A sync - resumed by a run or the safety sync tries once, and another failure stops it again. + sync is due (`sync_at_least_every`). It then starts again from the first kind of history, and + failed reads back off again before they stop it. - `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. - When a state table holds two rows with the same ID, such as one check result written twice, the server keeps the row with the earlier time. Estimated query costs are the exception: when two @@ -545,7 +545,7 @@ A location syncs when it starts, after each run ends, when someone asks, and on Where the interval is off, an idle location syncs only after a run ends or when someone asks, with **Sync now**, `sqb control sync` or a signed-in `sqb build`. A run has just used the warehouse, so -the sync after it rarely resumes anything. Set `sync_at_least_every` to turn the interval back on. +the sync after it rarely resumes anything. Setting `sync_at_least_every` doesn't change this. ### Follow a sync From ff2c9500b805dbd87bd3d1a68dd0dec8b9e1d6af Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:43:44 +0000 Subject: [PATCH 2/4] fix(location): repeat a skipped table's mirrored count A table the warehouse reports unchanged is skipped without delivery, so it reported no mirrored count and its total dropped out of the connections page once older reports aged out. The recorded change token now keeps the count from the read that recorded it, and the skipped read reports it again. Regenerate the server docs reference. --- .../_helpers/location/execution.rs | 1 + .../main/_location_metadata.rs | 2 +- .../src/native_commands/models.rs | 11 +++++++++ .../tests/test_location_metadata.rs | 23 +++++++++++++++++++ .../sqlbuild/references/docs/cli/server.md | 8 +++---- 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs index f495ee0637..decd1824c2 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs @@ -671,6 +671,7 @@ pub(crate) fn acknowledge_metadata_page(mut state: LocationMetadataSync) -> Loca token, reconciled: state.reconciling(), delivered: state.scan_delivered || state.observed_since.is_some(), + mirrored: state.progress.scan_mirrored, }; state.change_tokens.insert(state.index, recorded); } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index bc154fbe7e..5c4216cbe4 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -94,11 +94,11 @@ pub(crate) async fn sync_location_metadata( return Ok(Some(state)); } if page.unchanged { + state.skip_unchanged(); tokio::select! { result = &mut stopped => { result?; return Ok(None); }, () = metadata_health(&state, (client, poll), (started, None)) => {}, } - state.progress.scan_unchanged = true; return finish_metadata_page(next_scan(state)).await; } state = freeze_history_page(state, page, poll)?; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 6f9bc4e296..3e74194104 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -377,6 +377,8 @@ pub(crate) struct ScanToken { pub(crate) reconciled: bool, /// The read delivered rows or read on top of mirrored rows, so an empty mirror lost rows. pub(crate) delivered: bool, + /// The mirror's row count the server reported after the read, which a skipped read reports again. + pub(crate) mirrored: Option, } impl LocationMetadataSync { @@ -505,6 +507,15 @@ impl LocationMetadataSync { } /// The recorded token a scan may skip on, given whether it reconciles and the mirror holds rows. + /// Skip a table unchanged since its recorded read, reporting that read's mirrored count again. + pub(crate) fn skip_unchanged(&mut self) { + self.progress.scan_unchanged = true; + self.progress.scan_mirrored = self + .change_tokens + .get(&self.index) + .and_then(|recorded| recorded.mirrored); + } + pub(crate) fn unchanged_token(&self) -> Option<&str> { let recorded = self.change_tokens.get(&self.index)?; ((recorded.reconciled || !self.reconciling()) diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index ac6fb7505c..3c218c7726 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -405,6 +405,29 @@ fn given_recorded_change_token_when_scanning_again_then_only_proven_unchanged_re } } +#[test] +fn given_a_table_skipped_as_unchanged_when_reporting_health_then_its_last_mirrored_count_repeats() { + let mut state = metadata_sync((one_location(), 1), (3600, 0)); + state.scan_token = Some("t1".into()); + state.scan_delivered = true; + state.progress.scan_mirrored = Some(42); + let mut state = acknowledge_complete_pages(state, 1); + assert_eq!( + state.progress.scan_mirrored, None, + "the next table starts without a count" + ); + state.index = 0; + state.skip_unchanged(); + let health = metadata_health_of(&state, (Instant::now(), None, None), Utc::now()); + let LocationHealth::WarehouseSync { + mirrored_records, .. + } = health + else { + panic!("not a warehouse sync report"); + }; + assert_eq!(mirrored_records, Some(42)); +} + #[test] fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync_runs() { let test_cases = [ diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 67054f7e52..3c958c3dc7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -513,7 +513,7 @@ The location starts the sync within a few seconds, and the top bar shows when it | --- | --- | --- | --- | | `[control] server` | `SQLBUILD_CONTROL_SERVER` | none | HTTPS origin of the server, as `sqb login --server` takes it | | `[control] location` | `SQLBUILD_CONTROL_LOCATION` | none | Location name a signed-in `sqb build` asks to sync | -| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"`, or off on Databricks, ClickHouse and MotherDuck | Longest a location waits between syncs; `"0"` or `"off"` turns it off. See [warehouse cost](#warehouse-cost-of-an-always-on-location) | +| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"` | Longest a location waits between syncs; `"0"` or `"off"` turns it off. Databricks, ClickHouse and MotherDuck never sync on the interval. See [warehouse cost](#warehouse-cost-of-an-always-on-location) | | none | `SQLBUILD_CONTROL_CA_CERT` | none | Extra trusted PEM CA certificate for the server | An environment variable overrides the project file. `sqb location serve --sync-at-least-every` @@ -538,8 +538,8 @@ Details: or time retries a smaller page. Any other error, such as a missing permission or table, retries after 30 seconds, then waits twice as long after each failure in a row. After 5 failures in a row the sync stops until a run ends, you request one, the location publishes again, or the next safety - sync is due (`sync_at_least_every`). It then starts again from the first kind of history. A sync - resumed by a run or the safety sync tries once, and another failure stops it again. + sync is due (`sync_at_least_every`). It then starts again from the first kind of history, and + failed reads back off again before they stop it. - `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. - When a state table holds two rows with the same ID, such as one check result written twice, the server keeps the row with the earlier time. Estimated query costs are the exception: when two @@ -565,7 +565,7 @@ A location syncs when it starts, after each run ends, when someone asks, and on Where the interval is off, an idle location syncs only after a run ends or when someone asks, with **Sync now**, `sqb control sync` or a signed-in `sqb build`. A run has just used the warehouse, so -the sync after it rarely resumes anything. Set `sync_at_least_every` to turn the interval back on. +the sync after it rarely resumes anything. Setting `sync_at_least_every` doesn't change this. ### Follow a sync From 12901e3f4696d7b5796dd6aaab31059fb8a23501 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:49:39 +0000 Subject: [PATCH 3/4] fix(location): repeat the newest mirrored count of a skipped table's kind The server counts a kind's mirror across every state location, so a read of the same kind in another location replaces the count a skipped table repeats. Without this, a skipped table could report an older total than the read before it. --- .../_helpers/location/execution.rs | 9 +++ .../src/native_commands/models.rs | 7 +- .../tests/test_location_metadata.rs | 81 ++++++++++++++----- .../src/native_commands/tests/test_types.rs | 8 ++ 4 files changed, 81 insertions(+), 24 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs index decd1824c2..a973447b48 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs @@ -665,6 +665,15 @@ pub(crate) fn acknowledge_metadata_page(mut state: LocationMetadataSync) -> Loca if state.kind() == WarehouseHistoryKind::Statistics { state.statistics_limit = MAX_HISTORY_RECORDS; } + if let Some(count) = state.progress.scan_mirrored { + let kinds = WarehouseHistoryKind::ALL.len(); + let kind = state.index % kinds; + state + .change_tokens + .iter_mut() + .filter(|(index, _)| *index % kinds == kind) + .for_each(|(_, recorded)| recorded.mirrored = Some(count)); + } match state.scan_token.take() { Some(token) => { let recorded = ScanToken { diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 3e74194104..2607ce4386 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -377,7 +377,8 @@ pub(crate) struct ScanToken { pub(crate) reconciled: bool, /// The read delivered rows or read on top of mirrored rows, so an empty mirror lost rows. pub(crate) delivered: bool, - /// The mirror's row count the server reported after the read, which a skipped read reports again. + /// The newest row count the server reported for this kind's mirror, which covers every state + /// location, so a later read of the same kind elsewhere replaces it; a skipped read reports it again. pub(crate) mirrored: Option, } @@ -506,8 +507,7 @@ impl LocationMetadataSync { && (self.reconcile || self.kind() == WarehouseHistoryKind::Catalog) } - /// The recorded token a scan may skip on, given whether it reconciles and the mirror holds rows. - /// Skip a table unchanged since its recorded read, reporting that read's mirrored count again. + /// Skip a table unchanged since its recorded read, reporting the newest mirrored count of its kind again. pub(crate) fn skip_unchanged(&mut self) { self.progress.scan_unchanged = true; self.progress.scan_mirrored = self @@ -516,6 +516,7 @@ impl LocationMetadataSync { .and_then(|recorded| recorded.mirrored); } + /// The recorded token a scan may skip on, given whether it reconciles and the mirror holds rows. pub(crate) fn unchanged_token(&self) -> Option<&str> { let recorded = self.change_tokens.get(&self.index)?; ((recorded.reconciled || !self.reconciling()) diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index 3c218c7726..6b9083bc6e 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -19,7 +19,8 @@ use crate::native_commands::tests::test_types::{ }; use crate::native_commands::tests::test_types::{ MetadataBackoffTestCase, MetadataRetryTestCase, SafetyResumeTestCase, SafetySyncIdleTestCase, - SafetySyncSettingTestCase, StoppedPassTestCase, SyncCoalescingTestCase, UnchangedScanTestCase, + SafetySyncSettingTestCase, SkippedCountTestCase, StoppedPassTestCase, SyncCoalescingTestCase, + UnchangedScanTestCase, }; use crate::native_commands::tests::test_types::{ RowsTextTestCase, ScanLineTestCase, SyncProgressTestCase, WakingIdleTestCase, @@ -406,26 +407,64 @@ fn given_recorded_change_token_when_scanning_again_then_only_proven_unchanged_re } #[test] -fn given_a_table_skipped_as_unchanged_when_reporting_health_then_its_last_mirrored_count_repeats() { - let mut state = metadata_sync((one_location(), 1), (3600, 0)); - state.scan_token = Some("t1".into()); - state.scan_delivered = true; - state.progress.scan_mirrored = Some(42); - let mut state = acknowledge_complete_pages(state, 1); - assert_eq!( - state.progress.scan_mirrored, None, - "the next table starts without a count" - ); - state.index = 0; - state.skip_unchanged(); - let health = metadata_health_of(&state, (Instant::now(), None, None), Utc::now()); - let LocationHealth::WarehouseSync { - mirrored_records, .. - } = health - else { - panic!("not a warehouse sync report"); - }; - assert_eq!(mirrored_records, Some(42)); +fn given_a_table_skipped_as_unchanged_when_reporting_health_then_the_newest_mirrored_count_of_its_kind_repeats() + { + let test_cases = [ + SkippedCountTestCase { + description: "one state location repeats its own read's count", + locations: 1, + later_read: None, + expected_mirrored: Some(120), + }, + SkippedCountTestCase { + description: "a later read of the same kind in another location replaces the repeated count", + locations: 2, + later_read: Some(130), + expected_mirrored: Some(130), + }, + ]; + for test_case in test_cases { + let locations: Vec = (0..test_case.locations) + .map(|position| WarehouseStateLocation { + database: Some("analytics".into()), + schema: format!("state_{position}"), + }) + .collect(); + let kinds = WarehouseHistoryKind::ALL.len(); + let skipped = (test_case.locations - 1) * kinds; + let mut state = metadata_sync((locations, 1), (3600, 0)); + state.index = skipped; + state.scan_token = Some("t1".into()); + state.scan_delivered = true; + state.progress.scan_mirrored = Some(120); + let mut state = acknowledge_complete_pages(state, 1); + assert_eq!( + state.progress.scan_mirrored, None, + "{}: the next table starts without a count", + test_case.description + ); + if let Some(count) = test_case.later_read { + state.index = 0; + state.scan_token = Some("t2".into()); + state.scan_delivered = true; + state.progress.scan_mirrored = Some(count); + state = acknowledge_complete_pages(state, 1); + } + state.index = skipped; + state.skip_unchanged(); + let health = metadata_health_of(&state, (Instant::now(), None, None), Utc::now()); + let LocationHealth::WarehouseSync { + mirrored_records, .. + } = health + else { + panic!("{}: not a warehouse sync report", test_case.description); + }; + assert_eq!( + mirrored_records, test_case.expected_mirrored, + "{}", + test_case.description + ); + } } #[test] diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 41cc028ef7..ffa15282cd 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -304,3 +304,11 @@ pub(super) struct ReplanTestCase { pub(super) syncs: Vec<(bool, bool)>, pub(super) expected_replans: Vec, } + +pub(crate) struct SkippedCountTestCase { + pub(crate) description: &'static str, + pub(crate) locations: usize, + /// The count a read of the same kind in the first location reports after the skipped one's read. + pub(crate) later_read: Option, + pub(crate) expected_mirrored: Option, +} From 1d1b2cab8d055098726b19f142674625eb4165ff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:34:28 +0000 Subject: [PATCH 4/4] refactor(location): keep doc comments to one line and the count test branch-free --- .../_helpers/location/execution.rs | 3 +-- .../src/native_commands/models.rs | 3 +-- .../tests/test_location_metadata.rs | 19 +++++++++++-------- .../warehouse_metadata/_helpers/ingestion.rs | 3 +-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs index a973447b48..332e460f66 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs @@ -1368,8 +1368,7 @@ pub(crate) async fn stop_location_runtime( Ok(()) } -/// An outer absence means shutdown; an inner error is a failed read, never an empty table. -/// A state-table or catalog page that takes longer than `page_seconds` counts as failed. +/// An outer absence means shutdown; an inner error is a failed read, including a page slower than its limit. pub(crate) async fn read_metadata_page( (options, script, page_seconds): (&LocationServe, &Path, u64), mut state: LocationMetadataSync, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 2607ce4386..6cfa8faa12 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -377,8 +377,7 @@ pub(crate) struct ScanToken { pub(crate) reconciled: bool, /// The read delivered rows or read on top of mirrored rows, so an empty mirror lost rows. pub(crate) delivered: bool, - /// The newest row count the server reported for this kind's mirror, which covers every state - /// location, so a later read of the same kind elsewhere replaces it; a skipped read reports it again. + /// The server's newest count of this kind's mirror over every state location, which a skipped read repeats. pub(crate) mirrored: Option, } diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index 6b9083bc6e..3272a57351 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -437,19 +437,22 @@ fn given_a_table_skipped_as_unchanged_when_reporting_health_then_the_newest_mirr state.scan_token = Some("t1".into()); state.scan_delivered = true; state.progress.scan_mirrored = Some(120); - let mut state = acknowledge_complete_pages(state, 1); + let state = acknowledge_complete_pages(state, 1); assert_eq!( state.progress.scan_mirrored, None, "{}: the next table starts without a count", test_case.description ); - if let Some(count) = test_case.later_read { - state.index = 0; - state.scan_token = Some("t2".into()); - state.scan_delivered = true; - state.progress.scan_mirrored = Some(count); - state = acknowledge_complete_pages(state, 1); - } + let mut state = test_case + .later_read + .into_iter() + .fold(state, |mut state, count| { + state.index = 0; + state.scan_token = Some("t2".into()); + state.scan_delivered = true; + state.progress.scan_mirrored = Some(count); + acknowledge_complete_pages(state, 1) + }); state.index = skipped; state.skip_unchanged(); let health = metadata_health_of(&state, (Instant::now(), None, None), Utc::now()); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs index b6b3db46ba..82cc5820aa 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs @@ -80,8 +80,7 @@ pub(crate) async fn next_ingestion_chunk( .await } -/// Delivered rows take the scan's id; a complete scan deletes every row it did not deliver. -/// A read's last page also counts the records the target mirrors of `table`. +/// Delivered rows take the scan's id, a complete scan deletes the rest, and a last page counts `table`'s mirror. pub(crate) async fn commit_ingestion( mut transaction: ControlTransaction, (table, location_id, target): (&str, Uuid, &str),