Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 21 additions & 127 deletions clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -530,22 +530,6 @@ export interface paths {
patch?: never;
trace?: never;
};
"/api/control/locations/{location_id}/targets/{target}/mirrored-totals": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get: operations["readPersonMirroredTotals"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/control/locations/{location_id}/targets/{target}/node-states": {
parameters: {
query?: never;
Expand Down Expand Up @@ -1234,23 +1218,6 @@ export interface paths {
patch?: never;
trace?: never;
};
"/api/locations/{location_id}/targets/{target}/metadata/retention": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** How much state history a full read delivers; older servers answer 404 and the location reads it all. */
get: operations["read_metadata_retention"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/locations/{location_id}/targets/{target}/metadata/statistics-due": {
parameters: {
query?: never;
Expand Down Expand Up @@ -1765,7 +1732,7 @@ export interface components {
ChangedBuildReport: {
changed: string[];
downstream: string[];
/** @description Why the plan failed, sent with empty lists; only newer servers accept it. */
/** @description Why the plan failed, sent with empty lists. */
failing?: string | null;
/** Format: int64 */
generation: number;
Expand Down Expand Up @@ -2220,6 +2187,11 @@ export interface components {
/** @enum {string} */
kind: "warehouse_sync";
location: components["schemas"]["WarehouseStateLocation"];
/**
* Format: int64
* @description On the report of a table's last delivered page, the records the server then mirrored of that table across the target's state locations.
*/
mirrored_records?: number | null;
/**
* Format: date-time
* @description On the report that completes or stops a sync, when the next safety sync is due unless a run ends first; absent otherwise, where no timed sync follows, and from older locations.
Expand Down Expand Up @@ -2421,6 +2393,8 @@ export interface components {
/** Format: int64 */
generation: number;
kind: components["schemas"]["WarehouseHistoryKind"];
/** @description The table's read ends with this page, so the reply counts the records mirrored of it. */
last_page?: boolean;
records: Record<string, never>[];
scan?: null | components["schemas"]["MetadataScan"];
/** Format: uuid */
Expand Down Expand Up @@ -2452,12 +2426,6 @@ export interface components {
physical_generation_id: string;
target_name: string;
};
/** @description How many records the server holds for one mirrored state table of a target. */
MirroredTotal: {
/** Format: int64 */
records: number;
table: components["schemas"]["WarehouseHistoryKind"];
};
MutationResponse: {
applied: boolean;
};
Expand Down Expand Up @@ -2955,21 +2923,19 @@ export interface components {
synced_at?: string | null;
/** @description Set when the mirror is still at the read's known version: the records are left out. */
unchanged?: boolean;
/** @description The mirror version these records were read at; sent only to a versioned read. */
version?: string | null;
/** @description The mirror version these records were read at. */
version: string;
};
/** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */
PreviewCopyRead: {
/** @description Whether the copy carries consumption history; sent only when set, as older servers reject it. */
/** @description Whether the copy carries consumption history. */
consumption?: boolean;
/** Format: int64 */
generation: number;
/** @description The mirror version of the copy the location holds; an unchanged mirror returns no records. */
known_version?: string | null;
/** Format: uuid */
session_id: string;
/** @description Whether the location holds copies by mirror version; sent only when set, as older servers reject it. */
versioned?: boolean;
};
/** @description The newest successful build of a node, which a `cursor_max` observation must come from. */
PreviewLatestRun: {
Expand Down Expand Up @@ -4102,9 +4068,6 @@ export interface components {
};
/** @description What went wrong reading one page, without the warehouse's message, which may name hosts or users. */
WarehouseReadFailure: {
/** @enum {string} */
kind: "reader_start";
} | {
/** @enum {string} */
kind: "read_error";
} | {
Expand Down Expand Up @@ -4150,11 +4113,6 @@ export interface components {
warehouse_size: string;
warehouse_type?: string | null;
};
/** @description The newest versions per node or check a full read of state history needs to deliver. */
WarehouseRetention: {
/** Format: int32 */
state_history_versions: number;
};
/** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */
WarehouseStateLocation: {
database?: string | null;
Expand All @@ -4180,6 +4138,11 @@ export interface components {
now: string;
/** Format: date-time */
runs_ended_at?: string | null;
/**
* Format: int32
* @description The newest versions per node or check a full read of state history needs to deliver.
*/
state_history_versions: number;
};
/** @description A registered reader asks whether a run ended since its last synchronization began. */
WarehouseSyncDueRead: {
Expand Down Expand Up @@ -4222,6 +4185,11 @@ export interface components {
};
WarehouseSyncResponse: {
applied: boolean;
/**
* Format: int64
* @description Records the server mirrors of this table for the target, counted after a read's last page.
*/
mirrored_records?: number | null;
};
/** @description Derive progress from committed mirror rows, scoped to the active physical reader. */
WarehouseWatermarkRead: {
Expand Down Expand Up @@ -5586,44 +5554,6 @@ export interface operations {
};
};
};
readPersonMirroredTotals: {
parameters: {
query?: never;
header?: never;
path: {
location_id: string;
target: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Records the server mirrors of each state table of this target, in sync order */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MirroredTotal"][];
};
};
/** @description Session rejected */
401: {
headers: {
[name: string]: unknown;
};
content?: never;
};
403: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ApiErrorBody"];
};
};
};
};
readPersonNodeStates: {
parameters: {
query?: never;
Expand Down Expand Up @@ -7599,42 +7529,6 @@ export interface operations {
};
};
};
read_metadata_retention: {
parameters: {
query?: never;
header?: never;
path: {
location_id: string;
target: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["WarehouseRetention"];
};
};
/** @description Invalid credential */
401: {
headers: {
[name: string]: unknown;
};
content?: never;
};
/** @description Location not authorized */
403: {
headers: {
[name: string]: unknown;
};
content?: never;
};
};
};
statistics_due: {
parameters: {
query?: never;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ use crate::native_commands::_helpers::command::terminal::{live_screen, stderr_co
use crate::native_commands::_helpers::location::sync_report::{
metadata_health, read_timeout_seconds, rows_text, scan_line,
};
use crate::native_commands::constants::PLAN_NOT_PRODUCED;
use crate::native_commands::constants::SCREEN_INDENT;
use crate::native_commands::constants::{
ADMIN_COMMAND, CHANGED_PATHS_SHOWN, COLUMNS_VARIABLE, FALLBACK_COLUMNS,
Expand All @@ -42,7 +43,6 @@ use crate::native_commands::constants::{
METADATA_SYNC_CHECK_SECONDS, SAFETY_SYNC_ENV, SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS,
UNKNOWN_COMMAND,
};
use crate::native_commands::constants::{METADATA_PAGE_SECONDS, PLAN_NOT_PRODUCED};
use crate::native_commands::errors::PythonNeeded;
use crate::native_commands::main::_run_admin::run_admin;
use crate::native_commands::main::_run_control::run_control;
Expand Down Expand Up @@ -502,7 +502,6 @@ pub(crate) fn metadata_sync(
server_offset: chrono::Duration::zero(),
safety_seconds,
safety_at: Instant::now() + Duration::from_secs(safety_seconds),
busy: false,
failed_reads: 0,
halted: false,
change_tokens: HashMap::new(),
Expand All @@ -511,10 +510,8 @@ pub(crate) fn metadata_sync(
mirror_holds_rows: false,
reader: None,
state_history_versions: None,
page_seconds: METADATA_PAGE_SECONDS,
progress: SyncProgress::new(),
reads_wake_compute: false,
safety_explicit: true,
}
}

Expand Down Expand Up @@ -597,6 +594,7 @@ pub(crate) fn freeze_history_page(
location: state.location().clone(),
complete: page.next_after.is_none(),
}),
last_page: page.next_after.is_none(),
};
if serde_json::to_vec(&batch)
.map_err(|error| ControlError::Invalid(error.to_string()))?
Expand Down Expand Up @@ -648,10 +646,7 @@ pub(crate) async fn begin_after_run(
};
let ran = state.sync_due(due.runs_ended_at);
if ran {
state.state_history_versions = match client.metadata_retention(target).await {
Ok(retention) => retention.map(|retention| retention.state_history_versions),
Err(error) => return (state, Err(error)),
};
state.state_history_versions = Some(due.state_history_versions);
state.begin_sync(due.now);
}
(state, Ok(ran))
Expand All @@ -670,12 +665,22 @@ pub(crate) fn acknowledge_metadata_page(mut state: LocationMetadataSync) -> Loca
if state.kind() == WarehouseHistoryKind::Statistics {
state.statistics_limit = MAX_HISTORY_RECORDS;
}
if let Some(count) = state.progress.scan_mirrored {
let kinds = WarehouseHistoryKind::ALL.len();
let kind = state.index % kinds;
state
.change_tokens
.iter_mut()
.filter(|(index, _)| *index % kinds == kind)
.for_each(|(_, recorded)| recorded.mirrored = Some(count));
}
match state.scan_token.take() {
Some(token) => {
let recorded = ScanToken {
token,
reconciled: state.reconciling(),
delivered: state.scan_delivered || state.observed_since.is_some(),
mirrored: state.progress.scan_mirrored,
};
state.change_tokens.insert(state.index, recorded);
}
Expand Down Expand Up @@ -743,6 +748,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync
state.progress.sync_rows += state.progress.scan_rows;
state.progress.scan_rows = 0;
state.progress.scan_unchanged = false;
state.progress.scan_mirrored = None;
state.progress.scan_started = Instant::now();
state.index += 1;
state.scan_id = state.server_scan_id();
Expand Down Expand Up @@ -874,7 +880,6 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result<LocationServe, C
)
.map_err(ControlError::Invalid)?;
Ok(LocationServe {
safety_sync_explicit: safety_sync.is_some(),
safety_sync_seconds: safety_sync.unwrap_or(METADATA_SAFETY_SYNC_SECONDS),
project_dir,
server_url: text(namespace, "location_server_url").unwrap_or_default(),
Expand Down Expand Up @@ -1363,9 +1368,9 @@ pub(crate) async fn stop_location_runtime(
Ok(())
}

/// An outer absence means shutdown; an inner error is a failed read, never an empty table.
/// An outer absence means shutdown; an inner error is a failed read, including a page slower than its limit.
pub(crate) async fn read_metadata_page(
(options, script): (&LocationServe, &Path),
(options, script, page_seconds): (&LocationServe, &Path, u64),
mut state: LocationMetadataSync,
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<
Expand All @@ -1375,7 +1380,6 @@ pub(crate) async fn read_metadata_page(
)>,
ControlError,
> {
state.busy = false;
if let Some(statistics) = &state.statistics
&& statistics.names.is_empty()
{
Expand All @@ -1400,7 +1404,7 @@ pub(crate) async fn read_metadata_page(
}
},
};
let seconds = read_timeout_seconds(&state);
let seconds = read_timeout_seconds(&state, page_seconds);
tokio::pin!(stopped);
let read = tokio::select! {
result = &mut stopped => {
Expand All @@ -1417,7 +1421,6 @@ pub(crate) async fn read_metadata_page(
Ok(page)
}
Ok((reader, Ok(MetadataReply::Busy))) => {
state.busy = true;
state.reader = Some(reader);
Err(ReadFailure::Busy)
}
Expand Down Expand Up @@ -1633,14 +1636,13 @@ pub(crate) async fn settle_publication(
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<Settled>, ControlError> {
let (mut metadata, deferred) = if connect {
let mut sync: LocationMetadataSync = metadata_sync(
let sync: LocationMetadataSync = metadata_sync(
(locations, generation),
(
options.metadata_reconcile_seconds,
options.safety_sync_seconds,
),
);
sync.safety_explicit = options.safety_sync_explicit;
(Some(sync), None)
} else {
(metadata, Some((locations, generation)))
Expand Down
Loading
Loading