Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
2614533
feat(state): add the display-only observations state table
kvlonge Nov 12, 2026
a54e9bc
feat(state): record cursor observations after successful builds
kvlonge Nov 12, 2026
66ad5e5
feat(state): record described source columns after builds
kvlonge Nov 12, 2026
8cdf18f
feat(orchestration): mirror build observations into the control store
kvlonge Nov 12, 2026
925fdd7
test(policy): keep build observations out of planning and build decis…
kvlonge Nov 12, 2026
967bb8a
fix(observations): record the described SQL hash with source columns
kvlonge Nov 12, 2026
a2d5307
feat(server): serve the mirrored copy a plan preview plans from
kvlonge Nov 13, 2026
c948bd0
feat(plan): plan location previews from the synced warehouse copy
kvlonge Nov 13, 2026
4d1198a
refactor(state): share the fixed-column state table DDL between consu…
kvlonge Nov 13, 2026
4690a1b
refactor(plan): render the copy's optional database without a higher-…
kvlonge Nov 13, 2026
c090def
feat(ui): show what a plan preview planned from and each incremental …
kvlonge Nov 13, 2026
11231a9
docs(server): explain that plan previews read the synced warehouse copy
kvlonge Nov 13, 2026
a4aaed4
refactor(plan): read a synced-copy listing's scope in one place
kvlonge Nov 13, 2026
bf7ab33
test(plan): move the preview test's helpers beside the publication he…
kvlonge Nov 13, 2026
bd44c90
fix(plan): keep previews within the synced copy on every adapter and …
kvlonge Nov 13, 2026
9cf4bc3
fix(plan): keep source freshness and moved cursor windows unknown in …
kvlonge Nov 13, 2026
294ca6a
fix(plan): bind preview cursor intrinsics only for models with waterm…
kvlonge Nov 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_empty_cursor_inputs.py \
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{
"record": {
"state_database": null,
"state_schema": "main",
"observation_id": "observation-000",
"kind": "cursor_max",
"node": "orders_🧾",
"cursor_type": "timestamp",
"value": "2026-02-03 04:05:06.123456",
"run_id": "external-run",
"observed_at": "2026-02-03T04:05:06Z"
},
"source_rows": 104,
"duplicate_rows": 3,
"later_time": "2026-02-03T04:05:06.000001Z",
"page_lengths": [
100,
7
],
"later_index": 106,
"source_count": 107,
"model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id",
"retained_result": "observation-103",
"kind": "observations",
"source_key": "observation_id",
"timestamp_key": "observed_at",
"id_prefix": "observation",
"later_record": {
"observation_id": "observation-103",
"kind": "source_columns",
"node": "shop.orders_🧾",
"cursor_type": null,
"value": "{\"sql_hash\":\"928300f876a1cfe48302513f8c25ca06ec449c95f3480c06b84c694a39b6be0d\",\"columns\":[\"order_id\",\"amount_€\"]}",
"observed_at": "2026-02-03T04:05:06.000001Z"
},
"last_record": {
"observation_id": "observation-103",
"kind": "source_columns",
"node": "shop.orders_🧾",
"cursor_type": null,
"value": "{\"sql_hash\":\"928300f876a1cfe48302513f8c25ca06ec449c95f3480c06b84c694a39b6be0d\",\"columns\":[\"order_id\",\"amount_€\"]}",
"observed_at": "2026-02-03T04:05:06.000001Z"
},
"record_overrides": {
"observation-103": {
"kind": "source_columns",
"node": "shop.orders_🧾",
"cursor_type": null,
"value": "{\"sql_hash\":\"928300f876a1cfe48302513f8c25ca06ec449c95f3480c06b84c694a39b6be0d\",\"columns\":[\"order_id\",\"amount_€\"]}",
"observed_at": "2026-02-03T04:05:06.000001Z"
}
}
}
129 changes: 128 additions & 1 deletion clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1010,6 +1010,22 @@ export interface paths {
patch?: never;
trace?: never;
};
"/api/locations/{location_id}/targets/{target}/metadata/preview-copy": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
post: operations["preview_copy"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/locations/{location_id}/targets/{target}/metadata/statistics-due": {
parameters: {
query?: never;
Expand Down Expand Up @@ -2308,6 +2324,30 @@ export interface components {
*/
scheduler_lease_expires_at?: string | null;
};
/** @description What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. */
PreviewCopy: {
fingerprints: components["schemas"]["WarehouseFingerprintRecord"][];
latest_runs: components["schemas"]["PreviewLatestRun"][];
observations: components["schemas"]["WarehouseObservationRecord"][];
relations: components["schemas"]["WarehouseCatalogRecord"][];
/** Format: date-time */
synced_at?: string | null;
};
/** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */
PreviewCopyRead: {
/** Format: int64 */
generation: number;
/** Format: uuid */
session_id: string;
};
/** @description The newest successful build of a node, which a `cursor_max` observation must come from. */
PreviewLatestRun: {
node_name: string;
node_type: string;
run_id: string;
state_database?: string | null;
state_schema: string;
};
/** @description A changed subject, such as `model:orders`, or `role:analyst` (`role:` for those who can't assign roles). */
PublishedChange: {
change_kind: string;
Expand Down Expand Up @@ -3078,11 +3118,30 @@ export interface components {
state_database?: string | null;
state_schema: string;
};
/** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */
WarehouseFingerprintRecord: {
definition: string;
definition_hash: string;
event_id?: string | null;
metadata_json: string;
node_name: string;
node_type: string;
run_id: string;
schema_fingerprint: string;
state_database?: string | null;
state_schema: string;
target_database?: string | null;
target_name?: string | null;
target_schema?: string | null;
/** Format: date-time */
ts: string;
version_hash: string;
};
/**
* @description Catalog observations require full scans; only immutable history supports a watermark.
* @enum {string}
*/
WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "catalog" | "statistics";
WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "observations" | "catalog" | "statistics";
/** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */
WarehouseJanitorRecord: {
archive_name: string;
Expand Down Expand Up @@ -3131,6 +3190,19 @@ export interface components {
/** Format: date-time */
observed_at: string;
};
/** @description A display-only value a build observed, such as a cursor `MAX`; planning never reads it. */
WarehouseObservationRecord: {
cursor_type?: string | null;
kind: string;
node: string;
observation_id: string;
/** Format: date-time */
observed_at: string;
run_id: string;
state_database?: string | null;
state_schema: string;
value: string;
};
/** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */
WarehouseStateLocation: {
database?: string | null;
Expand Down Expand Up @@ -5889,6 +5961,61 @@ export interface operations {
};
};
};
preview_copy: {
parameters: {
query?: never;
header?: never;
path: {
location_id: string;
target: string;
};
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["PreviewCopyRead"];
};
};
responses: {
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["PreviewCopy"];
};
};
400: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ApiErrorBody"];
};
};
/** @description Invalid credential */
401: {
headers: {
[name: string]: unknown;
};
content?: never;
};
/** @description Location not authorized */
403: {
headers: {
[name: string]: unknown;
};
content?: never;
};
/** @description Reader generation fenced */
409: {
headers: {
[name: string]: unknown;
};
content?: never;
};
};
};
statistics_due: {
parameters: {
query?: never;
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlbuild-analysis/src/janitor_planning/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use sqlbuild_core::constants::STATE_TABLE_NAMES;

/// SQLBuild state tables the janitor never acts on.
pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 10] = STATE_TABLE_NAMES;
pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 11] = STATE_TABLE_NAMES;
/// Prefix of every strict janitor archive name.
pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__";
/// Separator between archive name segments.
Expand Down
3 changes: 3 additions & 0 deletions crates/sqlbuild-cache/src/digest/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,6 @@ pub(crate) const DIRECT_URL_FILE: &str = "direct_url.json";

/// Inputs at least this large are hashed across the rayon pool; the digest is the same either way.
pub(crate) const PARALLEL_HASH_BYTES: usize = 1 << 20;
/// The standard base64 alphabet.
pub const BASE64_ALPHABET: &[u8; 64] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
27 changes: 27 additions & 0 deletions crates/sqlbuild-cache/src/digest/main/base64_text.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
//! Standard padded base64 text of bytes, as Python's `base64.b64encode(data).decode()` gives.

use crate::digest::constants::BASE64_ALPHABET;

/// Standard padded base64 of `data`.
#[must_use]
pub fn base64_text(data: &[u8]) -> String {
let mut encoded = String::with_capacity(data.len().div_ceil(3) * 4);
for chunk in data.chunks(3) {
let bytes = [
chunk[0],
*chunk.get(1).unwrap_or(&0),
*chunk.get(2).unwrap_or(&0),
];
let triple = u32::from(bytes[0]) << 16 | u32::from(bytes[1]) << 8 | u32::from(bytes[2]);
for index in 0..4 {
if index <= chunk.len() {
encoded.push(char::from(
BASE64_ALPHABET[(triple >> (18 - 6 * index) & 63) as usize],
));
} else {
encoded.push('=');
}
}
}
encoded
}
1 change: 1 addition & 0 deletions crates/sqlbuild-cache/src/digest/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
pub mod base64_text;
pub mod bytes_digest;
pub mod compiled_code_identity;
pub mod content_digest;
Expand Down
42 changes: 39 additions & 3 deletions crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
//! A requested plan runs as an ordinary `sqb plan --json` child on the server-frozen node keys.
//! A requested plan runs as a `sqb plan --json` child on frozen node keys, from the synced copy.

use std::collections::{BTreeMap, BTreeSet};
use std::future::Future;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::process::Stdio;

use serde_json::Value;
use sqlbuild_core::plan_display::constants::PLAN_SYNCED_COPY_ENV;
use sqlbuild_orchestration::platform::models::LocationWorkClient;
use sqlbuild_orchestration::store::errors::ControlError;
use sqlbuild_orchestration::warehouse_metadata::models::PreviewCopyRead;
use tokio::io::AsyncReadExt;
use tokio::process::Child;
use uuid::Uuid;

use crate::argv::main::accepted_settings_arguments::accepted_settings_arguments;
use crate::native_commands::_helpers::execution::location_child;
Expand All @@ -21,9 +25,39 @@ use crate::native_commands::models::LocationServe;
pub(super) async fn plan_location(
(options, script): (&LocationServe, &Path),
(selection, settings): (&BTreeSet<String>, &BTreeMap<String, Value>),
(client, session_id, generation): (&LocationWorkClient, Uuid, i64),
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<Value>, ControlError> {
let mut child = match plan_child((options, script), (selection, settings)) {
let read = PreviewCopyRead {
session_id,
generation,
};
let copy = match client.preview_copy(&options.target, &read).await {
Ok(copy) => copy,
Err(error) => {
return Ok(Some(plan_failure(&format!(
"the synced warehouse copy is unavailable: {error}"
))));
}
};
let path: PathBuf =
std::env::temp_dir().join(format!("sqlbuild-plan-copy-{}.json", Uuid::now_v7()));
std::fs::write(
&path,
serde_json::to_vec(&copy).map_err(std::io::Error::other)?,
)?;
let planned = run_plan((options, script), (selection, settings), &path, stopped).await;
drop(std::fs::remove_file(&path));
planned
}

async fn run_plan(
(options, script): (&LocationServe, &Path),
(selection, settings): (&BTreeSet<String>, &BTreeMap<String, Value>),
copy: &Path,
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<Value>, ControlError> {
let mut child = match plan_child((options, script), (selection, settings), copy) {
Ok(child) => child?,
Err(message) => return Ok(Some(plan_failure(&message))),
};
Expand Down Expand Up @@ -57,6 +91,7 @@ pub(super) async fn plan_location(
fn plan_child(
(options, script): (&LocationServe, &Path),
(selection, settings): (&BTreeSet<String>, &BTreeMap<String, Value>),
copy: &Path,
) -> Result<std::io::Result<Child>, String> {
let flags = accepted_settings_arguments(PLAN_COMMAND, settings)?;
let mut command = location_child(options, script);
Expand All @@ -78,6 +113,7 @@ fn plan_child(
Ok(command
.env_remove("SQLBUILD_LOCATION_TOKEN")
.env_remove("SQLBUILD_RUN_TOKEN")
.env(PLAN_SYNCED_COPY_ENV, copy)
.stdout(Stdio::piped())
.spawn())
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use sqlbuild_orchestration::platform::models::{
};
use sqlbuild_orchestration::store::errors::ControlError;
use tokio::time::Instant;
use uuid::Uuid;

use crate::argv::models::Stream;
use crate::native_commands::_helpers::execution::check_location_connection;
Expand Down Expand Up @@ -56,7 +57,8 @@ pub(super) async fn run_location_work(
() = tokio::time::sleep_until(deadline) => Ok(()),
}
};
let checked = perform_work((options, script), claim.request.work, expires).await?;
let copy = (client, poll.session_id, claim.generation);
let checked = perform_work((options, script), claim.request.work, copy, expires).await?;
let Some((outcome, local_files)) = checked else {
drop(write_output(&[(
Stream::Stderr,
Expand Down Expand Up @@ -122,6 +124,7 @@ pub(super) async fn run_location_work(
async fn perform_work(
(options, script): (&LocationServe, &Path),
work: LocationWork,
copy: (&LocationWorkClient, Uuid, i64),
expires: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<(LocationWorkOutcome, Vec<PathBuf>)>, ControlError> {
let message = match &work {
Expand Down Expand Up @@ -150,7 +153,7 @@ async fn perform_work(
selection,
settings,
..
} => plan_location((options, script), (&selection, &settings), expires)
} => plan_location((options, script), (&selection, &settings), copy, expires)
.await?
.map(|document| (LocationWorkOutcome::Plan { document }, Vec::new())),
})
Expand Down
Loading
Loading