diff --git a/Makefile b/Makefile index 9e646384b7..6ed94078e8 100644 --- a/Makefile +++ b/Makefile @@ -147,6 +147,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_empty_cursor_inputs.py \ diff --git a/benchmarks/duckshop/edge_cases/orchestration/observation_history.json b/benchmarks/duckshop/edge_cases/orchestration/observation_history.json new file mode 100644 index 0000000000..e195f33703 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/observation_history.json @@ -0,0 +1,53 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "observation_id": "observation-000", + "kind": "cursor_max", + "node": "orders_🧾", + "cursor_type": "timestamp", + "value": "2026-02-03 04:05:06.123456", + "run_id": "external-run", + "observed_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "page_lengths": [ + 100, + 7 + ], + "later_index": 106, + "source_count": 107, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id", + "retained_result": "observation-103", + "kind": "observations", + "source_key": "observation_id", + "timestamp_key": "observed_at", + "id_prefix": "observation", + "later_record": { + "observation_id": "observation-103", + "kind": "source_columns", + "node": "shop.orders_🧾", + "cursor_type": null, + "value": "{\"sql_hash\":\"928300f876a1cfe48302513f8c25ca06ec449c95f3480c06b84c694a39b6be0d\",\"columns\":[\"order_id\",\"amount_€\"]}", + "observed_at": "2026-02-03T04:05:06.000001Z" + }, + "last_record": { + "observation_id": "observation-103", + "kind": "source_columns", + "node": "shop.orders_🧾", + "cursor_type": null, + "value": "{\"sql_hash\":\"928300f876a1cfe48302513f8c25ca06ec449c95f3480c06b84c694a39b6be0d\",\"columns\":[\"order_id\",\"amount_€\"]}", + "observed_at": "2026-02-03T04:05:06.000001Z" + }, + "record_overrides": { + "observation-103": { + "kind": "source_columns", + "node": "shop.orders_🧾", + "cursor_type": null, + "value": "{\"sql_hash\":\"928300f876a1cfe48302513f8c25ca06ec449c95f3480c06b84c694a39b6be0d\",\"columns\":[\"order_id\",\"amount_€\"]}", + "observed_at": "2026-02-03T04:05:06.000001Z" + } + } +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ccc5e5cc8a..38cff562b5 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1010,6 +1010,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/preview-copy": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["preview_copy"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/statistics-due": { parameters: { query?: never; @@ -2308,6 +2324,30 @@ export interface components { */ scheduler_lease_expires_at?: string | null; }; + /** @description What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. */ + PreviewCopy: { + fingerprints: components["schemas"]["WarehouseFingerprintRecord"][]; + latest_runs: components["schemas"]["PreviewLatestRun"][]; + observations: components["schemas"]["WarehouseObservationRecord"][]; + relations: components["schemas"]["WarehouseCatalogRecord"][]; + /** Format: date-time */ + synced_at?: string | null; + }; + /** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */ + PreviewCopyRead: { + /** Format: int64 */ + generation: number; + /** Format: uuid */ + session_id: string; + }; + /** @description The newest successful build of a node, which a `cursor_max` observation must come from. */ + PreviewLatestRun: { + node_name: string; + node_type: string; + run_id: string; + state_database?: string | null; + state_schema: string; + }; /** @description A changed subject, such as `model:orders`, or `role:analyst` (`role:` for those who can't assign roles). */ PublishedChange: { change_kind: string; @@ -3078,11 +3118,30 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */ + WarehouseFingerprintRecord: { + definition: string; + definition_hash: string; + event_id?: string | null; + metadata_json: string; + node_name: string; + node_type: string; + run_id: string; + schema_fingerprint: string; + state_database?: string | null; + state_schema: string; + target_database?: string | null; + target_name?: string | null; + target_schema?: string | null; + /** Format: date-time */ + ts: string; + version_hash: string; + }; /** * @description Catalog observations require full scans; only immutable history supports a watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "catalog" | "statistics"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "observations" | "catalog" | "statistics"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; @@ -3131,6 +3190,19 @@ export interface components { /** Format: date-time */ observed_at: string; }; + /** @description A display-only value a build observed, such as a cursor `MAX`; planning never reads it. */ + WarehouseObservationRecord: { + cursor_type?: string | null; + kind: string; + node: string; + observation_id: string; + /** Format: date-time */ + observed_at: string; + run_id: string; + state_database?: string | null; + state_schema: string; + value: string; + }; /** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */ WarehouseStateLocation: { database?: string | null; @@ -5889,6 +5961,61 @@ export interface operations { }; }; }; + preview_copy: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PreviewCopyRead"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PreviewCopy"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; statistics_due: { parameters: { query?: never; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index c68aa2bfc7..533e9fefe6 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -3,7 +3,7 @@ use sqlbuild_core::constants::STATE_TABLE_NAMES; /// SQLBuild state tables the janitor never acts on. -pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 10] = STATE_TABLE_NAMES; +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 11] = STATE_TABLE_NAMES; /// Prefix of every strict janitor archive name. pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; /// Separator between archive name segments. diff --git a/crates/sqlbuild-cache/src/digest/constants.rs b/crates/sqlbuild-cache/src/digest/constants.rs index b2f4bd45bc..c059f2f237 100644 --- a/crates/sqlbuild-cache/src/digest/constants.rs +++ b/crates/sqlbuild-cache/src/digest/constants.rs @@ -51,3 +51,6 @@ pub(crate) const DIRECT_URL_FILE: &str = "direct_url.json"; /// Inputs at least this large are hashed across the rayon pool; the digest is the same either way. pub(crate) const PARALLEL_HASH_BYTES: usize = 1 << 20; +/// The standard base64 alphabet. +pub const BASE64_ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; diff --git a/crates/sqlbuild-cache/src/digest/main/base64_text.rs b/crates/sqlbuild-cache/src/digest/main/base64_text.rs new file mode 100644 index 0000000000..69c91e1268 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/base64_text.rs @@ -0,0 +1,27 @@ +//! Standard padded base64 text of bytes, as Python's `base64.b64encode(data).decode()` gives. + +use crate::digest::constants::BASE64_ALPHABET; + +/// Standard padded base64 of `data`. +#[must_use] +pub fn base64_text(data: &[u8]) -> String { + let mut encoded = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let bytes = [ + chunk[0], + *chunk.get(1).unwrap_or(&0), + *chunk.get(2).unwrap_or(&0), + ]; + let triple = u32::from(bytes[0]) << 16 | u32::from(bytes[1]) << 8 | u32::from(bytes[2]); + for index in 0..4 { + if index <= chunk.len() { + encoded.push(char::from( + BASE64_ALPHABET[(triple >> (18 - 6 * index) & 63) as usize], + )); + } else { + encoded.push('='); + } + } + } + encoded +} diff --git a/crates/sqlbuild-cache/src/digest/main/mod.rs b/crates/sqlbuild-cache/src/digest/main/mod.rs index 3d1fb78930..d9de0036e4 100644 --- a/crates/sqlbuild-cache/src/digest/main/mod.rs +++ b/crates/sqlbuild-cache/src/digest/main/mod.rs @@ -1,3 +1,4 @@ +pub mod base64_text; pub mod bytes_digest; pub mod compiled_code_identity; pub mod content_digest; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs index 0f6ed68932..bf7c4e627c 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -1,14 +1,18 @@ -//! A requested plan runs as an ordinary `sqb plan --json` child on the server-frozen node keys. +//! A requested plan runs as a `sqb plan --json` child on frozen node keys, from the synced copy. use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Stdio; use serde_json::Value; +use sqlbuild_core::plan_display::constants::PLAN_SYNCED_COPY_ENV; +use sqlbuild_orchestration::platform::models::LocationWorkClient; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::warehouse_metadata::models::PreviewCopyRead; use tokio::io::AsyncReadExt; use tokio::process::Child; +use uuid::Uuid; use crate::argv::main::accepted_settings_arguments::accepted_settings_arguments; use crate::native_commands::_helpers::execution::location_child; @@ -21,9 +25,39 @@ use crate::native_commands::models::LocationServe; pub(super) async fn plan_location( (options, script): (&LocationServe, &Path), (selection, settings): (&BTreeSet, &BTreeMap), + (client, session_id, generation): (&LocationWorkClient, Uuid, i64), stopped: impl Future>, ) -> Result, ControlError> { - let mut child = match plan_child((options, script), (selection, settings)) { + let read = PreviewCopyRead { + session_id, + generation, + }; + let copy = match client.preview_copy(&options.target, &read).await { + Ok(copy) => copy, + Err(error) => { + return Ok(Some(plan_failure(&format!( + "the synced warehouse copy is unavailable: {error}" + )))); + } + }; + let path: PathBuf = + std::env::temp_dir().join(format!("sqlbuild-plan-copy-{}.json", Uuid::now_v7())); + std::fs::write( + &path, + serde_json::to_vec(©).map_err(std::io::Error::other)?, + )?; + let planned = run_plan((options, script), (selection, settings), &path, stopped).await; + drop(std::fs::remove_file(&path)); + planned +} + +async fn run_plan( + (options, script): (&LocationServe, &Path), + (selection, settings): (&BTreeSet, &BTreeMap), + copy: &Path, + stopped: impl Future>, +) -> Result, ControlError> { + let mut child = match plan_child((options, script), (selection, settings), copy) { Ok(child) => child?, Err(message) => return Ok(Some(plan_failure(&message))), }; @@ -57,6 +91,7 @@ pub(super) async fn plan_location( fn plan_child( (options, script): (&LocationServe, &Path), (selection, settings): (&BTreeSet, &BTreeMap), + copy: &Path, ) -> Result, String> { let flags = accepted_settings_arguments(PLAN_COMMAND, settings)?; let mut command = location_child(options, script); @@ -78,6 +113,7 @@ fn plan_child( Ok(command .env_remove("SQLBUILD_LOCATION_TOKEN") .env_remove("SQLBUILD_RUN_TOKEN") + .env(PLAN_SYNCED_COPY_ENV, copy) .stdout(Stdio::piped()) .spawn()) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index 41b65faff0..cc51433faa 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::platform::models::{ }; use sqlbuild_orchestration::store::errors::ControlError; use tokio::time::Instant; +use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::check_location_connection; @@ -56,7 +57,8 @@ pub(super) async fn run_location_work( () = tokio::time::sleep_until(deadline) => Ok(()), } }; - let checked = perform_work((options, script), claim.request.work, expires).await?; + let copy = (client, poll.session_id, claim.generation); + let checked = perform_work((options, script), claim.request.work, copy, expires).await?; let Some((outcome, local_files)) = checked else { drop(write_output(&[( Stream::Stderr, @@ -122,6 +124,7 @@ pub(super) async fn run_location_work( async fn perform_work( (options, script): (&LocationServe, &Path), work: LocationWork, + copy: (&LocationWorkClient, Uuid, i64), expires: impl Future>, ) -> Result)>, ControlError> { let message = match &work { @@ -150,7 +153,7 @@ async fn perform_work( selection, settings, .. - } => plan_location((options, script), (&selection, &settings), expires) + } => plan_location((options, script), (&selection, &settings), copy, expires) .await? .map(|document| (LocationWorkOutcome::Plan { document }, Vec::new())), }) diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 7eeb30d732..1dc5865b99 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -38,8 +38,10 @@ pub const MICROBATCH_TABLE_NAME: &str = "_sqlbuild_microbatches"; pub const AUDIT_RESULTS_TABLE_NAME: &str = "_sqlbuild_audit_results"; /// State table of janitor archive and drop events. pub const JANITOR_EVENTS_TABLE_NAME: &str = "_sqlbuild_janitor_events"; +/// State table of display-only observations, which plan previews read and planning never does. +pub const OBSERVATIONS_TABLE_NAME: &str = "_sqlbuild_observations"; /// Every state table SQLBuild writes in a target's state schema. -pub const STATE_TABLE_NAMES: [&str; 10] = [ +pub const STATE_TABLE_NAMES: [&str; 11] = [ FINGERPRINT_TABLE_NAME, MICROBATCH_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, @@ -50,6 +52,7 @@ pub const STATE_TABLE_NAMES: [&str; 10] = [ MIGRATION_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, + OBSERVATIONS_TABLE_NAME, ]; /// A zero-length cursor duration. pub const ZERO_DAY_DURATION: &str = "0d"; diff --git a/crates/sqlbuild-core/src/plan_display/constants.rs b/crates/sqlbuild-core/src/plan_display/constants.rs index aa6211db6f..19fb27808d 100644 --- a/crates/sqlbuild-core/src/plan_display/constants.rs +++ b/crates/sqlbuild-core/src/plan_display/constants.rs @@ -35,3 +35,11 @@ pub(crate) const MIGRATION_INCOMPATIBLE: &str = "incompatible"; pub(crate) const WATERMARK_STRATEGY: &str = "watermark"; pub(crate) const QUERY_DIFF_INDENT: &str = " "; pub(crate) const UNIFIED_DIFF_HEADER_PREFIXES: [&str; 3] = ["---", "+++", "@@"]; +/// Names the synced-copy file a location writes for a plan preview; set, `sqb plan` never connects. +pub const PLAN_SYNCED_COPY_ENV: &str = "SQLBUILD_PLAN_SYNCED_COPY"; +/// The plan JSON key describing the synced copy a preview planned from. +pub const SYNCED_COPY_KEY: &str = "synced_copy"; +/// Why a preview refuses `--defer-to`: deferral reads another target's live state. +pub const SYNCED_COPY_DEFER_REFUSAL: &str = + "plan previews from the synced copy cannot use --defer-to"; +pub const SYNCED_COPY_DEFER_HELP: &str = "Run re-plans live, including deferral."; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs index 24a318b3f9..adeed404f4 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs @@ -41,5 +41,5 @@ pub(crate) async fn recent( Path(location_id): Path, ) -> Result>, ApiError> { require_location(&identity, location_id)?; - Ok(Json(location_health(&store, location_id).await?)) + Ok(Json(location_health(&store, (location_id, None)).await?)) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index 5df6d1402d..eb4365481d 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -35,6 +35,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router ingest_node_results(&store, scope, &batch).await, WarehouseHistoryKind::Consumption => ingest_consumption(&store, scope, &batch).await, + WarehouseHistoryKind::Observations => ingest_observations(&store, scope, &batch).await, WarehouseHistoryKind::Catalog => ingest_catalog(&store, scope, &batch).await, WarehouseHistoryKind::Statistics => ingest_statistics(&store, scope, &batch).await, }?; @@ -115,3 +119,21 @@ pub(crate) async fn sync_due( read_sync_due(&store, (location_id, &target), &request).await?, )) } + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/preview-copy", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = PreviewCopyRead, + responses((status = 200, body = PreviewCopy), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Reader generation fenced")), + security(("bearer" = [])))] +pub(crate) async fn preview_copy( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(request): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + read_preview_copy(&store, (location_id, &target), &request).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 54d45356d9..77bba7b6f7 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -96,6 +96,7 @@ use crate::api::_helpers::resources::{ trigger_reads::events, trigger_reads::ticks, warehouse_metadata::report, + warehouse_metadata::preview_copy, warehouse_metadata::statistics_due, warehouse_metadata::sync_due, warehouse_metadata::watermark diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index eaf407d5aa..a44a93c5b3 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,8 +10,8 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - MetadataBatch, StatisticsDueRead, StatisticsRead, WarehouseSyncDue, WarehouseSyncDueRead, - WarehouseSyncResponse, WarehouseWatermarkRead, + MetadataBatch, PreviewCopy, PreviewCopyRead, StatisticsDueRead, StatisticsRead, + WarehouseSyncDue, WarehouseSyncDueRead, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -54,6 +54,16 @@ impl LocationWorkClient { .await } + /// The mirrored copy a plan preview plans from, so previewing never opens the warehouse. + pub async fn preview_copy( + &self, + target: &str, + request: &PreviewCopyRead, + ) -> Result { + self.post_metadata((target, Some("preview-copy")), request) + .await + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_metadata( &self, diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs index 148e6fb77f..a50785fabe 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/location_health.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs @@ -8,14 +8,15 @@ use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, TransactionMode}; +/// Every target's observations, or only `target`'s. pub async fn location_health( store: &ControlStore, - location_id: Uuid, + (location_id, target): (Uuid, Option<&str>), ) -> Result, ControlError> { let (transaction, observations) = recent_health( begin(store, TransactionMode::Read).await?, location_id, - None, + target, ) .await?; commit(transaction).await?; diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql index 215b451ebb..eb7b972977 100644 --- a/crates/sqlbuild-orchestration/src/store/schema.sql +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -502,6 +502,21 @@ CREATE TABLE sqb_warehouse_consumption ( scan_id {uuid} ); CREATE INDEX sqb_consumption_namespace ON sqb_warehouse_consumption (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_observations ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + kind TEXT NOT NULL, + node TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_observations_namespace ON sqb_warehouse_observations (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE INDEX sqb_observations_latest ON sqb_warehouse_observations (location_id,target,kind,node,observed_at,event_id); CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( location_id, target, (record_json ->> 'source_name'), (record_json ->> 'target_database'), (record_json ->> 'target_schema'), diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs index e016900106..57732bff90 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs @@ -1,9 +1,45 @@ -//! Mirror history pages share database scoping and the (observed_at, event_id) keyset cursor. +//! Mirror reads share database scoping and the keyset cursor; previews read each node's newest facts. + +use std::collections::{BTreeMap, BTreeSet}; use chrono::{DateTime, Utc}; +use serde::de::DeserializeOwned; +use serde_json::Value; use uuid::Uuid; -use crate::store::models::Parameter; +use crate::authorization::models::CurrentSnapshotId; +use crate::platform::constants::LOCATION_RETENTION_DAYS; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::decompress::decompress; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::types::Row; +use crate::warehouse_metadata::constants::PREVIEW_REQUIRED_TABLES; +use crate::warehouse_metadata::models::{ + CleanSync, PreviewLatestRun, WarehouseCatalogRecord, WarehouseConsumptionRecord, + WarehouseFingerprintRecord, WarehouseObservationRecord, +}; + +const RELATIONS_SQL: &str = "SELECT record_json FROM sqb_warehouse_catalog \ + WHERE location_id=$1 AND target=$2 ORDER BY state_database, state_schema, event_id"; +const FINGERPRINTS_SQL: &str = "SELECT f.record_json, t.body FROM (SELECT record_json, definition_text_hash, \ + ROW_NUMBER() OVER (PARTITION BY state_database, state_schema, node_type, node_name, target_database, \ + target_schema, target_name ORDER BY observed_at DESC, event_id DESC) AS newest \ + FROM sqb_warehouse_fingerprints WHERE location_id=$1 AND target=$2) f \ + JOIN sqb_source_texts t ON t.text_hash=f.definition_text_hash WHERE f.newest=1 \ + ORDER BY f.record_json ->> 'node_type', f.record_json ->> 'node_name'"; +const LATEST_RUNS_SQL: &str = "SELECT record_json FROM (SELECT record_json, ROW_NUMBER() OVER (PARTITION BY \ + state_database, state_schema, record_json ->> 'node_type', record_json ->> 'node_name' \ + ORDER BY observed_at DESC, event_id DESC) AS newest FROM sqb_warehouse_consumption \ + WHERE location_id=$1 AND target=$2 AND record_json ->> 'phase'='succeeded' \ + AND record_json ->> 'run_id' IS NOT NULL) runs WHERE newest=1 \ + ORDER BY record_json ->> 'node_type', record_json ->> 'node_name'"; +const OBSERVATIONS_SQL: &str = "SELECT record_json FROM (SELECT record_json, kind, node, ROW_NUMBER() OVER \ + (PARTITION BY state_database, state_schema, kind, node ORDER BY observed_at DESC, event_id DESC) AS newest \ + FROM sqb_warehouse_observations WHERE location_id=$1 AND target=$2) observations WHERE newest=1 \ + ORDER BY kind, node"; /// Adds scope, cursor and page size to the parameters, with the SQL after the caller's conditions. pub(crate) fn page_tail( @@ -34,3 +70,166 @@ pub(crate) fn page_tail( ); (parameters, tail) } + +fn scope_parameters((location_id, target): (Uuid, &str)) -> Vec { + vec![Parameter::Id(location_id), Parameter::Text(target.into())] +} + +fn decoded(mut row: Row) -> Result { + serde_json::from_value(row.remove("record_json").unwrap_or(Value::Null)) + .map_err(|error| ControlError::Invalid(format!("invalid mirrored record: {error}"))) +} + +async fn mirrored( + transaction: ControlTransaction, + sql: &str, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, sql, scope_parameters(scope)).await?; + let mut records: Vec = Vec::with_capacity(rows.len()); + for row in rows { + records.push(decoded(row)?); + } + Ok((transaction, records)) +} + +/// Every relation of the target's newest complete catalog scans. +pub(crate) async fn preview_relations( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + mirrored(transaction, RELATIONS_SQL, scope).await +} + +/// Each node's newest fingerprint, with its definition restored from the shared text store. +pub(crate) async fn preview_fingerprints( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, FINGERPRINTS_SQL, scope_parameters(scope)).await?; + let mut fingerprints: Vec = Vec::with_capacity(rows.len()); + for mut row in rows { + let body: Vec = decompress(&row.remove("body").unwrap_or(Value::Null))?; + let definition: String = + String::from_utf8(body).map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut record: Value = row.remove("record_json").unwrap_or(Value::Null); + if let Value::Object(object) = &mut record { + object.insert("definition".into(), Value::String(definition)); + } + fingerprints.push(serde_json::from_value(record).map_err(|error| { + ControlError::Invalid(format!("invalid mirrored fingerprint: {error}")) + })?); + } + Ok((transaction, fingerprints)) +} + +/// Each node's newest succeeded build that names its node and run. +pub(crate) async fn preview_latest_runs( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, consumption): (ControlTransaction, Vec) = + mirrored(transaction, LATEST_RUNS_SQL, scope).await?; + let mut runs: Vec = Vec::with_capacity(consumption.len()); + for record in consumption { + if let (Some(node_type), Some(node_name), Some(run_id)) = + (record.node_type, record.node_name, record.run_id) + { + runs.push(PreviewLatestRun { + state_database: record.state_database, + state_schema: record.state_schema, + node_type, + node_name, + run_id, + }); + } + } + Ok((transaction, runs)) +} + +/// Each node's newest observation of each kind. +pub(crate) async fn preview_observations( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + mirrored(transaction, OBSERVATIONS_SQL, scope).await +} + +/// When every schema the current snapshot builds into last synced each required table cleanly. +pub(crate) async fn preview_synced_at( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, Option>), ControlError> { + let (transaction, current) = query( + transaction, + "SELECT snapshot_id FROM sqb_current_snapshots WHERE location_id=$1 AND target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())], + ) + .await?; + let current: Option = + current.into_iter().next().map(decode_row).transpose()?; + let Some(current) = current else { + return Ok((transaction, None)); + }; + let (transaction, nodes) = read_snapshot_nodes(transaction, current.snapshot_id, None).await?; + let schemas: BTreeSet<(Option, String)> = nodes + .values() + .filter_map(|node| { + let destination = &node.definition["destination"]; + Some(( + destination["database"].as_str().map(str::to_owned), + destination["schema"].as_str()?.to_owned(), + )) + }) + .collect(); + let today = transaction.now.date_naive(); + let days = 0..=LOCATION_RETENTION_DAYS; + let invocations: Vec = days + .map(|day| { + Parameter::Text(format!( + "location/{location_id}/{}", + today - chrono::TimeDelta::days(day) + )) + }) + .collect(); + let target_placeholder = invocations.len() + 1; + let placeholders = Parameter::placeholders(1..=invocations.len()); + let mut parameters = invocations; + parameters.push(Parameter::Text(target.into())); + let (transaction, rows) = query( + transaction, + &format!( + "SELECT payload->'report'->'health'->'location'->>'database' AS state_database, \ + payload->'report'->'health'->'location'->>'schema' AS state_schema, \ + payload->'report'->'health'->>'table' AS state_table, MAX(occurred_at) AS synced_at \ + FROM recall_events WHERE invocation_id IN ({placeholders}) AND event_type='location_health' \ + AND payload->'report'->>'target'=${target_placeholder} \ + AND payload->'report'->'health'->>'kind'='warehouse_sync' \ + AND payload->'report'->'health'->>'failure' IS NULL GROUP BY 1, 2, 3" + ), + parameters, + ) + .await?; + let newest: BTreeMap<(Option, String, String), DateTime> = rows + .into_iter() + .map(|row| decode_row::(row).map(clean_sync_entry)) + .collect::>()?; + let mut oldest: Option> = None; + for (database, schema) in &schemas { + for table in PREVIEW_REQUIRED_TABLES { + let Some(at) = newest.get(&(database.clone(), schema.clone(), table.as_str().into())) + else { + return Ok((transaction, None)); + }; + oldest = Some(oldest.map_or(*at, |oldest| oldest.min(*at))); + } + } + Ok((transaction, oldest)) +} + +fn clean_sync_entry(sync: CleanSync) -> ((Option, String, String), DateTime) { + ( + (sync.state_database, sync.state_schema, sync.state_table), + sync.synced_at, + ) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 5ff4f0a6f6..7a15ec7660 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -1,7 +1,15 @@ //! Limits bound each authenticated warehouse observation transaction. +use crate::warehouse_metadata::models::WarehouseHistoryKind; + pub const MAX_HISTORY_RECORDS: usize = 100; pub(crate) const CATALOG_MIRROR: &str = "sqb_warehouse_catalog"; pub(crate) const DEFAULT_STATISTICS_RETENTION_DAYS: u16 = 90; /// Column-lineage node definitions read per query, well under every backend's bound parameter limit. pub(crate) const LINEAGE_DEFINITION_BATCH: usize = 1000; +/// Mirrored tables a plan preview needs synced cleanly; observations only refine it into knowns. +pub(crate) const PREVIEW_REQUIRED_TABLES: [WarehouseHistoryKind; 3] = [ + WarehouseHistoryKind::Catalog, + WarehouseHistoryKind::Fingerprints, + WarehouseHistoryKind::Consumption, +]; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs new file mode 100644 index 0000000000..c76912e03d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs @@ -0,0 +1,57 @@ +//! Mirror display-only build observations; planning reads the warehouse, never these rows. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseObservationRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_observations( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: record.observation_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.observed_at, + columns: vec![ + ("kind", Parameter::Text(record.kind.clone())), + ("node", Parameter::Text(record.node.clone())), + ], + record_json: json!(record), + }) + .collect(), + records + .iter() + .flat_map(|record| { + [ + &record.observation_id, + &record.state_schema, + &record.kind, + &record.node, + &record.run_id, + ] + }) + .any(|value| value.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs index aa95f66a37..e54d3083df 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs @@ -7,6 +7,7 @@ pub mod ingest_janitor_events; pub mod ingest_microbatches; pub mod ingest_migrations; pub mod ingest_node_results; +pub mod ingest_observations; pub mod ingest_old_name_views; pub mod ingest_source_freshness; pub mod ingest_statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs index 05bf3ab152..d54f5f9745 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs @@ -1,3 +1,4 @@ +pub mod read_preview_copy; pub mod read_statistics_due; pub mod read_sync_due; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs new file mode 100644 index 0000000000..c72203c06b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs @@ -0,0 +1,35 @@ +//! A plan preview plans from the mirror, so the copy carries only what its facade may answer. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; +use crate::warehouse_metadata::_helpers::mirror_pages::{ + preview_fingerprints, preview_latest_runs, preview_observations, preview_relations, + preview_synced_at, +}; +use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; + +pub async fn read_preview_copy( + store: &ControlStore, + scope: (Uuid, &str), + request: &PreviewCopyRead, +) -> Result { + let transaction = + begin_ingestion(store, scope, (request.session_id, request.generation)).await?; + let (transaction, relations) = preview_relations(transaction, scope).await?; + let (transaction, fingerprints) = preview_fingerprints(transaction, scope).await?; + let (transaction, latest_runs) = preview_latest_runs(transaction, scope).await?; + let (transaction, observations) = preview_observations(transaction, scope).await?; + let (transaction, synced_at) = preview_synced_at(transaction, scope).await?; + commit(transaction).await?; + Ok(PreviewCopy { + synced_at, + relations, + fingerprints, + latest_runs, + observations, + }) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 5281ecee88..42bc27126a 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -7,7 +7,8 @@ use uuid::Uuid; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, - NODE_RESULTS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, }; use crate::store::errors::ControlError; @@ -270,6 +271,21 @@ pub struct WarehouseConsumptionRecord { pub recorded_at: DateTime, } +/// A display-only value a build observed, such as a cursor `MAX`; planning never reads it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseObservationRecord { + pub state_database: Option, + pub state_schema: String, + pub observation_id: String, + pub kind: String, + pub node: String, + pub cursor_type: Option, + pub value: String, + pub run_id: String, + pub observed_at: DateTime, +} + /// Catalog observations require full scans; only immutable history supports a watermark. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] @@ -284,13 +300,14 @@ pub enum WarehouseHistoryKind { ColumnMigrations, NodeResults, Consumption, + Observations, Catalog, Statistics, } impl WarehouseHistoryKind { /// Catalog and statistics stay last: a failed statistics read rewinds one kind, to catalog. - pub const ALL: [Self; 12] = [ + pub const ALL: [Self; 13] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, @@ -301,6 +318,7 @@ impl WarehouseHistoryKind { Self::ColumnMigrations, Self::NodeResults, Self::Consumption, + Self::Observations, Self::Catalog, Self::Statistics, ]; @@ -358,6 +376,11 @@ impl WarehouseHistoryKind { "sqb_warehouse_consumption", Some(CONSUMPTION_TABLE_NAME), ), + Self::Observations => ( + "observations", + "sqb_warehouse_observations", + Some(OBSERVATIONS_TABLE_NAME), + ), Self::Catalog => ("catalog", "sqb_warehouse_catalog", None), Self::Statistics => ("statistics", "sqb_warehouse_statistics", None), } @@ -406,6 +429,45 @@ pub struct WarehouseSyncDue { pub now: DateTime, } +/// A location asks for the mirrored copy a plan preview plans from, as its fenced reader. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct PreviewCopyRead { + pub session_id: Uuid, + pub generation: i64, +} + +/// The newest successful build of a node, which a `cursor_max` observation must come from. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct PreviewLatestRun { + pub state_database: Option, + pub state_schema: String, + pub node_type: String, + pub node_name: String, + pub run_id: String, +} + +/// One state table's newest clean sync in one schema, from location health. +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct CleanSync { + pub state_database: Option, + pub state_schema: String, + pub state_table: String, + pub synced_at: DateTime, +} + +/// What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct PreviewCopy { + pub synced_at: Option>, + pub relations: Vec, + pub fingerprints: Vec, + pub latest_runs: Vec, + pub observations: Vec, +} + /// Private interpreter input; cursor values are data and receive adapter literal rendering. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -540,6 +602,7 @@ pub enum WarehouseHistoryRead { ColumnMigrations(StateEventsRead), NodeResults(NodeResultsRead), Consumption(ConsumptionRead), + Observations(StateEventsRead), Catalog(CatalogRead), Statistics(StatisticsRead), } @@ -557,6 +620,7 @@ pub enum WarehouseHistoryRecord { ColumnMigration(Box), NodeResult(Box), Consumption(Box), + Observation(Box), Catalog(WarehouseCatalogRecord), Statistics(WarehouseStatisticsRecord), } diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 24b70b4ab0..a8e01d7cd0 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -106,6 +106,8 @@ mod warehouse_events_helpers; pub(crate) mod warehouse_history_helpers; #[path = "warehouse_metadata/helpers.rs"] mod warehouse_metadata_helpers; +#[path = "warehouse_metadata/preview_copy/helpers.rs"] +mod warehouse_preview_copy_helpers; #[path = "warehouse_metadata/reconciliation/helpers.rs"] mod warehouse_reconciliation_helpers; #[path = "work/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 56536e0d3f..2a7b57f4eb 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -304,6 +304,16 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit let preview_session = Uuid::now_v7(); let mut preview = input.clone(); preview.target = "preview".into(); + // A plan preview's copy counts as synced only once every schema the snapshot builds into has. + for (name, node) in &mut preview.nodes { + node.definition["destination"] = + json!({"database": null, "schema": "main", "name": name}); + } + preview + .nodes + .get_mut("products") + .expect("products node") + .definition["destination"]["schema"] = json!("reporting"); publish_snapshot(store, &preview) .await .expect("preview snapshot"); @@ -402,6 +412,11 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit "consumption_history", "execution_id", ), + ( + WarehouseHistoryKind::Observations, + "observation_history", + "observation_id", + ), ] { crate::api::tests::warehouse_events_helpers::event_history( store, @@ -418,6 +433,12 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit (&client, &url, &token.bearer), ) .await; + crate::api::tests::warehouse_preview_copy_helpers::preview_copy_scope( + &outbound, + (session_id, preview_session), + (&client, &url, &token.bearer), + ) + .await; stop_api(api).await; close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs index c8f58887a4..6925eac3df 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs @@ -137,7 +137,8 @@ pub(crate) async fn event_history( table: kind.mirror_table(), operation: "INSERT", condition: "WHEN 'rejected-result' IN (NEW.record_json ->> 'event_id', NEW.record_json ->> 'result_id', \ - NEW.record_json ->> 'node_name', NEW.record_json ->> 'execution_id')", + NEW.record_json ->> 'node_name', NEW.record_json ->> 'execution_id', \ + NEW.record_json ->> 'observation_id')", sqlite_body: "SELECT RAISE(ABORT, 'injected audit failure')", postgres_body: "RAISE EXCEPTION 'injected audit failure'", }; diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs new file mode 100644 index 0000000000..aad9cb8ab9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs @@ -0,0 +1,299 @@ +//! A preview copy holds each node's newest mirrored facts, dated by the stalest required clean sync. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::platform::models::{LocationHealthReport, LocationWorkClient}; +use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; + +const NODE: &str = "preview_copy_orders"; + +async fn post(client: &Client, (url, bearer): (&str, &str), body: Value) { + let response = client + .post(url) + .bearer_auth(bearer) + .json(&body) + .send() + .await + .expect("metadata batch"); + assert_eq!(response.status(), StatusCode::OK, "{body}"); +} + +fn fingerprint(run_id: &str, definition: &str, ts: &str) -> Value { + json!({"state_database": null, "state_schema": "main", "node_type": "model", "node_name": NODE, + "target_database": null, "target_schema": "main", "target_name": NODE, "run_id": run_id, + "definition_hash": format!("definition-{run_id}"), "version_hash": format!("version-{run_id}"), + "schema_fingerprint": "id:INTEGER", "definition": definition, "metadata_json": "{}", "ts": ts}) +} + +fn consumption(execution_id: &str, (phase, run_id): (&str, &str), recorded_at: &str) -> Value { + json!({"state_database": null, "state_schema": "main", "execution_id": execution_id, "phase": phase, + "run_id": run_id, "node_type": "model", "node_name": NODE, "target_database": null, + "target_schema": "main", "target_name": NODE, "version_hash": "version", "input_count": 0, + "input_kind": null, "input_name": null, "input_version": null, "recorded_at": recorded_at}) +} + +fn observation( + id: &str, + (kind, node, value): (&str, &str, &str), + (run_id, at): (&str, &str), +) -> Value { + json!({"state_database": null, "state_schema": "main", "observation_id": id, "kind": kind, + "node": node, "cursor_type": (kind == "cursor_max").then_some("integer"), "value": value, + "run_id": run_id, "observed_at": at}) +} + +fn sync_report(session: Uuid, table: &str, failure: Option<&str>) -> LocationHealthReport { + schema_sync_report(session, ("main", table), failure) +} + +fn schema_sync_report( + session: Uuid, + (schema, table): (&str, &str), + failure: Option<&str>, +) -> LocationHealthReport { + serde_json::from_value(json!({ + "event_id": Uuid::now_v7(), "session_id": session, "target": "preview", + "health": {"kind": "warehouse_sync", "generation": 1, "location": {"database": null, "schema": schema}, + "table": table, "failure": failure, "records_sent": 1, "elapsed_millis": 1}, + })) + .expect("sync health") +} + +/// The preview target's copy: newest fingerprint, succeeded run and observation per node, and `synced_at`. +pub(crate) async fn preview_copy_scope( + outbound: &LocationWorkClient, + (session_id, preview_session): (Uuid, Uuid), + (client, url, bearer): (&Client, &str, &str), +) { + let metadata = url.replace("/development/", "/preview/"); + let batch = |kind: &str, records: Vec| json!({"kind": kind, "session_id": preview_session, "generation": 1, "records": records}); + post( + client, + (&metadata, bearer), + batch( + "fingerprints", + vec![ + fingerprint("copy-run-2", "SELECT 2 AS id", "2026-03-01T11:00:00Z"), + fingerprint("copy-run-1", "SELECT 1 AS id", "2026-03-01T10:00:00Z"), + ], + ), + ) + .await; + post( + client, + (&metadata, bearer), + batch( + "consumption", + vec![ + consumption( + "copy-execution-1", + ("succeeded", "copy-run-1"), + "2026-03-01T10:00:00Z", + ), + consumption( + "copy-execution-2", + ("succeeded", "copy-run-2"), + "2026-03-01T11:00:00Z", + ), + consumption( + "copy-execution-3", + ("started", "copy-run-3"), + "2026-03-01T12:00:00Z", + ), + ], + ), + ) + .await; + post( + client, + (&metadata, bearer), + batch( + "observations", + vec![ + observation( + "copy-observation-2", + ("cursor_max", NODE, "2"), + ("copy-run-2", "2026-03-01T11:00:01Z"), + ), + observation( + "copy-observation-1", + ("cursor_max", NODE, "1"), + ("copy-run-1", "2026-03-01T10:00:01Z"), + ), + observation( + "copy-observation-3", + ("source_columns", "raw_copy", "{}"), + ("copy-run-1", "2026-03-01T10:00:02Z"), + ), + ], + ), + ) + .await; + let mut catalog = batch( + "catalog", + vec![json!({"state_database": null, "state_schema": "main", + "name": NODE, "relation_type": "table", "columns": [{"name": "id", "data_type": "INTEGER"}]})], + ); + catalog["scan"] = json!({"scan_id": Uuid::now_v7(), "location": {"database": null, "schema": "main"}, "complete": true}); + post(client, (&metadata, bearer), catalog).await; + + let read = PreviewCopyRead { + session_id: preview_session, + generation: 1, + }; + let copy = || async { + outbound + .preview_copy("preview", &read) + .await + .expect("preview copy") + }; + let ours = |copy: &PreviewCopy| { + ( + copy.fingerprints + .iter() + .filter(|f| f.node_name == NODE) + .map(|f| (f.run_id.clone(), f.definition.clone())) + .collect::>(), + copy.latest_runs + .iter() + .filter(|r| r.node_name == NODE) + .map(|r| r.run_id.clone()) + .collect::>(), + copy.observations + .iter() + .filter(|o| o.observation_id.starts_with("copy-")) + .map(|o| (o.kind.clone(), o.value.clone(), o.run_id.clone())) + .collect::>(), + copy.relations + .iter() + .filter(|r| r.name == NODE) + .map(|r| r.columns.len()) + .collect::>(), + ) + }; + let unsynced = copy().await; + assert_eq!(unsynced.synced_at, None, "nothing synced cleanly yet"); + assert_eq!( + ours(&unsynced), + ( + vec![("copy-run-2".into(), "SELECT 2 AS id".into())], + vec!["copy-run-2".into()], + vec![ + ("cursor_max".into(), "2".into(), "copy-run-2".into()), + ("source_columns".into(), "{}".into(), "copy-run-1".into()), + ], + vec![1], + ), + "newest fingerprint with its definition, newest succeeded run, newest observation per kind" + ); + + let fingerprints_clean = sync_report(preview_session, "fingerprints", None); + for report in [ + sync_report(preview_session, "catalog", None), + fingerprints_clean.clone(), + sync_report(preview_session, "catalog", None), + sync_report(preview_session, "consumption", Some("read")), + ] { + assert!( + outbound + .report_health(&report) + .await + .expect("report health") + ); + } + assert_eq!( + copy().await.synced_at, + None, + "consumption never synced cleanly" + ); + for report in [ + sync_report(preview_session, "consumption", None), + sync_report(preview_session, "fingerprints", Some("read")), + sync_report(preview_session, "observations", Some("read")), + ] { + assert!( + outbound + .report_health(&report) + .await + .expect("report health") + ); + } + assert_eq!( + copy().await.synced_at, + None, + "every table of main synced, but products builds into reporting, which never synced" + ); + for table in ["catalog", "fingerprints", "consumption"] { + assert!( + outbound + .report_health(&schema_sync_report( + preview_session, + ("reporting", table), + None + )) + .await + .expect("report health") + ); + } + let health_url = format!( + "{}/health", + metadata.split("/targets/").next().expect("location URL") + ); + let observations: Vec = client + .get(&health_url) + .bearer_auth(bearer) + .send() + .await + .expect("health") + .json() + .await + .expect("health reply"); + let fingerprints_at = observations + .iter() + .find(|observation| observation["report"]["event_id"] == json!(fingerprints_clean.event_id)) + .and_then(|observation| observation["received_at"].as_str()) + .map(|at| at.parse::>().expect("time")); + assert_eq!( + copy().await.synced_at, + fingerprints_at, + "the stalest required table's newest clean sync; failures and observations do not count" + ); + + let development = outbound + .preview_copy( + "development", + &PreviewCopyRead { + session_id, + generation: 1, + }, + ) + .await + .expect("development copy"); + assert_eq!( + ours(&development), + (vec![], vec![], vec![], vec![]), + "another target's mirror never leaks" + ); + + let refused = |session: Uuid, generation: i64, target: &str| { + let endpoint = format!( + "{}/preview-copy", + metadata.replace("/preview/", &format!("/{target}/")) + ); + let request = client + .post(endpoint) + .bearer_auth(bearer) + .json(&json!({"session_id": session, "generation": generation})); + async move { request.send().await.expect("refused copy").status() } + }; + assert_eq!( + refused(preview_session, 2, "preview").await, + StatusCode::CONFLICT + ); + assert_eq!( + refused(session_id, 1, "preview").await, + StatusCode::BAD_REQUEST + ); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 3f45271352..15c8a81788 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -21,6 +21,9 @@ use crate::bindings::_helpers::build_command::planning::{ use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::observations::source_capture::{ + arm_source_capture, take_source_capture, +}; use crate::bindings::_helpers::records::fields::field as native_field; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { @@ -56,8 +59,16 @@ fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) - within(&cache_scope, |_| run_cached_build(context, providers)) } -/// Plan and run the build, sharing one relation metadata cache from planning to the run's end. +/// Plan and run the build, capturing the source columns planning describes for its observations. fn run_cached_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { + arm_source_capture(); + let built: PyResult = run_captured_build(context, providers); + let _ = take_source_capture(); + built +} + +/// Plan and run the build, sharing one relation metadata cache from planning to the run's end. +fn run_captured_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; if let Some(worker) = &context.worker { worker.enforce_plan_writers(&pipeline)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 824b6119f8..82ce827200 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -41,6 +41,7 @@ use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::consumption_state::recorder_callbacks; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::results::ReadResult; +use crate::bindings::_helpers::observations::publication::append_build_observations; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::fields::field as native_field; @@ -483,6 +484,15 @@ pub(crate) fn execute_build_plan<'py>( &connection_config, &native_field(&project, "run_id")?, )?; + for warning in append_build_observations( + (&project, &successful_models), + (&adapter, &connection_config), + &native_field(&project, "run_id")? + .str()? + .extract::()?, + ) { + emit(&context.stream, &format!("warning: {warning}\n"))?; + } lifecycle.finalize(py)?; Ok(Outcome { result: native_result, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 21e77b8f61..9367712a19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -29,6 +29,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { plan_command::registration::register(module)?; freshness::registration::register(module)?; crate::bindings::_helpers::native_host::consumption_state::register(module)?; + crate::bindings::_helpers::native_host::observations_state::register(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; execution_history::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs index ea38f66ca7..2fcc1fd162 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs @@ -52,8 +52,9 @@ use crate::bindings::_helpers::execution_history::statements::{ column, execute, execute_plain, execute_query, fetch_one, tuple_of, }; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, sqlite_params}; -use crate::bindings::_helpers::native_host::sqlite_cursor::SqliteCursor; +use crate::bindings::_helpers::native_host::sqlite_connection::{ + SqliteConnection, SqliteCursor, sqlite_params, +}; use crate::bindings::_helpers::native_threads::reentrant_lock::ReentrantLock; use crate::bindings::_helpers::records::fields::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs index cbb9f83ccf..a595566ade 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs @@ -5,8 +5,9 @@ use pyo3::types::PyTuple; use sqlbuild_state::execution_history::models::{Query, SqlParam}; use crate::bindings::_helpers::execution_history::conversion::python_params; -use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, sqlite_params}; -use crate::bindings::_helpers::native_host::sqlite_cursor::{SqliteCursor, SqliteRow}; +use crate::bindings::_helpers::native_host::sqlite_connection::{ + SqliteConnection, SqliteCursor, SqliteRow, sqlite_params, +}; /// `target.execute(sql, params)`. pub(crate) fn execute<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 13a47eadb0..c7ecfe0ed6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -20,9 +20,10 @@ use sqlbuild_core::command_output::constants::{ use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, - NODE_RESULTS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, - PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, - STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, + NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, + PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SETTINGS_SECTION, + SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, + ZERO_DAY_DURATION, }; use sqlbuild_core::diagnostic_routing::constants::{ SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD, @@ -93,6 +94,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("NODE_RESULTS_TABLE_NAME", NODE_RESULTS_TABLE_NAME), ("AUDIT_RESULTS_TABLE_NAME", AUDIT_RESULTS_TABLE_NAME), ("JANITOR_EVENTS_TABLE_NAME", JANITOR_EVENTS_TABLE_NAME), + ("OBSERVATIONS_TABLE_NAME", OBSERVATIONS_TABLE_NAME), ("COLUMN_NODE_TYPE", COLUMN_NODE_TYPE), ("COLUMN_NODE_NAME", COLUMN_NODE_NAME), ("COLUMN_TARGET_DATABASE", COLUMN_TARGET_DATABASE), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index 33854ecfda..687ba18014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -22,6 +22,7 @@ use crate::bindings::_helpers::freshness::observation::{ use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::freshness::support::text; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::WarehouseAccessRefusedError; use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -32,9 +33,14 @@ use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; const ADAPTER_STRATEGY: &str = "adapter"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; +/// Why a plan preview leaves a source's freshness unknown; its plan lists freshness as unknown. +pub(crate) const PREVIEW_MESSAGE: &str = "a plan preview does not observe source freshness"; -/// The message of an `AdapterUserError` or a source freshness observation error, if it is one. +/// The message of an adapter, freshness observation or preview access error, if it is one. fn handled_message<'py>(py: Python<'py>, error: &PyErr) -> PyResult>> { + if error.is_instance_of::(py) { + return Ok(Some(text(py, PREVIEW_MESSAGE))); + } let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); if value.is_instance(&host::entry(py, AdapterHost::AdapterUserError)?)? { return native_field(&value, "message").map(Some); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index fceeda9567..9378dea579 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -25,7 +25,9 @@ const EXECUTION_LOGGER: &str = "sqlbuild.execution"; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; -use sqlbuild_core::constants::{CONSUMPTION_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_core::constants::{ + CONSUMPTION_TABLE_NAME, OBSERVATIONS_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, +}; /// The adapter and its open connection. pub(crate) struct Warehouse<'py> { @@ -475,7 +477,7 @@ pub(crate) fn gather_warehouse_facts<'py>( }) } -/// Fingerprint, source freshness and consumption history prune candidates for target schemas. +/// Fingerprint, source freshness, consumption and observation history prune candidates for target schemas. pub(crate) fn direct_state_prune_candidates<'py>( warehouse: &Warehouse<'py>, target_schemas: &[SchemaKey], @@ -495,6 +497,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( SOURCE_FRESHNESS_TABLE_NAME, )); locations.push((database.clone(), schema.clone(), CONSUMPTION_TABLE_NAME)); + locations.push((database.clone(), schema.clone(), OBSERVATIONS_TABLE_NAME)); } let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut candidates: Vec> = Vec::new(); @@ -512,6 +515,10 @@ pub(crate) fn direct_state_prune_candidates<'py>( CONSUMPTION_TABLE_NAME, "render_prune_consumption_history_sql", ), + ( + OBSERVATIONS_TABLE_NAME, + "render_prune_observations_history_sql", + ), ] { if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 1b33b89fce..213941ba16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -67,6 +67,7 @@ pub(crate) mod microbatches; pub(crate) mod model_config; pub(crate) mod native_host; pub(crate) mod native_threads; +pub(crate) mod observations; pub(crate) mod output_capture; pub(crate) mod plan_command; pub(crate) mod plan_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index 4586fd655d..3677e129f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; /// Rows per `INSERT`; SQL Server accepts at most 1,000 rows in one `VALUES` list. -const ROWS_PER_STATEMENT: usize = 500; +pub(crate) const ROWS_PER_STATEMENT: usize = 500; /// Warehouse timestamp texts with a UTC offset, as drivers render aware values. const OFFSET_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f%:z", "%Y-%m-%d %H:%M:%S%.f%:z"]; /// Warehouse timestamp texts without an offset, which the consumption table stores as UTC. @@ -46,7 +46,7 @@ impl Location { } } - fn keywords<'py>(&self, py: Python<'py>) -> Vec<(&'static str, Bound<'py, PyAny>)> { + pub(crate) fn keywords<'py>(&self, py: Python<'py>) -> Vec<(&'static str, Bound<'py, PyAny>)> { vec![ ( "database", @@ -106,28 +106,38 @@ fn build_consumption_create_table_sql( render_qualified_name: &Bound<'_, PyAny>, render_framework_type: &Bound<'_, PyAny>, ) -> PyResult { - let py: Python<'_> = database.py(); - let Some(table) = state_table_name( - render_qualified_name, + fixed_state_table_sql( (database, schema), - CONSUMPTION_TABLE_NAME, - )? + (render_qualified_name, render_framework_type), + ( + CONSUMPTION_TABLE_NAME, + &CONSUMPTION_COLUMN_TYPES, + &NULLABLE_COLUMNS, + ), + ) +} + +/// Create-if-missing DDL for a state table with fixed columns, typed by the adapter. +pub(crate) fn fixed_state_table_sql( + (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (render_qualified_name, render_framework_type): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (table_name, column_types, nullable): (&str, &[(&str, &str)], &[&str]), +) -> PyResult { + let py: Python<'_> = database.py(); + let Some(table) = state_table_name(render_qualified_name, (database, schema), table_name)? else { return Ok(String::new()); }; - let columns: Bound<'_, PyAny> = PyTuple::new( - py, - CONSUMPTION_COLUMN_TYPES.iter().map(|(column, _)| *column), - )? - .into_any(); + let columns: Bound<'_, PyAny> = + PyTuple::new(py, column_types.iter().map(|(column, _)| *column))?.into_any(); let types: Bound<'_, PyDict> = PyDict::new(py); - for (column, declared) in CONSUMPTION_COLUMN_TYPES { + for (column, declared) in column_types { types.set_item(column, declared)?; } - let required: Vec<&str> = CONSUMPTION_COLUMN_TYPES + let required: Vec<&str> = column_types .iter() .map(|(column, _)| *column) - .filter(|column| !NULLABLE_COLUMNS.contains(column)) + .filter(|column| !nullable.contains(column)) .collect(); state_table_create_sql( &table, @@ -158,7 +168,7 @@ fn build_consumption_prune_sql( .unwrap_or_default()) } -fn execute<'py>( +pub(crate) fn execute<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, sql: &str, @@ -314,7 +324,7 @@ fn read_row(row: &Bound<'_, PyAny>) -> PyResult { }) } -fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { +pub(crate) fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { let value: Bound<'_, PyAny> = value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)); render_state_sql_literal(py, &value, STRING_STATE_VALUE_TYPE, None) @@ -392,7 +402,7 @@ impl ConsumptionConnection { } } - fn connected<'py>(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn connected<'py>(&self, py: Python<'py>) -> PyResult> { let mut open = self .connection .lock() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 3fb073463c..59eca3d642 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod interrupt; pub(crate) mod model_lookup; +pub(crate) mod observations_state; pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; pub(crate) mod orchestration_sensors; @@ -14,8 +15,8 @@ pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; -pub(crate) mod sqlite_cursor; pub(crate) mod state_column_evolution; +pub(crate) mod synced_copy; pub(crate) mod warehouse_catalog; pub(crate) mod warehouse_metadata; pub(crate) mod warehouse_statistics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs new file mode 100644 index 0000000000..b049b9f4c0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs @@ -0,0 +1,132 @@ +//! DDL and pruning of the display-only observations table through the adapter. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::constants::OBSERVATIONS_TABLE_NAME; +use sqlbuild_state::observations::constants::{ + NULLABLE_OBSERVATION_COLUMNS, OBSERVATION_COLUMN_TYPES, +}; +use sqlbuild_state::observations::main::prune_observations_sql::prune_observations_sql; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; +use uuid::Uuid; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::native_host::consumption_state::{ + Location, ROWS_PER_STATEMENT, execute, fixed_state_table_sql, literal, state_table_name, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Create-if-missing DDL for the observations table, with adapter-rendered column types. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] +fn build_observations_create_table_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + render_framework_type: &Bound<'_, PyAny>, +) -> PyResult { + fixed_state_table_sql( + (database, schema), + (render_qualified_name, render_framework_type), + ( + OBSERVATIONS_TABLE_NAME, + &OBSERVATION_COLUMN_TYPES, + &NULLABLE_OBSERVATION_COLUMNS, + ), + ) +} + +/// SQL deleting observations beyond the newest `retain_versions` per kind and node. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] +fn build_observations_prune_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + retain_versions: i64, +) -> PyResult { + Ok(state_table_name( + render_qualified_name, + (database, schema), + OBSERVATIONS_TABLE_NAME, + )? + .map(|table| prune_observations_sql(&table, retain_versions)) + .unwrap_or_default()) +} + +/// One observation to append; the warehouse clock fills `observed_at`. +pub(crate) struct ObservationAppend { + pub(crate) kind: &'static str, + pub(crate) node: String, + pub(crate) cursor_type: Option, + pub(crate) value: String, + pub(crate) run_id: String, +} + +/// Create the schema and observations table in `location` if missing, then append `rows`. +pub(crate) fn append_observations( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, + rows: &[ObservationAppend], +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + let keywords = location.keywords(py); + let Some(table) = state_table_name( + &native_field(adapter, "render_qualified_name")?, + (&keywords[0].1, &keywords[1].1), + OBSERVATIONS_TABLE_NAME, + )? + else { + return Ok(()); + }; + for statement in + call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? + { + execute(adapter, connection, &statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(adapter, "render_create_observations_table_sql")?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + execute(adapter, connection, &create)?; + } + let observed_at: String = + call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; + let columns: Vec<&str> = OBSERVATION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + for chunk in rows.chunks(ROWS_PER_STATEMENT) { + let cells: Vec> = chunk + .iter() + .map(|row| { + Ok(vec![ + literal(py, Some(&Uuid::now_v7().to_string()))?, + literal(py, Some(row.kind))?, + literal(py, Some(&row.node))?, + literal(py, row.cursor_type.as_deref())?, + literal(py, Some(&row.value))?, + literal(py, Some(&row.run_id))?, + observed_at.clone(), + ]) + }) + .collect::>()?; + execute( + adapter, + connection, + &values_insert_sql(&table, &columns, &cells), + )?; + } + Ok(()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_observations_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_observations_prune_sql, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs index 95ebe934fd..0439de3ce2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs @@ -6,11 +6,16 @@ use pyo3::wrap_pyfunction; use sqlbuild_host::entries::main::host_entry_labels::host_entry_labels; use crate::bindings::_helpers::native_host::orchestration_declarations::declaration_site; +use crate::bindings::_helpers::native_host::synced_copy::WarehouseAccessRefusedError; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add( "HOST_ENTRIES", PyTuple::new(module.py(), host_entry_labels())?, )?; + module.add( + "WarehouseAccessRefusedError", + module.py().get_type::(), + )?; module.add_function(wrap_pyfunction!(declaration_site, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs index c82f67e182..d2132bea74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs @@ -1,18 +1,19 @@ -//! Rust-owned SQLite connection; Python views exist only for connection overrides. +//! Rust-owned SQLite connection and its rows; Python views exist only for connection overrides. use std::collections::VecDeque; use std::path::PathBuf; use std::sync::{Arc, Mutex, PoisonError}; -use pyo3::exceptions::PyValueError; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::{PyIndexError, PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; +use pyo3::types::PyBytes; use rusqlite::types::Value; use rusqlite::{Connection, Error, ErrorCode, OpenFlags, params_from_iter}; use sqlbuild_state::execution_history::constants::MEMORY_PATH; use sqlbuild_state::execution_history::models::SqlParam; use crate::bindings::_helpers::boundary::python_calls::imported; -use crate::bindings::_helpers::native_host::sqlite_cursor::{SqliteCursor, SqliteRow}; /// Preserve DB-API exception classes at the public storage-error boundary. fn database_error(py: Python<'_>, error: Error) -> PyErr { @@ -187,3 +188,92 @@ impl SqliteConnection { Ok(()) } } + +#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteRow")] +pub(crate) struct SqliteRow { + pub(crate) columns: Arc>, + pub(crate) values: Vec, +} + +impl SqliteRow { + pub(crate) fn value<'py>(&self, py: Python<'py>, index: usize) -> PyResult> { + match self.values.get(index) { + Some(Value::Null) => Ok(py.None().into_bound(py)), + Some(Value::Integer(value)) => value.into_bound_py_any(py), + Some(Value::Real(value)) => value.into_bound_py_any(py), + Some(Value::Text(value)) => value.into_bound_py_any(py), + Some(Value::Blob(value)) => Ok(PyBytes::new(py, value).into_any()), + None => Err(PyIndexError::new_err("tuple index out of range")), + } + } + + pub(crate) fn column<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let index: usize = self + .columns + .iter() + .position(|column| column.eq_ignore_ascii_case(name)) + .ok_or_else(|| PyIndexError::new_err("No item with that key"))?; + self.value(py, index) + } +} + +#[pymethods] +impl SqliteRow { + fn __getitem__<'py>(&self, key: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(index) = key.extract::() { + let index = if index < 0 { + self.values.len().checked_add_signed(index) + } else { + Some(index as usize) + }; + return self.value(key.py(), index.unwrap_or(usize::MAX)); + } + if let Ok(name) = key.extract::() { + return self.column(key.py(), &name); + } + Err(PyTypeError::new_err("Index must be int or string")) + } +} + +#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteCursor")] +pub(crate) struct SqliteCursor { + pub(crate) rows: Mutex>, + pub(crate) last_row_id: i64, +} + +impl SqliteCursor { + pub(crate) fn next_row(&self) -> Option { + self.rows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .pop_front() + } +} + +#[pymethods] +impl SqliteCursor { + fn __iter__(slf: Bound<'_, Self>) -> Bound<'_, Self> { + slf + } + + fn __next__(&self) -> Option { + self.next_row() + } + + fn fetchone(&self) -> Option { + self.next_row() + } + + fn fetchall(&self) -> Vec { + self.rows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .drain(..) + .collect() + } + + #[getter] + fn lastrowid(&self) -> i64 { + self.last_row_id + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_cursor.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_cursor.rs deleted file mode 100644 index 2a9796015c..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_cursor.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Native SQLite rows, with a DB-API view for the existing connection override boundary. - -use std::collections::VecDeque; -use std::sync::{Arc, Mutex, PoisonError}; - -use pyo3::IntoPyObjectExt; -use pyo3::exceptions::{PyIndexError, PyTypeError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; -use pyo3::types::PyBytes; -use rusqlite::types::Value; - -#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteRow")] -pub(crate) struct SqliteRow { - pub(crate) columns: Arc>, - pub(crate) values: Vec, -} - -impl SqliteRow { - pub(crate) fn value<'py>(&self, py: Python<'py>, index: usize) -> PyResult> { - match self.values.get(index) { - Some(Value::Null) => Ok(py.None().into_bound(py)), - Some(Value::Integer(value)) => value.into_bound_py_any(py), - Some(Value::Real(value)) => value.into_bound_py_any(py), - Some(Value::Text(value)) => value.into_bound_py_any(py), - Some(Value::Blob(value)) => Ok(PyBytes::new(py, value).into_any()), - None => Err(PyIndexError::new_err("tuple index out of range")), - } - } - - pub(crate) fn column<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { - let index: usize = self - .columns - .iter() - .position(|column| column.eq_ignore_ascii_case(name)) - .ok_or_else(|| PyIndexError::new_err("No item with that key"))?; - self.value(py, index) - } -} - -#[pymethods] -impl SqliteRow { - fn __getitem__<'py>(&self, key: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(index) = key.extract::() { - let index = if index < 0 { - self.values.len().checked_add_signed(index) - } else { - Some(index as usize) - }; - return self.value(key.py(), index.unwrap_or(usize::MAX)); - } - if let Ok(name) = key.extract::() { - return self.column(key.py(), &name); - } - Err(PyTypeError::new_err("Index must be int or string")) - } -} - -#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteCursor")] -pub(crate) struct SqliteCursor { - pub(crate) rows: Mutex>, - pub(crate) last_row_id: i64, -} - -impl SqliteCursor { - pub(crate) fn next_row(&self) -> Option { - self.rows - .lock() - .unwrap_or_else(PoisonError::into_inner) - .pop_front() - } -} - -#[pymethods] -impl SqliteCursor { - fn __iter__(slf: Bound<'_, Self>) -> Bound<'_, Self> { - slf - } - - fn __next__(&self) -> Option { - self.next_row() - } - - fn fetchone(&self) -> Option { - self.next_row() - } - - fn fetchall(&self) -> Vec { - self.rows - .lock() - .unwrap_or_else(PoisonError::into_inner) - .drain(..) - .collect() - } - - #[getter] - fn lastrowid(&self) -> i64 { - self.last_row_id - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs new file mode 100644 index 0000000000..63867edcdc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs @@ -0,0 +1,506 @@ +//! A plan preview's adapter: warehouse reads come from the location's mirrored copy, nothing connects. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Mutex, PoisonError}; + +use chrono::SecondsFormat; +use pyo3::IntoPyObject; +use pyo3::create_exception; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use serde::Deserialize; +use sqlbuild_analysis::fingerprint_storage::constants::FINGERPRINT_COLUMNS; +use sqlbuild_analysis::fingerprint_storage::main::read_latest_sql::read_latest_sql; +use sqlbuild_cache::digest::main::base64_text::base64_text; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; +use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, STATE_TABLE_NAMES}; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_orchestration::warehouse_metadata::models::{PreviewCopy, WarehouseCatalogRecord}; +use sqlbuild_state::execution_history::constants::MEMORY_PATH; +use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; + +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, SqliteCursor}; +use crate::bindings::_helpers::records::fields::field as native_field; + +create_exception!( + sqlbuild._native, + WarehouseAccessRefusedError, + PyRuntimeError, + "Raised when a plan preview reaches past the synced copy for the warehouse." +); + +/// Adapter methods that only describe the dialect, never reaching the warehouse. +const PASSTHROUGH_NAMES: [&str; 12] = [ + "_render_source_relation_cast_subquery_with_columns", + "adapter_name", + "default_database", + "default_schema", + "default_table_promotion_mode", + "expression_inference_profile", + "metadata_inspection_concurrency", + "python_functions_inherit_default_namespace", + "requires_derived_table_aliases", + "sql_analysis_dialect", + "sql_analysis_dialect_name", + "star_exclude_keyword", +]; +/// Rendering and capability methods pass through; every other unknown attribute is refused. +const PASSTHROUGH_PREFIXES: [&str; 2] = ["render_", "supports_"]; +/// Local table holding the copy's fingerprints; previews read it through per-location views. +const FINGERPRINT_STORE: &str = "__sqlbuild_synced_fingerprints"; +const FINGERPRINT_STORE_COLUMNS: [&str; 2] = ["state_database", "state_schema"]; +const MODEL_NODE_TYPE: &str = "model"; + +/// Items a preview reports as known only once the run plans live. +pub(crate) const PREVIEW_UNKNOWNS: [&str; 4] = [ + "source freshness", + "retention", + "column rename support", + "sources outside mirrored schemas", +]; + +/// A source's described columns, keyed to the exact SQL that was described. +#[derive(Deserialize)] +struct DescribedColumns { + sql_hash: String, + columns: Vec, +} + +fn refused(access: &str) -> pyo3::PyErr { + WarehouseAccessRefusedError::new_err(format!( + "plan previews read only the synced copy; refused warehouse access: {access}" + )) +} + +fn same(left: Option<&str>, right: Option<&str>) -> bool { + left.unwrap_or_default().to_lowercase() == right.unwrap_or_default().to_lowercase() +} + +/// Whether `wanted` is unset or holds `value`, compared as the warehouse folds case. +fn any_same(wanted: Option<&[String]>, value: &str) -> bool { + let Some(wanted) = wanted else { + return true; + }; + for item in wanted { + if same(Some(value), Some(item)) { + return true; + } + } + false +} + +fn literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +/// A connection that refuses every use; like the adapter, it lives in the adapter contract's module. +#[pyclass( + frozen, + module = "sqlbuild.adapter.contract", + name = "SyncedCopyConnection" +)] +pub(crate) struct SyncedCopyConnection; + +#[pymethods] +impl SyncedCopyConnection { + fn __getattr__(&self, name: &str) -> PyResult<()> { + Err(refused(&format!("connection.{name}"))) + } +} + +/// Answers listing, column and fingerprint reads from the copy; dialect methods use `adapter`. +#[pyclass( + frozen, + module = "sqlbuild.adapter.contract", + name = "SyncedCopyAdapter" +)] +pub(crate) struct SyncedCopyAdapter { + adapter: Py, + copy: PreviewCopy, + store: SqliteConnection, + views: Mutex, String)>>, + unknowns: Mutex>, + windows: Mutex>, +} + +impl SyncedCopyAdapter { + /// The facade over `adapter` answering from the copy written at `path`. + pub(crate) fn load(py: Python<'_>, adapter: Bound<'_, PyAny>, path: &str) -> PyResult { + let text: String = std::fs::read_to_string(path)?; + let copy: PreviewCopy = serde_json::from_str(&text) + .map_err(|error| PyRuntimeError::new_err(format!("invalid synced copy: {error}")))?; + let store: SqliteConnection = SqliteConnection::open(py, MEMORY_PATH)?; + let columns: Vec<&str> = FINGERPRINT_STORE_COLUMNS + .iter() + .chain(FINGERPRINT_COLUMNS.iter()) + .copied() + .collect(); + store.execute_native( + py, + &format!("CREATE TABLE {FINGERPRINT_STORE} ({})", columns.join(", ")), + &[], + )?; + let placeholders: String = vec!["?"; columns.len()].join(", "); + for record in ©.fingerprints { + let values: [Option; 14] = [ + record.state_database.clone(), + Some(record.state_schema.clone()), + Some(record.node_type.clone()), + Some(record.node_name.clone()), + record.target_database.clone(), + record.target_schema.clone(), + record.target_name.clone(), + Some(record.run_id.clone()), + Some(record.definition_hash.clone()), + Some(record.version_hash.clone()), + Some(record.schema_fingerprint.clone()), + Some(base64_text(record.definition.as_bytes())), + Some(base64_text(record.metadata_json.as_bytes())), + Some(record.ts.to_rfc3339_opts(SecondsFormat::AutoSi, true)), + ]; + let values: Vec = values + .into_iter() + .map(|value| { + value.map_or(rusqlite::types::Value::Null, rusqlite::types::Value::Text) + }) + .collect(); + store.execute_native( + py, + &format!("INSERT INTO {FINGERPRINT_STORE} VALUES ({placeholders})"), + &values, + )?; + } + Ok(Self { + adapter: adapter.unbind(), + copy, + store, + views: Mutex::new(Vec::new()), + unknowns: Mutex::new(BTreeSet::new()), + windows: Mutex::new(BTreeMap::new()), + }) + } + + /// When the copy was synced, as ISO 8601 text; `None` before every required table synced. + pub(crate) fn synced_at(&self) -> Option { + self.copy + .synced_at + .map(|at| at.to_rfc3339_opts(SecondsFormat::AutoSi, true)) + } + + /// What the preview could not know, in a stable order. + pub(crate) fn unknowns(&self) -> Vec { + let mut unknowns: Vec = PREVIEW_UNKNOWNS + .iter() + .map(|item| (*item).to_owned()) + .collect(); + unknowns.extend( + self.unknowns + .lock() + .unwrap_or_else(PoisonError::into_inner) + .iter() + .cloned(), + ); + unknowns + } + + /// Note one item the preview reports as unknown until the run plans live. + pub(crate) fn mark_unknown(&self, item: String) { + self.unknowns + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(item); + } + + /// Note that `model` reads from `start` up to the latest data when the run starts. + pub(crate) fn mark_window(&self, model: &str, start: &str) { + self.windows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(model.to_owned(), start.to_owned()); + } + + /// Each model's window start, read up to the latest data when the run starts. + pub(crate) fn windows(&self) -> BTreeMap { + self.windows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// The model's cursor `MAX` text, only when observed by its newest successful build. + pub(crate) fn target_max(&self, model: &str) -> Option<&str> { + for run in &self.copy.latest_runs { + if run.node_type != MODEL_NODE_TYPE || run.node_name != model { + continue; + } + for observation in &self.copy.observations { + if observation.kind == CURSOR_MAX_KIND + && observation.node == model + && observation.run_id == run.run_id + && observation.state_schema == run.state_schema + && observation.state_database == run.state_database + { + return Some(observation.value.as_str()); + } + } + } + None + } + + /// The source's columns as last described, only when it described exactly `sql`. + pub(crate) fn described_columns( + &self, + source: &str, + sql: &str, + ) -> PyResult>> { + let hash: String = sha256_hex(sql.as_bytes()); + for observation in &self.copy.observations { + if observation.kind != SOURCE_COLUMNS_KIND || observation.node != source { + continue; + } + let described: DescribedColumns = + serde_json::from_str(&observation.value).map_err(|error| { + PyRuntimeError::new_err(format!( + "invalid source_columns observation for {source}: {error}" + )) + })?; + if described.sql_hash == hash { + return Ok(Some(described.columns)); + } + } + Ok(None) + } + + /// The relations a listing call named by its `database`, `schemas` and optional `names`. + fn listed_by<'py>( + &self, + py: Python<'py>, + function: &str, + options: Option>, + ) -> PyResult> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + function, + options, + (&["connection", "database", "schemas"], &["names"]), + )?; + let database: Option = arguments.value("database")?.extract()?; + let schemas: Option> = arguments.value("schemas")?.extract()?; + let names: Option> = arguments.value("names")?.extract()?; + Ok(self.listed(database.as_deref(), schemas.as_deref(), names.as_deref())) + } + + fn listed( + &self, + database: Option<&str>, + schemas: Option<&[String]>, + names: Option<&[String]>, + ) -> Vec<&WarehouseCatalogRecord> { + let mut listed: Vec<&WarehouseCatalogRecord> = Vec::new(); + for record in &self.copy.relations { + if (STATE_TABLE_NAMES.contains(&record.name.as_str()) + && record.name != FINGERPRINT_TABLE_NAME) + || !(database.is_none() + || record.state_database.is_none() + || same(record.state_database.as_deref(), database)) + || !any_same(schemas, &record.state_schema) + || !any_same(names, &record.name) + { + continue; + } + listed.push(record); + } + listed + } + + fn relation_info<'py>( + &self, + py: Python<'py>, + record: &WarehouseCatalogRecord, + ) -> PyResult> { + call_with( + &host::entry(py, AdapterHost::RelationInfo)?, + &[ + ( + "database", + record + .state_database + .as_deref() + .into_pyobject(py)? + .into_any(), + ), + ("schema", PyString::new(py, &record.state_schema).into_any()), + ("name", PyString::new(py, &record.name).into_any()), + ( + "relation_type", + PyString::new(py, &record.relation_type).into_any(), + ), + ], + ) + } + + fn columns<'py>( + &self, + py: Python<'py>, + record: &WarehouseCatalogRecord, + ) -> PyResult> { + let column_info: Bound<'py, PyAny> = host::entry(py, AdapterHost::ColumnInfo)?; + let mut columns: Vec> = Vec::with_capacity(record.columns.len()); + for column in &record.columns { + columns.push(call_with( + &column_info, + &[ + ("name", PyString::new(py, &column.name).into_any()), + ("type", PyString::new(py, &column.data_type).into_any()), + ], + )?); + } + PyTuple::new(py, columns) + } +} + +#[pymethods] +impl SyncedCopyAdapter { + #[pyo3(signature = (connection_config = None))] + fn connect(&self, connection_config: Option>) -> SyncedCopyConnection { + drop(connection_config); + SyncedCopyConnection + } + + #[pyo3(signature = (connection = None))] + fn close(&self, connection: Option>) { + drop(connection); + } + + #[pyo3(name = "list_relations", signature = (**options))] + fn relation_listing<'py>( + &self, + py: Python<'py>, + options: Option>, + ) -> PyResult> { + let relations: Vec> = self + .listed_by(py, "list_relations", options)? + .into_iter() + .map(|record| self.relation_info(py, record)) + .collect::>()?; + PyTuple::new(py, relations) + } + + #[pyo3(name = "get_all_columns", signature = (**options))] + fn all_columns<'py>( + &self, + py: Python<'py>, + options: Option>, + ) -> PyResult> { + let columns: Bound<'py, PyDict> = PyDict::new(py); + for record in self.listed_by(py, "get_all_columns", options)? { + columns.set_item(&record.name, self.columns(py, record)?)?; + } + Ok(columns) + } + + #[pyo3(name = "get_columns_for_relations", signature = (*, connection, relations))] + fn columns_by_relation<'py>( + &self, + py: Python<'py>, + connection: Bound<'py, PyAny>, + relations: Bound<'py, PyAny>, + ) -> PyResult> { + drop(connection); + let found: Bound<'py, PyDict> = PyDict::new(py); + for relation in relations.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + let database: Option = native_field(&relation, "database")?.extract()?; + let schema: Option = native_field(&relation, "schema")?.extract()?; + let name: String = native_field(&relation, "name")?.extract()?; + let schemas: Option> = schema.clone().map(|schema| vec![schema]); + let names: [String; 1] = [name.clone()]; + if let Some(record) = self + .listed(database.as_deref(), schemas.as_deref(), Some(&names)) + .first() + { + found.set_item((database, schema, name), self.columns(py, record)?)?; + } + } + Ok(found) + } + + /// The latest-row read over this location's copied fingerprints, through a local view. + #[pyo3(signature = (*, database, schema))] + fn render_read_latest_fingerprints_sql( + &self, + py: Python<'_>, + database: Option, + schema: String, + ) -> PyResult { + let mut views = self.views.lock().unwrap_or_else(PoisonError::into_inner); + let key: (Option, String) = (database.clone(), schema.clone()); + let position: usize = match views.iter().position(|view| *view == key) { + Some(position) => position, + None => { + let mut condition: String = + format!("lower(state_schema) = lower({})", literal(&schema)); + if let Some(database) = &database { + condition.push_str(&format!( + " AND (state_database IS NULL OR lower(state_database) = lower({}))", + literal(database) + )); + } + self.store.execute_native( + py, + &format!( + "CREATE VIEW {FINGERPRINT_STORE}_{} AS SELECT {} FROM {FINGERPRINT_STORE} \ + WHERE {condition}", + views.len(), + FINGERPRINT_COLUMNS.join(", ") + ), + &[], + )?; + views.push(key); + views.len() - 1 + } + }; + Ok(read_latest_sql(&format!("{FINGERPRINT_STORE}_{position}"))) + } + + /// Only reads of the copied fingerprints run; any other statement would reach the warehouse. + #[pyo3(signature = (*, connection, sql))] + fn execute( + &self, + py: Python<'_>, + connection: Bound<'_, PyAny>, + sql: &str, + ) -> PyResult { + drop(connection); + if !sql.contains(&format!("{FINGERPRINT_STORE}_")) { + return Err(refused(sql.trim().lines().next().unwrap_or_default())); + } + self.store.execute_native(py, sql, &[]) + } + + #[pyo3(signature = (*, connection, sql))] + fn query_column_names(&self, connection: Bound<'_, PyAny>, sql: &str) -> PyResult<()> { + drop(connection); + Err(refused(&format!( + "DESCRIBE {}", + sql.trim().lines().next().unwrap_or_default() + ))) + } + + /// Values and dialect methods come from the real adapter; any other method is refused. + fn __getattr__<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let value: Bound<'py, PyAny> = native_field(self.adapter.bind(py), name)?; + if !value.is_callable() + || PASSTHROUGH_NAMES.contains(&name) + || PASSTHROUGH_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) + { + return Ok(value); + } + Err(refused(&format!("adapter.{name}"))) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index ec29ddfbb0..18b2a1ef0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -11,7 +11,8 @@ use sqlbuild_config::command_connection::constants::{ use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, - NODE_RESULTS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, }; use sqlbuild_core::models::SqlValue; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; @@ -26,6 +27,7 @@ use sqlbuild_state::events::constants::{ use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, }; +use sqlbuild_state::observations::constants::OBSERVATION_COLUMN_TYPES; use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; use sqlbuild_state::state_tables::models::HistoryRead; @@ -148,7 +150,8 @@ pub(crate) fn read_history<'py>( | WarehouseHistoryRead::Migrations(read) | WarehouseHistoryRead::OldNameViews(read) | WarehouseHistoryRead::Microbatches(read) - | WarehouseHistoryRead::ColumnMigrations(read) => &read.location, + | WarehouseHistoryRead::ColumnMigrations(read) + | WarehouseHistoryRead::Observations(read) => &read.location, WarehouseHistoryRead::NodeResults(read) => &read.location, WarehouseHistoryRead::Consumption(read) => &read.location, }; @@ -181,6 +184,7 @@ fn read_page<'py>( let (adapter, connection) = (&session.adapter, &session.connection); let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); let consumption_columns = CONSUMPTION_COLUMN_TYPES.map(|(name, _)| name); + let observation_columns = OBSERVATION_COLUMN_TYPES.map(|(name, _)| name); let event_columns: Vec = match request { WarehouseHistoryRead::AuditResults(_) => py .import("sqlbuild.executor.audit_results.constants")? @@ -294,6 +298,15 @@ fn read_page<'py>( consumption_columns.as_slice(), RECORDED_AT_COLUMN, ), + WarehouseHistoryRead::Observations(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + OBSERVATIONS_TABLE_NAME, + observation_columns.as_slice(), + OBSERVED_AT_COLUMN, + ), }; let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); if limit == 0 || limit > MAX_HISTORY_RECORDS { @@ -406,6 +419,7 @@ fn read_page<'py>( "input_version", RECORDED_AT_COLUMN, ]), + WarehouseHistoryRead::Observations(_) => Some(&["observation_id", OBSERVED_AT_COLUMN]), _ => None, }; let mut cursor_indexes: Vec = columns @@ -564,6 +578,7 @@ fn stream_page<'py>( WarehouseHistoryRecord::ColumnMigration(record) => record.created_at, WarehouseHistoryRecord::NodeResult(record) => record.ts, WarehouseHistoryRecord::Consumption(record) => record.recorded_at, + WarehouseHistoryRecord::Observation(record) => record.observed_at, }; if stream.since.is_none_or(|since| observed >= since) { records.push(record); @@ -635,7 +650,8 @@ fn unchanged_token(request: &WarehouseHistoryRead) -> Option<&str> { | WarehouseHistoryRead::Migrations(read) | WarehouseHistoryRead::OldNameViews(read) | WarehouseHistoryRead::Microbatches(read) - | WarehouseHistoryRead::ColumnMigrations(read) => read.unchanged_token.as_deref(), + | WarehouseHistoryRead::ColumnMigrations(read) + | WarehouseHistoryRead::Observations(read) => read.unchanged_token.as_deref(), WarehouseHistoryRead::NodeResults(read) => read.unchanged_token.as_deref(), WarehouseHistoryRead::Consumption(read) => read.unchanged_token.as_deref(), WarehouseHistoryRead::Catalog(_) | WarehouseHistoryRead::Statistics(_) => None, @@ -694,7 +710,8 @@ fn decode_record( ), WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::Migrations(read) - | WarehouseHistoryRead::ColumnMigrations(read) => { + | WarehouseHistoryRead::ColumnMigrations(read) + | WarehouseHistoryRead::Observations(read) => { return decode_event_record(&read.location, row, (columns, &[timestamp])); } WarehouseHistoryRead::NodeResults(NodeResultsRead { location, .. }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs new file mode 100644 index 0000000000..eac8f7714f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod publication; +pub(crate) mod source_capture; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs new file mode 100644 index 0000000000..1f495e5b93 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs @@ -0,0 +1,217 @@ +//! Record what a build observed, cursor `MAX` values and described source columns, for plan previews. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; +use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; +use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::{model_reads, project_reads}; +use crate::bindings::_helpers::consumption::relation_reads::relation; +use crate::bindings::_helpers::native_host::consumption_state::{ConsumptionConnection, Location}; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::observations_state::{ + ObservationAppend, append_observations, +}; +use crate::bindings::_helpers::observations::source_capture::take_source_capture; +use crate::bindings::_helpers::planner_phases::objects::config_str; +use crate::bindings::_helpers::planner_warehouse::constants::RUN_BOUNDED; +use crate::bindings::_helpers::planner_warehouse::cursors::{adapter_cursor_sql, bound_task}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Dependency kind of a source in a model's `dep_rows`. +const SOURCE_DEPENDENCY: &str = "source"; +/// Most concurrent cursor reads after a build, whatever the adapter allows. +const MAX_OBSERVATION_READS: i64 = 4; + +/// One successful incremental model's cursor read: its name, state location, cursor type and SQL. +struct CursorRead { + model: String, + location: Location, + cursor_type: String, + sql: String, +} + +/// The cursor read of every successful incremental model, rendered as the planner renders it. +fn cursor_reads( + project: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, + successful_models: &[String], +) -> PyResult> { + let succeeded: HashSet<&str> = successful_models.iter().map(String::as_str).collect(); + let mut reads: Vec = Vec::new(); + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let name: String = model_reads::name_object(&model)?.extract()?; + if !succeeded.contains(name.as_str()) + || config_str(&model, "materialized")?.as_deref() != Some("incremental") + { + continue; + } + let Some(cursor_column) = config_str(&model, "cursor")? else { + continue; + }; + let destination: Bound<'_, PyAny> = model_reads::destination(&model)?; + let qualified: Bound<'_, PyAny> = native_field(&destination, "qualified_name")?; + if qualified.is_none() || native_field(&destination, "schema")?.is_none() { + continue; + } + let Some(query) = cursor_query_groups(&[CursorTagRequest { + relation: qualified.extract()?, + cursor_column, + cursor_type: config_str(&model, "cursor_type")?, + min_tag: None, + max_tag: name.clone(), + }]) + .into_iter() + .next() else { + continue; + }; + reads.push(CursorRead { + location: Location::of(&relation(&destination)?), + sql: adapter_cursor_sql(adapter, &query)?, + cursor_type: query.cursor_type, + model: name, + }); + } + Ok(reads) +} + +/// One `source_columns` row per described source, in the state schema of each model reading it. +fn source_column_rows( + project: &Bound<'_, PyAny>, + described: &BTreeMap, + run_id: &str, +) -> PyResult>> { + let mut readers: BTreeMap> = BTreeMap::new(); + if described.is_empty() { + return Ok(BTreeMap::new()); + } + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let destination: Bound<'_, PyAny> = model_reads::destination(&model)?; + if native_field(&destination, "schema")?.is_none() { + continue; + } + let sources: Vec = model_reads::dep_rows(&model)? + .into_iter() + .filter(|(kind, name)| kind == SOURCE_DEPENDENCY && described.contains_key(name)) + .map(|(_, name)| name) + .collect(); + if !sources.is_empty() { + readers + .entry(Location::of(&relation(&destination)?)) + .or_default() + .extend(sources); + } + } + let mut rows: BTreeMap> = BTreeMap::new(); + for (location, sources) in readers { + for source in sources { + let row: ObservationAppend = ObservationAppend { + kind: SOURCE_COLUMNS_KIND, + value: described[&source].clone(), + node: source, + cursor_type: None, + run_id: run_id.to_owned(), + }; + rows.entry(location.clone()).or_default().push(row); + } + } + Ok(rows) +} + +/// Read each cursor `MAX`, then append it with `rows`, by state location; one warning per failure. +fn observe<'py>( + reads: &[CursorRead], + mut rows: BTreeMap>, + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + run_id: &str, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let execute: Bound<'py, PyAny> = native_field(adapter, "execute")?; + let tasks: Vec> = reads + .iter() + .map(|read| bound_task(py, read.sql.clone(), (connection, &execute))) + .collect::>()?; + let concurrency: i64 = native_field(adapter, "metadata_inspection_concurrency")? + .extract::()? + .clamp(1, MAX_OBSERVATION_READS); + let outcomes: Vec> = call_with( + &host::entry(py, RUN_BOUNDED)?, + &[ + ("tasks", pyo3::types::PyTuple::new(py, tasks)?.into_any()), + ("concurrency", concurrency.into_pyobject(py)?.into_any()), + ], + )? + .try_iter()? + .collect::>()?; + let mut warnings: Vec = Vec::new(); + for (read, outcome) in reads.iter().zip(outcomes) { + let error: Bound<'py, PyAny> = outcome.get_item(2)?; + if !error.is_none() { + warnings.push(format!( + "could not read the cursor of model {}: {}; plan previews show it as unknown", + read.model, + error.str()? + )); + continue; + } + let row: Bound<'py, PyAny> = outcome.get_item(0)?; + if row.is_none() || row.get_item(0)?.is_none() { + continue; + } + rows.entry(read.location.clone()) + .or_default() + .push(ObservationAppend { + kind: CURSOR_MAX_KIND, + node: read.model.clone(), + cursor_type: Some(read.cursor_type.clone()), + value: row.get_item(0)?.str()?.extract()?, + run_id: run_id.to_owned(), + }); + } + for (location, rows) in rows { + if let Err(error) = append_observations(adapter, connection, &location, &rows) { + warnings.push(format!( + "could not record observations in schema {}: {error}; plan previews show them as unknown", + location.schema + )); + } + } + Ok(warnings) +} + +/// Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. +pub(crate) fn append_build_observations( + (project, successful_models): (&Bound<'_, PyAny>, &[String]), + (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + run_id: &str, +) -> Vec { + let described: BTreeMap = take_source_capture().into_iter().collect(); + let recorded: PyResult> = (|| { + let reads: Vec = cursor_reads(project, adapter, successful_models)?; + let sources: BTreeMap> = + source_column_rows(project, &described, run_id)?; + if reads.is_empty() && sources.is_empty() { + return Ok(Vec::new()); + } + let py: Python<'_> = adapter.py(); + let connection = ConsumptionConnection::new(adapter, connection_config); + let observed: PyResult> = connection + .connected(py) + .and_then(|handle| observe(&reads, sources, (adapter, &handle), run_id)); + let closed: PyResult<()> = connection.close(py); + let warnings: Vec = observed?; + closed?; + Ok(warnings) + })(); + recorded.unwrap_or_else(|error| { + vec![format!( + "could not record observations: {error}; plan previews show them as unknown" + )] + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs new file mode 100644 index 0000000000..b33192b5fd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs @@ -0,0 +1,47 @@ +//! The expression-source columns a build's planning described, kept for its observations. + +use std::cell::RefCell; + +use serde_json::{Map, Value}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; +use sqlbuild_state::observations::constants::{ + SOURCE_COLUMNS_NAMES_KEY, SOURCE_COLUMNS_SQL_HASH_KEY, +}; + +/// Each described source's name and observation value, in the order planning described them. +pub(crate) type DescribedSources = Vec<(String, String)>; + +thread_local! { + /// Per thread, since a build plans and records on the thread that runs it; `None` when not armed. + static CAPTURED: RefCell> = const { RefCell::new(None) }; +} + +/// Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. +pub(crate) fn arm_source_capture() { + CAPTURED.with(|slot| *slot.borrow_mut() = Some(Vec::new())); +} + +/// Keep one source's described SQL hash and column names when a build armed the capture. +pub(crate) fn capture_source_columns(source: &str, sql: &str, columns: &[String]) { + CAPTURED.with(|slot| { + if let Some(captured) = slot.borrow_mut().as_mut() { + let mut value: Map = Map::new(); + value.insert( + SOURCE_COLUMNS_SQL_HASH_KEY.to_owned(), + Value::String(sha256_hex(sql.as_bytes())), + ); + value.insert( + SOURCE_COLUMNS_NAMES_KEY.to_owned(), + Value::from(columns.to_vec()), + ); + captured.push((source.to_owned(), Value::Object(value).to_string())); + } + }); +} + +/// The captured source columns, disarming the capture. +pub(crate) fn take_source_capture() -> DescribedSources { + CAPTURED + .with(|slot| slot.borrow_mut().take()) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 9ca83a6dce..8baa0f2968 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -5,6 +5,10 @@ use pyo3::prelude::{ }; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use pyo3::wrap_pyfunction; +use sqlbuild_config::command_connection::errors::CommandConnectionError; +use sqlbuild_core::plan_display::constants::{ + PLAN_SYNCED_COPY_ENV, SYNCED_COPY_DEFER_HELP, SYNCED_COPY_DEFER_REFUSAL, SYNCED_COPY_KEY, +}; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::keywords; @@ -16,11 +20,13 @@ use crate::bindings::_helpers::build_progress::progress_reporters::{ use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspection; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; +use crate::bindings::_helpers::command_runtime::runtime_errors::raised; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::plan_output::plan_json::format_plan_json; use crate::bindings::_helpers::plan_output::plan_text::format_plan; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; @@ -67,6 +73,33 @@ struct PlanInvocation<'py> { connection_progress: Bound<'py, PyAny>, planning_progress: Bound<'py, PyAny>, load_sources: Bound<'py, PyAny>, + /// Set for a location's plan preview: the adapter that answers from the synced copy. + synced_copy: Option>, +} + +/// The synced-copy adapter when a location asked for a preview; a preview never defers. +fn synced_copy<'py>( + request: &Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = request.py(); + let Ok(path) = std::env::var(PLAN_SYNCED_COPY_ENV) else { + return Ok(None); + }; + if native_field(request, "defer_to")?.is_truthy()? { + return Err(raised( + py, + CommandConnectionError::User { + code: None, + message: SYNCED_COPY_DEFER_REFUSAL.to_owned(), + help: Some(SYNCED_COPY_DEFER_HELP.to_owned()), + }, + )); + } + Ok(Some(Bound::new( + py, + SyncedCopyAdapter::load(py, adapter, &path)?, + )?)) } fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult> { @@ -135,6 +168,8 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult> = + synced_copy(request, native_field(&context, "adapter")?)?; Ok(PlanInvocation { project_dir, discovered, @@ -144,6 +179,7 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult( py, &[ ("discovered_inputs", invocation.discovered.clone()), - ("adapter", native_field(&invocation.context, "adapter")?), + ( + "adapter", + match &invocation.synced_copy { + Some(facade) => facade.clone().into_any(), + None => native_field(&invocation.context, "adapter")?, + }, + ), ("options", pipeline_options), ("hooks", connection_hooks(invocation)?), ], @@ -248,13 +290,15 @@ fn write_output( let plan: Bound<'_, PyAny> = native_field(pipeline, "plan_output")?; let python_entries: Bound<'_, PyAny> = native_field(pipeline, "python_plan_entries")?; if native_field(request, "json_output")?.is_truthy()? { - let text: String = format_plan_json( - py, - Some(keywords( - py, - &[("plan", plan), ("python_plan_entries", python_entries)], - )?), - )?; + let mut options: Vec<(&str, Bound<'_, PyAny>)> = + vec![("plan", plan), ("python_plan_entries", python_entries)]; + if let Some(facade) = &invocation.synced_copy { + options.push(( + SYNCED_COPY_KEY, + synced_copy_json(py, facade.get())?.into_any(), + )); + } + let text: String = format_plan_json(py, Some(keywords(py, &options)?))?; imported(py, "builtins", "print")?.call1((text,))?; return Ok(()); } @@ -285,7 +329,51 @@ fn write_output( ], )?), )?; - print_line(py, &format!("\n{text}")) + print_line(py, &format!("\n{text}"))?; + if let Some(facade) = &invocation.synced_copy { + print_line(py, &synced_copy_text(facade.get()))?; + } + Ok(()) +} + +/// The plan JSON's account of the synced copy: when it synced, window starts and unknowns. +fn synced_copy_json<'py>( + py: Python<'py>, + facade: &SyncedCopyAdapter, +) -> PyResult> { + let synced: Bound<'py, PyDict> = PyDict::new(py); + synced.set_item("synced_at", facade.synced_at())?; + synced.set_item("cursor_window_starts", facade.windows())?; + synced.set_item("unknown_until_run", facade.unknowns())?; + Ok(synced) +} + +/// The preview footer: when the copy was synced and what stays unknown until the run. +fn synced_copy_text(facade: &SyncedCopyAdapter) -> String { + let mut lines: Vec = vec![format!( + "Planned from the synced copy ({}); the run re-plans live.", + facade.synced_at().map_or_else( + || "not fully synced yet".to_owned(), + |at| format!("synced {at}") + ) + )]; + let windows = facade.windows(); + if !windows.is_empty() { + lines.push("Cursor windows (up to the latest data when the run starts):".to_owned()); + lines.extend( + windows + .iter() + .map(|(model, start)| format!(" - {model}: from {start}")), + ); + } + lines.push("Unknown until run:".to_owned()); + lines.extend( + facade + .unknowns() + .into_iter() + .map(|item| format!(" - {item}")), + ); + lines.join("\n") } /// Run one `sqb plan` request. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 94b43956a1..551852218b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_outputs::json_report::indented_json; @@ -748,7 +749,7 @@ pub(crate) fn format_plan_json( py, "format_plan_json", options, - (&["plan"], &["python_plan_entries"]), + (&["plan"], &["python_plan_entries", SYNCED_COPY_KEY]), )?; let plan: Bound<'_, PyAny> = arguments.value("plan")?; let python_entries: Vec> = if arguments.passed("python_plan_entries")? { @@ -820,6 +821,10 @@ pub(crate) fn format_plan_json( if metadata.is_truthy()? { result.set_item("metadata", metadata)?; } + let synced_copy: Bound<'_, PyAny> = arguments.value(SYNCED_COPY_KEY)?; + if !synced_copy.is_none() { + result.set_item(SYNCED_COPY_KEY, synced_copy)?; + } indented_json(result.as_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index f6b5bf0ede..4378d81db8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::model_config::typed_config::{config_value, typed_values}; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_phases::constants::{ BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, PERMANENT_TABLE_TYPE, RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, SNOWFLAKE_ADAPTER, @@ -113,9 +114,13 @@ pub(crate) fn plan_table_types<'py>( if !snowflake { return tuple(py, Vec::new()); } + let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; + if let Ok(facade) = adapter.downcast::() { + facade.get().mark_unknown("table types".to_owned()); + return tuple(py, Vec::new()); + } let target = effective_target_config(runtime)?; let relations = native_field(&native_field(warehouse, "snapshot")?, "existing_relations")?; - let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let materialization = record_class(py, RecordId::MaterializationType)?; let mut entries: Vec<_> = Vec::new(); for model in models { @@ -600,7 +605,9 @@ pub(crate) fn plan_retention<'py>( models.push(model); } } - if models.is_empty() { + if models.is_empty() + || runtime_field(runtime, |runtime| &runtime.adapter)?.is_instance_of::() + { return tuple(py, Vec::new()); } let entries = if adapter_is(runtime, BIGQUERY_ADAPTER)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 67578a72dd..00be1bffb5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -45,7 +45,8 @@ fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult) -> PyResult> { +/// Bounds the run resolves when it starts, as for inputs another node of the run produces. +pub(crate) fn unbounded(py: Python<'_>) -> PyResult> { let sentinel = host::entry(py, ExecutorHost::BoundSentinel)?; new_record_with( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 8ed2dd220c..e8f1c2e014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -10,9 +10,12 @@ use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::typed_config::{ config_node_of, config_text_is, config_text_object, config_value, model_values, }; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_resolution::cursor_model::compute_model_cursor_bounds; +use crate::bindings::_helpers::planner_resolution::cursor_model::{ + compute_model_cursor_bounds, unbounded, +}; use crate::bindings::_helpers::planner_resolution::models::{CursorFiltering, ModelSqlRequest}; use crate::bindings::_helpers::planner_resolution::refs::{ resolve_dbt_ref_references, resolve_refs, resolve_table_function_references, @@ -216,14 +219,21 @@ pub(crate) fn resolve_model_sql<'py>( let (_, uses_intrinsics) = resolve_cursor_intrinsics(py, &sql, None, None)?; let name: String = model_reads::model_name(model)?; if uses_intrinsics { - if bounds.is_none() { - let cause = match &request.full_rebuild_cause { - Some(cause) => Some(cause.clone()), - None if request.full_refresh => Some("--full-refresh".to_owned()), - None => None, - }; - return Err(unresolved_bounds_error(py, &name, cause.as_deref())?); - } + let cause = match &request.full_rebuild_cause { + Some(cause) => Some(cause.clone()), + None if request.full_refresh => Some("--full-refresh".to_owned()), + None => None, + }; + let bounds = match bounds { + bounds if !bounds.is_none() => bounds, + _ if cause.is_none() + && resolution.adapter.is_instance_of::() + && native_field(&roles, "watermark_inputs")?.is_truthy()? => + { + unbounded(py)? + } + _ => return Err(unresolved_bounds_error(py, &name, cause.as_deref())?), + }; sql = intrinsic_bounds(&sql, &bounds, &cursor_type, &resolution.adapter)?; } resolution.assert_resolved(&sql, &format!("Model '{name}'"))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 32ff170938..e586e6b1b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -13,6 +13,7 @@ use crate::bindings::_helpers::cursor::scalars::render_value; use crate::bindings::_helpers::cursor::scalars::to_inclusive; use crate::bindings::_helpers::model_config::typed_config::{config_cursor_bound, model_values}; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ by_name, config_str, model_key, native_member, optional_text, planner_error, planner_model, @@ -40,6 +41,7 @@ use pyo3::IntoPyObject; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFloat, PyTuple}; +use sqlbuild_analysis::planner_changes::constants::FORWARD_ONLY_POLICY; use sqlbuild_analysis::planner_warehouse::main::cursor_bounds::cursor_bounds; use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use sqlbuild_analysis::planner_warehouse::main::cursor_query_identity::cursor_query_identity; @@ -90,6 +92,10 @@ struct CursorModel<'py> { effective_grain: Bound<'py, PyAny>, start_cursor_config: Bound<'py, PyAny>, has_start_override: bool, + /// Lookback, a replay on change, a start override or microbatches move the start off the last cursor. + replays: bool, + /// The `cursor_start` floor's text, which lifts the start when it lies past the last cursor. + floor: Option, watermark_mode: Bound<'py, PyAny>, watermark_all: bool, watermark_strategy: bool, @@ -312,6 +318,29 @@ fn input_grain(reference: &Bound<'_, PyAny>, maps: &ModelMaps<'_>) -> PyResult, cursor_type: Option<&str>) -> PyResult> { + if !cursor_type.is_some_and(|kind| [CURSOR_TYPE_TIMESTAMP, CURSOR_TYPE_INTEGER].contains(&kind)) + { + return Ok(None); + } + config_cursor_bound(model.py(), &model_values(model)?, "cursor_start")? + .map(|raw| raw.str()?.extract()) + .transpose() +} + +/// Whether a `cursor_start` floor lifts the run's start past the last build's cursor. +fn floor_lifts(info: &CursorModel<'_>, observed: &Bound<'_, PyAny>) -> PyResult { + let py = observed.py(); + let (Some(floor), Some(cursor_type)) = (&info.floor, &info.cursor_type) else { + return Ok(false); + }; + match scalars::try_parse(&text(py, floor), &text(py, cursor_type))? { + Some(floor) => Ok(compare_values(&floor, observed)? > 0), + None => Ok(false), + } +} + fn start_override_set(model: &Bound<'_, PyAny>, overrides: &Bound<'_, PyAny>) -> PyResult { if overrides.is_none() { return Ok(false); @@ -471,6 +500,7 @@ fn collect_models<'py>( .filter(|value| !value.is_empty()) .unwrap_or_else(|| "all".to_owned()); let watermark_mode = native_member(py, RecordId::CursorWatermarkMode, &mode)?; + let has_start_override = start_override_set(&model, &inputs.cursor_overrides)?; models.push(CursorModel { target_tag, target_relation, @@ -485,7 +515,13 @@ fn collect_models<'py>( upstreams, effective_grain, start_cursor_config: resolve_start_cursor_config(&model, &inputs.start_cursor_config)?, - has_start_override: start_override_set(&model, &inputs.cursor_overrides)?, + has_start_override, + replays: config_str(&model, "lookback")?.is_some() + || config_str(&model, "batch_size")?.is_some() + || config_str(&model, "replay_on_change")? + .is_some_and(|policy| policy != FORWARD_ONLY_POLICY) + || has_start_override, + floor: cursor_floor(&model, cursor_type.as_deref())?, watermark_all: watermark_mode.eq("all")?, watermark_mode, watermark_strategy: strategy.as_deref() == Some("watermark"), @@ -620,7 +656,7 @@ fn report_cursor_query( ) } -fn bound_task<'py>( +pub(crate) fn bound_task<'py>( py: Python<'py>, sql: String, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -697,12 +733,12 @@ fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResul } /// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. -fn adapter_cursor_sql( - request: &SnapshotRequest<'_>, +pub(crate) fn adapter_cursor_sql( + adapter: &Bound<'_, PyAny>, query: &PhysicalCursorQuery, ) -> PyResult { - let py = request.adapter.py(); - let render = native_field(&request.adapter, CURSOR_TEXT_METHOD)?; + let py = adapter.py(); + let render = native_field(adapter, CURSOR_TEXT_METHOD)?; let failure: RefCell> = RefCell::new(None); let sql: String = cursor_query_sql(query, &|expression: &str| { call_with( @@ -740,7 +776,8 @@ fn execute_queries<'py>( if !on_progress.is_none() { on_progress.call1((format!("Inspecting cursor bounds {identity}..."),))?; } - let outcome = read_cursor_query(py, &adapter_cursor_sql(request, query)?, handles)?; + let outcome = + read_cursor_query(py, &adapter_cursor_sql(&request.adapter, query)?, handles)?; report_cursor_query(py, query, (&outcome, &identity), on_progress)?; outcomes.push(outcome); } @@ -756,7 +793,7 @@ fn execute_queries<'py>( for query in queries { tasks.push(bound_task( py, - adapter_cursor_sql(request, query)?, + adapter_cursor_sql(&request.adapter, query)?, handles, )?); } @@ -1149,6 +1186,51 @@ fn assemble<'py>( Ok(snapshots) } +/// Plan previews: target `MAX` values from the synced copy; inputs are read when the run starts. +fn synced_copy_values<'py>( + facade: &SyncedCopyAdapter, + models: &mut [CursorModel<'py>], +) -> PyResult> { + let mut values = Values::new(); + for info in models { + let reads_inputs: bool = !info.upstreams.is_empty(); + info.upstreams.clear(); + let observed: Option<&str> = info + .target_tag + .as_ref() + .and_then(|_| facade.target_max(&info.model_name)); + match (&info.target_tag, observed) { + (Some(tag), Some(observed)) => { + let value = parse_warehouse_cursor(&text(info.effective_grain.py(), observed))?; + let lifted = floor_lifts(info, &value)?; + values.insert(tag.clone(), value); + if reads_inputs && (info.replays || lifted) { + facade.mark_unknown(format!( + "cursor window of {} (the run starts it away from the last build's cursor)", + info.model_name + )); + } else if reads_inputs { + facade.mark_window(&info.model_name, observed); + } + if !info.start_cursor_config.is_none() + && !native_field(&info.start_cursor_config, "max_ahead")?.is_none() + { + facade.mark_unknown(format!("future-cursor safety of {}", info.model_name)); + } + } + (Some(_), None) => facade.mark_unknown(format!( + "cursor position of {} (no successful build recorded it)", + info.model_name + )), + (None, _) if reads_inputs => { + facade.mark_unknown(format!("cursor window of {}", info.model_name)); + } + (None, _) => {} + } + } + Ok(values) +} + /// Cursor MIN/MAX values of the selected incremental models, by model name. pub(crate) fn cursor_snapshots<'py>( request: &SnapshotRequest<'py>, @@ -1162,10 +1244,14 @@ pub(crate) fn cursor_snapshots<'py>( seeds: by_name(&request.project, "seeds")?, functions: by_name(&request.project, "functions")?, }; - let models = collect_models(request, existing_relations, inputs, &maps)?; + let mut models = collect_models(request, existing_relations, inputs, &maps)?; if models.is_empty() { return Ok(PyDict::new(py)); } + if let Ok(facade) = request.adapter.downcast::() { + let values = synced_copy_values(facade.get(), &mut models)?; + return assemble(py, &models, (&values, &HashSet::new())); + } let queries = physical_queries(&models); let started = Instant::now(); let concurrency = inspection_concurrency(&request.adapter, &request.connection)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index b99fd5cebe..fef2043627 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -4,6 +4,8 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; +use crate::bindings::_helpers::observations::source_capture::capture_source_columns; use crate::bindings::_helpers::planner_phases::objects::{frozenset, text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ COLUMN_INFO, METADATA_NAME_FILTER_LIMIT, RELATION_LOOKUP, @@ -92,18 +94,43 @@ pub(crate) fn source_inspection<'py>( let expression = native_field(entry, "expression")?; if !expression.is_none() { if native_field(entry, "type_enforcement")?.is_truthy()? { - let names = call_with( - &native_field(adapter, "query_column_names")?, - &[("connection", connection.clone()), ("sql", expression)], - )?; + let sql: String = expression.extract()?; + let source: Bound<'py, PyAny> = native_field(entry, "name")?; + let source_name: String = source.extract()?; + let described: Vec = match adapter.downcast::() { + Ok(facade) => match facade.get().described_columns(&source_name, &sql)? { + Some(described) => described, + None => { + facade.get().mark_unknown(format!( + "query columns of expression source {source_name} (planned as declared)" + )); + native_field(entry, "columns")? + .try_iter()? + .map(|column| native_field(&column?, "name")?.extract()) + .collect::>()? + } + }, + Err(_) => { + let names = call_with( + &native_field(adapter, "query_column_names")?, + &[("connection", connection.clone()), ("sql", expression)], + )?; + let described: Vec = names + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + capture_source_columns(&source_name, &sql, &described); + described + } + }; let mut columns: Vec<_> = Vec::new(); - for name in names.try_iter()? { + for name in &described { columns.push(call_with( &column_info, - &[("name", name?), ("type", text(py, ""))], + &[("name", text(py, name)), ("type", text(py, ""))], )?); } - result.set_item(native_field(entry, "name")?, tuple(py, columns)?)?; + result.set_item(source, tuple(py, columns)?)?; } continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 3ce4fbb65f..d2d6e229c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -4,10 +4,12 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::resource_reads; +use crate::bindings::_helpers::freshness::planning::PREVIEW_MESSAGE; use crate::bindings::_helpers::freshness::planning::{ build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, }; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_phases::objects::{native, replace, text, tuple}; use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; @@ -35,7 +37,9 @@ pub(crate) fn source_freshness_unknown_warnings<'py>( { let unknown = unknown?; let reason: String = native_field(&unknown, "reason")?.str()?.extract()?; - if freshness_unknown_message(&reason, &[]).is_none() { + if freshness_unknown_message(&reason, &[]).is_none() + || native_field(&unknown, "message")?.eq(PREVIEW_MESSAGE)? + { continue; } let name: String = native_field(&unknown, "source_name")?.extract()?; @@ -142,11 +146,15 @@ pub(crate) fn build_planner_source_freshness_result<'py>( lowered_state.insert(schema?.extract()?); } } + let synced_copy = adapter.is_instance_of::(); let exists = PyDict::new(py); for schema in &schemas { exists.set_item( schema, - PyBool::new(py, lowered_state.contains(&lower(&text(py, schema))?)), + PyBool::new( + py, + !synced_copy && lowered_state.contains(&lower(&text(py, schema))?), + ), )?; } let sources = tuple( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index 80cde938c6..6724ab781e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -6,6 +6,7 @@ use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::discovery::locations::yaml_entry_line; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::{ @@ -121,7 +122,7 @@ pub(crate) fn check_selected_source_tables_exist<'py>( sorted.sort(); unlisted.push((source, entry, sorted)); } - if unlisted.is_empty() { + if unlisted.is_empty() || adapter.is_instance_of::() { return Ok(()); } let probe = native_field(adapter, "probe_relation_read")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 2a10b7918f..615b0a4fa3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -35,9 +35,6 @@ pub(crate) const DEFAULT_MAX_CELL_BYTES: usize = 4096; pub(crate) const JSON_FORMAT: &str = "json"; /// How text output shows SQL NULL. pub(crate) const NULL_TEXT: &str = "NULL"; -/// The standard base64 alphabet binary JSON cells are encoded with. -pub(crate) const BASE64_ALPHABET: &[u8; 64] = - b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; /// Join keys `sqb recall schema` points out. pub(crate) const JOIN_KEYS: [&str; 5] = [ "invocation_id", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs index f2a5cd4abc..ce75b9d4e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -2,6 +2,7 @@ use pyo3::prelude::{PyResult, Python}; use serde_json::{Map, Value, json}; +use sqlbuild_cache::digest::main::base64_text::base64_text; use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_recall::records::models::RecallValue; use sqlbuild_recall::schema::models::Dialect; @@ -9,31 +10,7 @@ use sqlbuild_recall::store::models::{DescribedColumn, DescribedRelation, QueryRe use crate::bindings::_helpers::query_command::text_output::render_text; use crate::bindings::_helpers::recall::clickhouse_calls::QueryStatistics; -use crate::bindings::_helpers::recall::constants::{ - BASE64_ALPHABET, JOIN_KEYS, JSON_FORMAT, NULL_TEXT, -}; - -fn base64(data: &[u8]) -> String { - let mut encoded = String::with_capacity(data.len().div_ceil(3) * 4); - for chunk in data.chunks(3) { - let bytes = [ - chunk[0], - *chunk.get(1).unwrap_or(&0), - *chunk.get(2).unwrap_or(&0), - ]; - let triple = u32::from(bytes[0]) << 16 | u32::from(bytes[1]) << 8 | u32::from(bytes[2]); - for index in 0..4 { - if index <= chunk.len() { - encoded.push(char::from( - BASE64_ALPHABET[(triple >> (18 - 6 * index) & 63) as usize], - )); - } else { - encoded.push('='); - } - } - } - encoded -} +use crate::bindings::_helpers::recall::constants::{JOIN_KEYS, JSON_FORMAT, NULL_TEXT}; fn truncated(text: &str, max_bytes: Option) -> String { let Some(limit) = max_bytes.filter(|limit| text.len() > *limit) else { @@ -84,7 +61,7 @@ fn json_cell(value: &RecallValue) -> Value { RecallValue::Text(text) => json!(text), RecallValue::Blob(data) => match std::str::from_utf8(data) { Ok(text) => json!(text), - Err(_) => json!({ "base64": base64(data) }), + Err(_) => json!({ "base64": base64_text(data) }), }, } } diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index d0fcba2598..3fc9db2953 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -7,4 +7,5 @@ pub mod events; pub mod execution_history; pub mod freshness; pub mod microbatches; +pub mod observations; pub mod state_tables; diff --git a/crates/sqlbuild-state/src/observations/constants.rs b/crates/sqlbuild-state/src/observations/constants.rs new file mode 100644 index 0000000000..009848b760 --- /dev/null +++ b/crates/sqlbuild-state/src/observations/constants.rs @@ -0,0 +1,26 @@ +//! Columns and kinds of the display-only observations state table. + +use crate::freshness::constants::{STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE}; + +/// The column the warehouse clock fills on insert. +pub const OBSERVED_AT_COLUMN: &str = "observed_at"; +/// Columns in declaration order with their state value types; `observed_at` is last. +pub const OBSERVATION_COLUMN_TYPES: [(&str, &str); 7] = [ + ("observation_id", STRING_STATE_VALUE_TYPE), + ("kind", STRING_STATE_VALUE_TYPE), + ("node", STRING_STATE_VALUE_TYPE), + ("cursor_type", STRING_STATE_VALUE_TYPE), + ("value", STRING_STATE_VALUE_TYPE), + ("run_id", STRING_STATE_VALUE_TYPE), + (OBSERVED_AT_COLUMN, TEXT_TIMESTAMP_STATE_VALUE_TYPE), +]; +/// Columns that may be NULL: only `cursor_max` observations have a cursor type. +pub const NULLABLE_OBSERVATION_COLUMNS: [&str; 1] = ["cursor_type"]; +/// Observation of a model's cursor `MAX` after a successful run. +pub const CURSOR_MAX_KIND: &str = "cursor_max"; +/// Observation of the columns a live `DESCRIBE` returned for an expression source. +pub const SOURCE_COLUMNS_KIND: &str = "source_columns"; +/// Key of a `source_columns` value holding the SHA-256 of the SQL that was described. +pub const SOURCE_COLUMNS_SQL_HASH_KEY: &str = "sql_hash"; +/// Key of a `source_columns` value holding the described column names in order. +pub const SOURCE_COLUMNS_NAMES_KEY: &str = "columns"; diff --git a/crates/sqlbuild-state/src/observations/main/mod.rs b/crates/sqlbuild-state/src/observations/main/mod.rs new file mode 100644 index 0000000000..6e65780594 --- /dev/null +++ b/crates/sqlbuild-state/src/observations/main/mod.rs @@ -0,0 +1 @@ +pub mod prune_observations_sql; diff --git a/crates/sqlbuild-state/src/observations/main/prune_observations_sql.rs b/crates/sqlbuild-state/src/observations/main/prune_observations_sql.rs new file mode 100644 index 0000000000..5f835804a0 --- /dev/null +++ b/crates/sqlbuild-state/src/observations/main/prune_observations_sql.rs @@ -0,0 +1,13 @@ +//! Prune old observations from one observations table. + +use crate::observations::constants::OBSERVED_AT_COLUMN; + +/// Delete every observation ranked beyond the newest `retain` per kind and node. +pub fn prune_observations_sql(qualified_name: &str, retain: i64) -> String { + format!( + "DELETE FROM {qualified_name} WHERE observation_id IN (SELECT observation_id FROM \ +(SELECT observation_id, ROW_NUMBER() OVER (PARTITION BY kind, node ORDER BY \ +{OBSERVED_AT_COLUMN} DESC, observation_id DESC) AS observation_rank FROM {qualified_name}) \ +AS ranked WHERE observation_rank > {retain})" + ) +} diff --git a/crates/sqlbuild-state/src/observations/mod.rs b/crates/sqlbuild-state/src/observations/mod.rs new file mode 100644 index 0000000000..aae420f1a5 --- /dev/null +++ b/crates/sqlbuild-state/src/observations/mod.rs @@ -0,0 +1,2 @@ +pub mod constants; +pub mod main; diff --git a/fensu.toml b/fensu.toml index e9afc99dd8..9fc2f7c2aa 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1630,9 +1630,10 @@ tooling = [] rule_packs = ["rust"] rule_modules = [ "scripts.fensu_policy.rules.native_boundary", + "scripts.fensu_policy.rules.observation_boundary", "scripts.fensu_policy.rules.unused_state", ] -select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077"] +select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "XSB079"] [targets.rust.rule_options.XSB076] schema_paths = ["crates/sqlbuild-orchestration/src/store"] @@ -1646,6 +1647,35 @@ allowed = [ "sqb_warehouse_migrations.decision=Reader comes with the UI epic (D132), which filters on it.", ] +[targets.rust.rule_options.XSB078] +allowed_paths = [ + "crates/sqlbuild-core/src/constants.rs=Names the observation state table.", + "crates/sqlbuild-orchestration=Mirrors observations into the control store and serves them for display.", + "crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs=Registers the observation state bindings.", + "crates/sqlbuild-python/src/bindings/_helpers/exported_constants=Exports the table name to Python adapters, which create it.", + "crates/sqlbuild-python/src/bindings/_helpers/janitor=Prunes observation history.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs=Declares the observation state bindings module.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs=Renders observation state SQL for adapters.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs=Reads observations for the control-store mirror.", + "crates/sqlbuild-python/src/bindings/_helpers/observations=Writes observations after a build.", +] + +[targets.rust.rule_options.XSB079] +allowed_paths = [ + "crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs=A location's requested plan names the synced copy for its sqb plan child.", + "crates/sqlbuild-core/src/plan_display/constants.rs=Names the synced-copy environment variable.", + "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs=Registers the synced copy's refusal error with Python.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs=Defines the synced-copy adapter.", + "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only sqb plan activates the synced copy and reports what it could not know.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=Preview branch: target MAX from the copy, watermark inputs read when the run starts.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=Preview branch: table types and retention stay unknown.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=Preview branch: cursor intrinsics bind to bounds the run resolves.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=Preview branch: expression columns from the copy instead of DESCRIBE.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=Preview branch: the copy has no source freshness state to read.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=Preview branch: sources outside the copy are not probed.", +] + [targets.rust.rule_options.XSB077] construction_required = [ "Action=A permission is live only when a route requires it; role tables and classifiers do not count.", @@ -1815,6 +1845,18 @@ forbidden_owners = ["src/sqlbuild/adapter/contract/classes/base_adapter.py:BaseA paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/classes/*"] reason = "tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py requires every first-class adapter to define each StrictAdapter method itself instead of inheriting it from BaseAdapter." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs"] +reason = "Each state table has its own Python-facing create and prune bindings. The DDL and table naming live in the shared fixed_state_table_sql and state_table_name; each prune keeps its own retention rule." + +[[dupes.allowlist]] +paths = ["src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py"] +reason = "SQL Server overrides each state table's create SQL to add that table's own key index; the keyed DDL lives in _keyed_state_table_sql." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] +reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." + [[dupes.allowlist]] paths = ["crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs"] reason = "Resource asset projections intentionally mirror their public JSON schemas: seeds include planning reasons, functions include warnings and their function kind, and loads include loader and skip metadata." @@ -1828,7 +1870,7 @@ paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerpr reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_column_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_consumption.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_node_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_old_name_views.rs"] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_column_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_consumption.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_node_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_old_name_views.rs"] reason = "Typed event projections retain distinct required identities and indexed payload columns. Event persistence, fencing, atomic retries and reconciliation share the events ingestion owner; no per-table lifecycle is copied." [[targets.python.rule_exceptions]] @@ -2832,10 +2874,12 @@ module_layers = [ "native_host=infra", "native_host/consumption_state=front", "native_host/model_lookup=project", + "native_host/observations_state=front", "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", "native_host/warehouse_metadata=commands", "native_threads=infra", + "observations=plan", "output_capture=output", "plan_command=commands", "plan_output=plan", diff --git a/scripts/fensu_policy/_helpers/path_checks.py b/scripts/fensu_policy/_helpers/path_checks.py index 3f4ff27c48..e61f0e43db 100644 --- a/scripts/fensu_policy/_helpers/path_checks.py +++ b/scripts/fensu_policy/_helpers/path_checks.py @@ -32,3 +32,13 @@ def is_adapter_implementation_path(*, path: str) -> bool: "/adapter/contract/classes/duckdb_backed_adapter.py", ) ) + + +def allowed_prefix(*, path: str, prefixes: list[str]) -> str | None: + """The allowed file or directory entry that covers a repository-relative path.""" + + prefix: str + for prefix in prefixes: + if path == prefix or path.startswith(prefix.rstrip("/") + "/"): + return prefix + return None diff --git a/scripts/fensu_policy/_helpers/unused_state.py b/scripts/fensu_policy/_helpers/unused_state.py index 0571284126..5a9d00018e 100644 --- a/scripts/fensu_policy/_helpers/unused_state.py +++ b/scripts/fensu_policy/_helpers/unused_state.py @@ -1,12 +1,15 @@ -"""Find control-store state nothing reads and Rust enum variants nothing names.""" +"""Find control-store state nothing reads, Rust enum variants nothing names, and state uses.""" from __future__ import annotations import itertools import re -from collections.abc import Iterable +from collections.abc import Callable, Iterable from pathlib import Path +from fensu import Fault, RuleContext, RuleOption + +from scripts.fensu_policy._helpers.path_checks import allowed_prefix from scripts.fensu_policy.constants import ( BRACE_CLOSE, BRACE_OPEN, @@ -14,8 +17,10 @@ BRACKET_OPEN, LIST_SEPARATOR, NATIVE_PATH_SEPARATOR, + OBSERVATION_STATE_PATTERN, PARENTHESIS_CLOSE, PARENTHESIS_OPEN, + POLICY_CONFIG_FILE, REASON_SEPARATOR, REMOVED_CODE, RUST_ATTRIBUTE_PATTERN, @@ -23,6 +28,7 @@ RUST_ENUM_PATTERN, RUST_FROM_ATTRIBUTE_PATTERN, RUST_IMPL_HEADER_PATTERN, + RUST_LINE_COMMENT, RUST_LINE_CONTINUATION, RUST_LITERAL_PATTERN, RUST_LOOKBEHIND, @@ -63,9 +69,12 @@ SQL_VIEW_PREFIX_PATTERN, SQL_WORD_PATTERN, SQL_WRITE_WORDS, + SYNCED_COPY_PATTERN, ) from scripts.fensu_policy.models import EnumVariant, StateDefinition, StateReads +_OBSERVATION_STATE: re.Pattern[str] = re.compile(OBSERVATION_STATE_PATTERN) +_SYNCED_COPY: re.Pattern[str] = re.compile(SYNCED_COPY_PATTERN) _LEXEMES: re.Pattern[str] = re.compile(RUST_LITERAL_PATTERN, re.DOTALL) _CONTINUATION: re.Pattern[str] = re.compile(RUST_LINE_CONTINUATION) _TEST_MODULE: re.Pattern[str] = re.compile(RUST_TEST_MODULE_PATTERN) @@ -589,3 +598,67 @@ def _state_key(*, definition: StateDefinition) -> str: if definition.column is None: return definition.table return f"{definition.table}{SQL_QUALIFIER}{definition.column}" + + +def observation_reference_lines(*, source: str) -> list[int]: + """One-based lines whose code, not a line comment, names display-only observation state.""" + + return _reference_lines(source=source, pattern=_OBSERVATION_STATE) + + +def synced_copy_reference_lines(*, source: str) -> list[int]: + """One-based lines whose code, not a line comment, names the plan preview's synced copy.""" + + return _reference_lines(source=source, pattern=_SYNCED_COPY) + + +def _reference_lines(*, source: str, pattern: re.Pattern[str]) -> list[int]: + lines: list[int] = [] + number: int + line: str + for number, line in enumerate(source.splitlines(), start=1): + if pattern.search(line.partition(RUST_LINE_COMMENT)[0]): + lines.append(number) + return lines + + +def confined_references( + *, + ctx: RuleContext, + option: RuleOption[tuple[str, ...]], + references: Callable[..., list[int]], + subject: str, +) -> list[Fault]: + """Faults for each reference outside the allowed paths, and for stale allowed paths.""" + + reasons: dict[str, str] + malformed: list[str] + reasons, malformed = reasoned_entries(entries=ctx.option(option)) + prefixes: list[str] = list(reasons) + used: set[str] = set() + faults: list[Fault] = [] + for facts in ctx.rust.files: + path: str = facts.file.path.value + if facts.test or is_rust_test_path(path=path): + continue + lines: list[int] = references(source=facts.source) + if not lines: + continue + prefix: str | None = allowed_prefix(path=path, prefixes=prefixes) + if prefix is not None: + used.add(prefix) + continue + faults.extend( + ctx.fault_for(path=ctx.repo_root / path, line=line, column=0) for line in lines + ) + problems: list[str] = [ + f"allowed_paths entry {entry!r} needs `path=reason`" for entry in malformed + ] + [ + f"allowed path {prefix} no longer names {subject}; remove the entry" + for prefix in prefixes + if prefix not in used + ] + return [ + ctx.fault_for(path=ctx.repo_root / POLICY_CONFIG_FILE, line=1, column=0, message=problem) + for problem in problems + ] + faults diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index e6f5bcab4e..083dc0fc7f 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -476,3 +476,27 @@ description="`table.column=reason` entries for state read outside production code.", ) POLICY_CONFIG_FILE: str = "fensu.toml" +OBSERVATION_ALLOWED_PATHS: RuleOption[tuple[str, ...]] = RuleOption.string_list( + name="allowed_paths", + default=(), + description=( + "`path=reason` entries for the Rust files or directories that may name observation state." + ), +) +OBSERVATION_STATE_PATTERN: str = ( + r"\b(?:observations_state|OBSERVATIONS_TABLE_NAME|WarehouseObservationRecord" + r"|WarehouseHistoryKind::Observations|_sqlbuild_observations|sqb_warehouse_observations)\b" +) +SYNCED_COPY_ALLOWED_PATHS: RuleOption[tuple[str, ...]] = RuleOption.string_list( + name="allowed_paths", + default=(), + description=( + "`path=reason` entries for the Rust files or directories that may name the plan " + "preview's synced-copy adapter." + ), +) +SYNCED_COPY_PATTERN: str = ( + r"\b(?:SyncedCopyAdapter|SyncedCopyConnection|PLAN_SYNCED_COPY_ENV|" + r"WarehouseAccessRefusedError)\b" +) +RUST_LINE_COMMENT: str = "//" diff --git a/scripts/fensu_policy/rules/observation_boundary.py b/scripts/fensu_policy/rules/observation_boundary.py new file mode 100644 index 0000000000..3c52e2e000 --- /dev/null +++ b/scripts/fensu_policy/rules/observation_boundary.py @@ -0,0 +1,58 @@ +"""Rules keeping build observations and the plan preview's synced copy out of live decisions.""" + +from __future__ import annotations + +from fensu import AnalyzerId, Family, Fault, Project, RuleContext, rule + +from scripts.fensu_policy._helpers.unused_state import ( + confined_references, + observation_reference_lines, + synced_copy_reference_lines, +) +from scripts.fensu_policy.constants import OBSERVATION_ALLOWED_PATHS, SYNCED_COPY_ALLOWED_PATHS + + +@rule( + code="XSB078", + family=Family.CUSTOM, + slug="observation-display-only", + message="build observations are display-only and must not reach planning or build decisions", + remediation=( + "Read the live warehouse fact instead; only the files in allowed_paths, which write, " + "prune, mirror or display observations, may name observation state." + ), + cacheable=False, + analyzers=(AnalyzerId.RUST,), + options=(OBSERVATION_ALLOWED_PATHS,), +) +def observation_display_only(*, project: Project, ctx: RuleContext) -> list[Fault]: + del project + return confined_references( + ctx=ctx, + option=OBSERVATION_ALLOWED_PATHS, + references=observation_reference_lines, + subject="observation state", + ) + + +@rule( + code="XSB079", + family=Family.CUSTOM, + slug="synced-copy-preview-only", + message="only sqb plan previews may plan from the synced copy", + remediation=( + "Plan from the live adapter; only the files in allowed_paths, which define, activate or " + "branch on the plan preview's synced copy, may name it." + ), + cacheable=False, + analyzers=(AnalyzerId.RUST,), + options=(SYNCED_COPY_ALLOWED_PATHS,), +) +def synced_copy_preview_only(*, project: Project, ctx: RuleContext) -> list[Fault]: + del project + return confined_references( + ctx=ctx, + option=SYNCED_COPY_ALLOWED_PATHS, + references=synced_copy_reference_lines, + subject="the synced copy", + ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index d315dd698d..4637f7e8b0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -262,9 +262,23 @@ checked this way. Anyone with `view` on every node of a target can see what a `build` command would do before it runs. The web UI's plan page does this for every Run button: the location serving the target runs -[`sqb plan --json`](plan.md) on the command's nodes against the warehouse, and the page -shows each model's action and reason, backfills, migrations and warnings. Running the command -still needs `run.adhoc`. +[`sqb plan --json`](plan.md) on the command's nodes, and the page shows each model's action +and reason, backfills, migrations and warnings. Running the command still needs `run.adhoc`. + +A plan reads the server's synced copy of the target's warehouse metadata, not the warehouse, so +planning never resumes a suspended warehouse. The page shows when that copy synced, or that it +hasn't fully synced yet. Each incremental model's range starts where its last successful build +ended and runs to the latest data when the run starts. The run plans again against the warehouse. +These stay unknown until it does: + +- source freshness, retention, table types and column rename support; +- sources outside the schemas the server syncs; +- the cursor position of an incremental model with no successful build; +- the range start of an incremental model with lookback, a replay on change, a start override, + microbatches or a `cursor_start` past its last build; +- whether a start set with `cursor_start_max_ahead` sits too far ahead; +- the columns of an expression source whose SQL changed since it was last described, which the + plan takes as declared. Over the API, queue a plan, then read it until `result_json` is set: @@ -277,7 +291,9 @@ GET /api/control/locations/{location_id}/targets/{target}/plans/{request_id} `result_json.outcome.kind` is `plan`, with the `sqb plan --json` document in `document`, or `superseded` when the location's project files changed while it planned; plan again. A plan never -appears in Runs and changes nothing in the warehouse. +appears in Runs and never connects to the warehouse. In `document`, `synced_copy` holds +`synced_at` (`null` until the copy has fully synced), `cursor_window_starts` and +`unknown_until_run`. | Limit | Value | | --- | --- | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a77ff62373..1ffa6ed00d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -190,6 +190,7 @@ COLUMN_MIGRATION_TABLE_NAME: str OLD_NAME_VIEW_TABLE_NAME: str MICROBATCH_TABLE_NAME: str NODE_RESULTS_TABLE_NAME: str +OBSERVATIONS_TABLE_NAME: str AUDIT_RESULTS_TABLE_NAME: str JANITOR_EVENTS_TABLE_NAME: str COLUMN_NODE_TYPE: str @@ -5502,6 +5503,20 @@ def build_consumption_prune_sql( render_qualified_name: Callable[..., str | None], retain_versions: int, ) -> str: ... +def build_observations_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def build_observations_prune_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + retain_versions: int, +) -> str: ... def build_source_freshness_create_table_sql( *, database: str | None, @@ -6351,6 +6366,7 @@ class CostArtifactError(ValueError): ... class CommandTerminated(KeyboardInterrupt): ... class SourceFreshnessInputError(ValueError): ... class SourceFreshnessObservationError(ValueError): ... +class WarehouseAccessRefusedError(RuntimeError): ... class CostTelemetryHealth: def mark_ledger_failure(self, *, run_id: str, error: BaseException) -> None: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 030a92b905..7366fe01c9 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2301,6 +2301,18 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s render_framework_type=self.render_framework_type, ) + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the observations table when it is missing.""" + + from sqlbuild._native import build_observations_create_table_sql + + return build_observations_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_create_consumption_index_sqls( self, *, @@ -2476,6 +2488,18 @@ def render_prune_consumption_history_sql( del database, schema, retain_versions raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def render_prune_observations_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old observations.""" + + del database, schema, retain_versions + raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def sql_analysis_dialect(self) -> str | None: """Return the configured SQL analysis dialect name, if any.""" diff --git a/src/sqlbuild/adapter/contract/classes/observations.py b/src/sqlbuild/adapter/contract/classes/observations.py new file mode 100644 index 0000000000..91bcc7d7cf --- /dev/null +++ b/src/sqlbuild/adapter/contract/classes/observations.py @@ -0,0 +1,37 @@ +"""Default adapter implementations for the display-only observations state table.""" + +from __future__ import annotations + +from typing import Any, cast + + +class ObservationsMixin: + """Provide portable observations-table DDL and pruning for first-class warehouse adapters.""" + + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the observations table when it is missing.""" + + from sqlbuild._native import build_observations_create_table_sql + + adapter: Any = cast(Any, self) + return build_observations_create_table_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) + + def render_prune_observations_history_sql( + self, *, database: str | None, schema: str, retain_versions: int + ) -> str: + """Render SQL deleting observations beyond the newest `retain_versions` per node.""" + + from sqlbuild._native import build_observations_prune_sql + + adapter: Any = cast(Any, self) + return build_observations_prune_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + retain_versions=retain_versions, + ) diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 3504c8f4c4..1e40f0a464 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -1039,6 +1039,11 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s """Render DDL that creates the consumption table when it is missing.""" ... + @abstractmethod + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the observations table when it is missing.""" + ... + @abstractmethod def render_create_consumption_index_sqls( self, @@ -1147,6 +1152,17 @@ def render_prune_consumption_history_sql( """Render SQL that prunes old consumption executions.""" ... + @abstractmethod + def render_prune_observations_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old observations.""" + ... + @abstractmethod def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index bfc30561b6..4658bb818c 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -38,6 +38,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -151,7 +152,9 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class BigQueryAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class BigQueryAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """BigQuery adapter backed by google-cloud-bigquery.""" def get_columns_for_relations( diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 4155286972..cc8d367733 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -32,6 +32,7 @@ from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_snapshot_sql import render_is_distinct_from from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -154,7 +155,9 @@ ) -class ClickHouseAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class ClickHouseAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """ClickHouse adapter that builds tables aside and rewrites whole partitions.""" adapter_name: ClassVar[str] = BuiltinAdapter.CLICKHOUSE.value @@ -1926,6 +1929,11 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s super().render_create_consumption_table_sql(database=database, schema=schema) ) + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_observations_table_sql(database=database, schema=schema) + ) + def render_create_node_result_table_sql(self, *, database: str | None, schema: str) -> str: return _state_table_ddl( super().render_create_node_result_table_sql(database=database, schema=schema) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 6a4533c9da..b5a1c03c8a 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -39,6 +39,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -144,7 +145,9 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class DatabricksAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class DatabricksAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """Databricks adapter backed by databricks-sql-connector.""" def get_columns_for_relations( diff --git a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py index faf2a9effb..820c5d6447 100644 --- a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py +++ b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py @@ -10,11 +10,12 @@ from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.microbatches.constants import MICROBATCH_GENERATION_COMMENT_PREFIX -class DuckDbAdapter(MicrobatchMixin, ConsumptionMixin, DuckDbBackedAdapter): +class DuckDbAdapter(MicrobatchMixin, ConsumptionMixin, ObservationsMixin, DuckDbBackedAdapter): """DuckDB adapter allowing implicit main for targetless and named local targets.""" adapter_name: ClassVar[str] = BuiltinAdapter.DUCKDB.value diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index c3ef1b029d..6419cae9f0 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -11,6 +11,7 @@ from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.types import BuiltinAdapter, LoaderLogicalType from sqlbuild.adapters.motherduck.constants import ( @@ -22,7 +23,7 @@ ) -class MotherDuckAdapter(MicrobatchMixin, ConsumptionMixin, DuckDbBackedAdapter): +class MotherDuckAdapter(MicrobatchMixin, ConsumptionMixin, ObservationsMixin, DuckDbBackedAdapter): """MotherDuck adapter backed by DuckDB's MotherDuck connection support.""" adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index b0d22ccdc4..7907c13f32 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -39,6 +39,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -122,7 +123,9 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class PostgresAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class PostgresAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """PostgreSQL adapter backed by psycopg.""" def get_columns_for_relations( diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index b8fb21158e..a0469a709a 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -40,6 +40,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -197,7 +198,9 @@ _BULK_COLUMN_RELATION_CHUNK_SIZE: int = 2_000 -class SnowflakeAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class SnowflakeAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """Snowflake adapter backed by snowflake-connector-python.""" adapter_name: ClassVar[str] = BuiltinAdapter.SNOWFLAKE.value @@ -660,6 +663,15 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s schema=schema, ) + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + return self._with_state_table_retention( + create_sql=super().render_create_observations_table_sql( + database=database, schema=schema + ), + database=database, + schema=schema, + ) + def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: return self._with_state_table_retention( create_sql=super().render_create_microbatch_state_table_sql( diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 89ecddecf3..3ae20ff28b 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -44,6 +44,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -166,7 +167,9 @@ _FALSE_SEED_VALUES: frozenset[str] = frozenset({"false", "f", "no", "n", "0"}) -class SqliteAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class SqliteAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """SQLite adapter: one database file, SQLBuild schemas stored as `schema__name` tables.""" adapter_name: ClassVar[str] = BuiltinAdapter.SQLITE.value diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 0a7af7476b..f6ecbc9cf5 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -40,6 +40,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -133,7 +134,9 @@ def _render_tsql_distinct_condition(*, left: str, right: str) -> str: ) -class SqlServerAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class SqlServerAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """Microsoft SQL Server adapter backed by pymssql.""" def get_columns_for_relations( @@ -504,6 +507,17 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s key_columns=("target_schema", "target_name"), ) + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import OBSERVATIONS_TABLE_NAME + + return self._keyed_state_table_sql( + create_sql=super().render_create_observations_table_sql( + database=database, schema=schema + ), + table=(database, schema, OBSERVATIONS_TABLE_NAME), + key_columns=("kind", "node"), + ) + def render_create_consumption_index_sqls( self, *, database: str | None, schema: str ) -> tuple[str, ...]: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index a6152e378e..e9449ca7af 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -363,6 +363,37 @@ class ModelBackedCursorBuildE2ETestCase: expected_full_refresh_counts: tuple[tuple[str, int], ...] = field(default_factory=tuple) +@dataclass(frozen=True) +class CursorObservationBuildE2ETestCase: + """Builds whose successful incremental models must record their cursor MAX as observations.""" + + description: str + builds: int + expected_observations: tuple[tuple[str, str, str, int], ...] + + +@dataclass(frozen=True) +class DisplayOnlyObservationE2ETestCase: + """A wrong cursor observation that plan and build must ignore in favour of the live MAX. + + A misled build would append nothing; the live bound is inclusive, so it re-reads the MAX row. + """ + + description: str + injected_observations: tuple[tuple[str, str, str], ...] + added_seed_rows: str + expected_row_counts: tuple[tuple[str, int], ...] + + +@dataclass(frozen=True) +class SourceColumnObservationE2ETestCase: + """Commands after which each build, and no plan, must have recorded described source columns.""" + + description: str + commands: tuple[tuple[str, ...], ...] + expected_observations: tuple[tuple[str, str, int], ...] + + @dataclass(frozen=True) class AppendCursorBuildE2ETestCase: """Test case for append cursor lower-bound behavior across reruns.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py new file mode 100644 index 0000000000..bab2d7366a --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py @@ -0,0 +1,252 @@ +"""A build records cursor MAX values and described source columns for plan previews.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + CursorObservationBuildE2ETestCase, + DisplayOnlyObservationE2ETestCase, + SourceColumnObservationE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": dedent( + """ + name = "cursor_observation_project" + adapter = "duckdb" + + [connection] + database = "observations.duckdb" + """ + ).strip() + + "\n", + "seeds/raw_orders.csv": "id,ordered_at\n1,2026-01-01 00:00:00\n2,2026-01-01 01:00:00\n", + "seeds/raw_orders.yml": dedent( + """ + seeds: + - name: raw_orders + description: Raw orders. + columns: + - name: id + type: INTEGER + - name: ordered_at + type: TIMESTAMP + """ + ).strip() + + "\n", + "models/orders.sql": dedent( + """ + MODEL (description "Orders by timestamp cursor.", + materialized incremental, + incremental_strategy append, + cursor ordered_at, + cursor_type timestamp, + cursor_grain second, + ); + + SELECT id, ordered_at + FROM __seed("raw_orders") + """ + ).strip() + + "\n", + "models/order_ids.sql": dedent( + """ + MODEL (description "Orders by integer cursor.", + materialized incremental, + incremental_strategy append, + cursor id, + cursor_type integer, + ); + + SELECT id FROM __seed("raw_orders") + """ + ).strip() + + "\n", + "sources/raw.yml": dedent( + """ + sources: + - name: raw_refunds + description: Refunds described from an inline query. + expression: | + SELECT * FROM (VALUES (1, 100)) AS refunds(refund_id, amount_cents) + type_enforcement: true + columns: + - name: refund_id + type: INTEGER + - name: amount_cents + type: INTEGER + """ + ).strip() + + "\n", + "models/refunds.sql": dedent( + """ + MODEL (description "Refunds.", materialized table); + + SELECT refund_id, amount_cents FROM __source("raw_refunds") + """ + ).strip() + + "\n", + "models/order_count.sql": dedent( + """ + MODEL (description "Order count without a cursor.", materialized table); + + SELECT COUNT(*) AS order_count FROM __seed("raw_orders") + """ + ).strip() + + "\n", +} + + +@pytest.mark.parametrize( + "test_case", + [ + CursorObservationBuildE2ETestCase( + description="each build appends one cursor_max per incremental model", + builds=2, + expected_observations=( + ("order_ids", "integer", "2", 2), + ("orders", "timestamp", "2026-01-01 01:00:00", 2), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_incremental_models_when_building_then_cursor_max_observations_are_recorded( + tmp_path: Path, test_case: CursorObservationBuildE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="cursor_observations", repo_files=_PROJECT_FILES + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=("build",), project_dir=project_dir) for _ in range(test_case.builds) + ] + observations: list[tuple[object, ...]] = query_duckdb( + db_path=project_dir / "observations.duckdb", + sql=( + "SELECT node, cursor_type, value, COUNT(DISTINCT run_id) " + "FROM main._sqlbuild_observations WHERE kind = 'cursor_max' " + "GROUP BY node, cursor_type, value ORDER BY node" + ), + ) + + assert [result.returncode for result in results] == [0] * test_case.builds, results[-1].stderr + assert "cursor observations" not in "".join(result.stderr for result in results) + assert tuple(observations) == test_case.expected_observations + + +@pytest.mark.parametrize( + "test_case", + [ + SourceColumnObservationE2ETestCase( + description="builds record the described SQL hash and columns and plans record none", + commands=(("build",), ("plan",), ("build",)), + expected_observations=( + ( + "raw_refunds", + json.dumps( + { + "sql_hash": hashlib.sha256( + b"SELECT * FROM (VALUES (1, 100)) AS refunds(refund_id, amount_cents)\n" + ).hexdigest(), + "columns": ["refund_id", "amount_cents"], + }, + separators=(",", ":"), + ), + 2, + ), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_expression_source_when_building_then_source_column_observations_are_recorded( + tmp_path: Path, test_case: SourceColumnObservationE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="source_observations", repo_files=_PROJECT_FILES + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=command, project_dir=project_dir) for command in test_case.commands + ] + observations: list[tuple[object, ...]] = query_duckdb( + db_path=project_dir / "observations.duckdb", + sql=( + "SELECT node, value, COUNT(DISTINCT run_id) " + "FROM main._sqlbuild_observations WHERE kind = 'source_columns' " + "GROUP BY node, value ORDER BY node" + ), + ) + + assert [result.returncode for result in results] == [0] * len(test_case.commands), results[ + -1 + ].stderr + assert tuple(observations) == test_case.expected_observations + + +@pytest.mark.parametrize( + "test_case", + [ + DisplayOnlyObservationE2ETestCase( + description="cursor_max past the live MAX still appends from the inclusive live bound", + injected_observations=( + ("order_ids", "integer", "999"), + ("orders", "timestamp", "2099-01-01 00:00:00"), + ), + added_seed_rows="3,2026-01-01 02:00:00\n", + expected_row_counts=(("order_ids", 4), ("orders", 4)), + ), + ], + ids=lambda case: case.description, +) +def test_given_wrong_cursor_observation_when_planning_and_building_then_live_max_is_used( + tmp_path: Path, test_case: DisplayOnlyObservationE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="display_only_observations", repo_files=_PROJECT_FILES + ) + database: Path = project_dir / "observations.duckdb" + first: subprocess.CompletedProcess[str] = run_sqb(command=("build",), project_dir=project_dir) + assert first.returncode == 0, first.stderr + seed: Path = project_dir / "seeds/raw_orders.csv" + seed.write_text(seed.read_text() + test_case.added_seed_rows) + + honest: subprocess.CompletedProcess[str] = run_sqb(command=("plan",), project_dir=project_dir) + node: str + cursor_type: str + value: str + for node, cursor_type, value in test_case.injected_observations: + execute_duckdb( + db_path=database, + sql=( + "INSERT INTO main._sqlbuild_observations VALUES " + f"('injected-{node}', 'cursor_max', '{node}', '{cursor_type}', '{value}', " + "'injected-run', '2099-01-01 00:00:00')" + ), + ) + misled: subprocess.CompletedProcess[str] = run_sqb(command=("plan",), project_dir=project_dir) + build: subprocess.CompletedProcess[str] = run_sqb(command=("build",), project_dir=project_dir) + counts: tuple[tuple[str, int], ...] = tuple( + (name, query_duckdb(db_path=database, sql=f"SELECT COUNT(*) FROM main.{name}")[0][0]) + for name, _ in test_case.expected_row_counts + ) + + assert honest.returncode == misled.returncode == 0, misled.stderr + assert misled.stdout.partition("Plan ready")[2] == honest.stdout.partition("Plan ready")[2] + assert "Plan ready" in honest.stdout + assert build.returncode == 0, build.stderr + assert counts == test_case.expected_row_counts diff --git a/tests/integration/scripts/fensu_policy/_test_types.py b/tests/integration/scripts/fensu_policy/_test_types.py index ef287d1b65..33d3fb804d 100644 --- a/tests/integration/scripts/fensu_policy/_test_types.py +++ b/tests/integration/scripts/fensu_policy/_test_types.py @@ -8,3 +8,13 @@ class NativeBoundaryTestCase: description: str metadata: str expected_fault_count: int + + +@dataclass(frozen=True) +class ObservationBoundaryTestCase: + description: str + rule: str + allowed: tuple[str, ...] + planner: str + expected_returncode: int + expected_faults: tuple[str, ...] diff --git a/tests/integration/scripts/fensu_policy/helpers.py b/tests/integration/scripts/fensu_policy/helpers.py new file mode 100644 index 0000000000..e0592a79f3 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/helpers.py @@ -0,0 +1,61 @@ +"""Build a minimal repository and run the real Fensu CLI on its Rust target.""" + +import shutil +import subprocess +import sys +from pathlib import Path + + +def write_observation_project( + *, root: Path, rule: str, allowed: tuple[str, ...], planner: str +) -> None: + """A crate whose observations module names both preview-only states, plus the given planner.""" + + repository: Path = Path(__file__).resolve().parents[4] + shutil.copytree( + repository / "scripts/fensu_policy", + root / "scripts/fensu_policy", + ignore=shutil.ignore_patterns("__pycache__"), + ) + (root / "scripts/__init__.py").write_text("") + (root / "Cargo.toml").write_text( + '[workspace]\nmembers = ["crates/sqlbuild-python"]\nresolver = "2"\n' + ) + crate: Path = root / "crates/sqlbuild-python" + (crate / "src/observations").mkdir(parents=True) + (crate / "Cargo.toml").write_text( + '[package]\nname = "sqlbuild-python"\nversion = "0.1.0"\nedition = "2021"\n' + ) + (crate / "src/lib.rs").write_text("mod observations;\nmod planner;\n") + (crate / "src/observations/mod.rs").write_text( + "pub fn table() -> &'static str { OBSERVATIONS_TABLE_NAME }\n" + "pub struct SyncedCopyAdapter;\n" + ) + (crate / "src/planner.rs").write_text(planner) + entries: str = ", ".join(f'"{path}=Writes observations."' for path in allowed) + (root / "fensu.toml").write_text( + '[targets.rust]\nanalyzer = "rust"\nroots = ["crates"]\n' + 'rule_modules = ["scripts.fensu_policy.rules.observation_boundary"]\n' + f'select = ["{rule}"]\n\n' + f"[targets.rust.rule_options.{rule}]\nallowed_paths = [{entries}]\n" + ) + + +def check_rust(*, root: Path) -> subprocess.CompletedProcess[str]: + """Run `fensu check` on the Rust target without a cache.""" + + return subprocess.run( + [ + str(Path(sys.executable).with_name("fensu")), + "check", + "--target", + "rust", + "--no-cache", + "--no-color", + ], + cwd=root, + capture_output=True, + text=True, + timeout=30, + check=False, + ) diff --git a/tests/integration/scripts/fensu_policy/test_observation_boundary.py b/tests/integration/scripts/fensu_policy/test_observation_boundary.py new file mode 100644 index 0000000000..70bdb1ad5d --- /dev/null +++ b/tests/integration/scripts/fensu_policy/test_observation_boundary.py @@ -0,0 +1,87 @@ +"""Exercise the observation and synced-copy boundaries through Fensu's real Rust analyzer.""" + +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.scripts.fensu_policy._test_types import ObservationBoundaryTestCase +from tests.integration.scripts.fensu_policy.helpers import check_rust, write_observation_project + + +@pytest.mark.parametrize( + "test_case", + [ + ObservationBoundaryTestCase( + description="planner mentioning observations only in a comment", + rule="XSB078", + allowed=("crates/sqlbuild-python/src/observations",), + planner="// OBSERVATIONS_TABLE_NAME is display-only\npub fn plan() {}\n", + expected_returncode=0, + expected_faults=(), + ), + ObservationBoundaryTestCase( + description="planner matching the observations history kind", + rule="XSB078", + allowed=("crates/sqlbuild-python/src/observations",), + planner=( + "pub fn plan() {}\n" + "pub fn mirrored(kind: Kind) -> bool { kind == WarehouseHistoryKind::Observations }\n" + ), + expected_returncode=1, + expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:2:0",), + ), + ObservationBoundaryTestCase( + description="planner reading the observation table by its SQL name", + rule="XSB078", + allowed=("crates/sqlbuild-python/src/observations",), + planner='pub const READ: &str = "SELECT value FROM _sqlbuild_observations";\n', + expected_returncode=1, + expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:1:0",), + ), + ObservationBoundaryTestCase( + description="allowed path that no longer names observation state", + rule="XSB078", + allowed=( + "crates/sqlbuild-python/src/observations", + "crates/sqlbuild-python/src/planner.rs", + ), + planner="pub fn plan() {}\n", + expected_returncode=1, + expected_faults=("allowed path crates/sqlbuild-python/src/planner.rs no longer names",), + ), + ObservationBoundaryTestCase( + description="live planner naming the synced copy only in a comment", + rule="XSB079", + allowed=("crates/sqlbuild-python/src/observations",), + planner="// SyncedCopyAdapter serves plan previews only\npub fn plan() {}\n", + expected_returncode=0, + expected_faults=(), + ), + ObservationBoundaryTestCase( + description="live planner branching on the synced copy", + rule="XSB079", + allowed=("crates/sqlbuild-python/src/observations",), + planner=( + "pub fn plan() {}\n" + "pub fn preview(adapter: &Any) -> bool { adapter.is::() }\n" + ), + expected_returncode=1, + expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:2:0",), + ), + ], + ids=lambda case: case.description, +) +def test_given_rust_naming_preview_only_state_when_checking_then_only_unallowed_code_faults( + tmp_path: Path, test_case: ObservationBoundaryTestCase +) -> None: + write_observation_project( + root=tmp_path, rule=test_case.rule, allowed=test_case.allowed, planner=test_case.planner + ) + + result: subprocess.CompletedProcess[str] = check_rust(root=tmp_path) + + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert result.stdout.count(test_case.rule) == len(test_case.expected_faults), result.stdout + assert all(fault in result.stdout for fault in test_case.expected_faults), result.stdout + assert "src/observations/mod.rs" not in result.stdout diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index e542ca01a9..a0c1b3b2ae 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -374,3 +374,13 @@ class AdhocRunTestCase: job_settings: str = "" expected_scheduled_status: str = "success" expected_worker_log: str = "" + + +@dataclass(frozen=True) +class PlanPreviewTestCase: + description: str + edited_files: dict[str, str] + injected_sql: tuple[str, ...] + expected_window_models: tuple[str, ...] + expected_unplanned_cursor_models: tuple[str, ...] + expected_added_unknowns: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py index 65f57034c2..417c30573d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -87,3 +87,28 @@ def wait_for_statistics( records = [json.loads(row[0]) for row in rows] time.sleep(0.05) return records + + +def request_plan(control: Path, selection: list[str]) -> str: + """Queue the plan work a person's Plan creates, frozen to exact node keys.""" + request_id: str = str(uuid4()) + now: datetime = datetime.now(UTC) + with closing(sqlite3.connect(control)) as database: + _ = database.execute( + "INSERT INTO sqb_location_work (request_id,location_id,target,snapshot_id,generation," + "request_json,requested_by,requested_at,expires_at) SELECT ?,c.location_id,c.target," + "c.snapshot_id,s.generation,json_object('request_id',?,'location_id',c.location_id," + "'target',c.target,'reason','Plan','work',json_object('kind','plan','command'," + "json('[\"build\"]'),'selection',json(?),'settings',json_object()))," + "'test-admin',?,? FROM sqb_current_snapshots c JOIN sqb_snapshots s " + "ON s.snapshot_id=c.snapshot_id", + ( + request_id, + request_id, + json.dumps(selection), + now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + (now + timedelta(minutes=5)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + ), + ) + database.commit() + return request_id diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index aa7f70188d..6e041ea16f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -40,6 +40,7 @@ OldNameViewEventType, StateSqlValueType, build_consumption_create_table_sql, + build_observations_create_table_sql, render_state_sql_literal, write_fingerprint, write_migration_state_event, @@ -317,6 +318,39 @@ def append_consumption_history(adapter: Any, connection: Any, fixture: dict[str, ) +def append_observation_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Observation rows in the real table layout; retried appends repeat identical rows.""" + _ = adapter.execute( + connection=connection, + sql=build_observations_create_table_sql( + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ), + ) + columns: list[str] = list(fixture["record"])[2:] + types: dict[str, StateSqlValueType] = dict.fromkeys(columns, StateSqlValueType.STRING) | { + "observed_at": StateSqlValueType.TEXT_TIMESTAMP + } + rows: list[dict[str, Any]] = [ + row | {"observed_at": datetime.fromisoformat(row["observed_at"])} + for row in identified_rows(fixture) + ] + for row in rows + [rows[0]] * fixture["duplicate_rows"]: + values: list[str] = [ + render_state_sql_literal( + value=row[column], declared_type=types[column], timestamp_type="TIMESTAMP" + ) + for column in columns + ] + _ = adapter.execute( + connection=connection, + sql=f"INSERT INTO main._sqlbuild_observations ({', '.join(columns)}) " + f"VALUES ({', '.join(values)})", + ) + + def identified_rows(fixture: dict[str, Any]) -> list[dict[str, Any]]: """The fixture record once per source identity, with its per-identity overrides.""" return [ @@ -1474,3 +1508,32 @@ def security_audit_rows(control_store: Path) -> int: """Rows in the security audit, so a test can show a request recorded nothing.""" with sqlite3.connect(control_store) as control: return control.execute("SELECT count(*) FROM sqb_security_audit").fetchone()[0] + + +def run_sqb(project: Path, *arguments: str) -> subprocess.CompletedProcess[str]: + """Run this environment's `sqb` on `project` without raising on failure.""" + return subprocess.run( + [str(Path(sys.executable).with_name("sqb")), "--project-dir", str(project), *arguments], + capture_output=True, + text=True, + check=False, + ) + + +def live_plan(project: Path, selection: list[str]) -> subprocess.CompletedProcess[str]: + """Run `sqb plan --json` against the warehouse for node keys such as `model:orders`.""" + selected: list[str] = [] + for key in selection: + selected.extend(("--select", key.split(":")[1])) + return run_sqb(project, "plan", "--target", "development", "--json", "--no-color", *selected) + + +def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: + """Hold the warehouse's write lock, so any process that opens it fails.""" + deadline: float = time.monotonic() + 15 + while True: + try: + return duckdb.connect(database) + except duckdb.IOException: + assert time.monotonic() < deadline + time.sleep(0.05) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py new file mode 100644 index 0000000000..9c4f1d7e4f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -0,0 +1,271 @@ +"""A location plans from the server's synced copy and never opens the warehouse to do it.""" + +import json +import os +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path +from textwrap import dedent +from typing import Any + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + PlanPreviewTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + request_plan, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + hold_write_lock, + live_plan, + location_process, + run_sqb, + snapshot_api, + wait_for_generation, + wait_for_location_work, + wait_for_warehouse_sync_health, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + +_EXPRESSION: str = "SELECT * FROM (VALUES (1, 100)) AS refunds(refund_id, amount_cents)" +_SOURCES: str = dedent( + """ + sources: + - name: raw_refunds + description: Refunds described from an inline query. + expression: | + {expression} + type_enforcement: true + freshness: + column: refund_id + columns: + - name: refund_id + type: INTEGER + - name: amount_cents + type: INTEGER + """ +).strip() +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n[connections.local]\ndatabase = "warehouse.db"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": "", + "seeds/raw_orders.csv": "id,ordered_at\n1,2026-01-01 00:00:00\n2,2026-01-01 01:00:00\n", + "seeds/raw_orders.yml": dedent( + """ + seeds: + - name: raw_orders + description: Raw orders. + columns: + - name: id + type: INTEGER + - name: ordered_at + type: TIMESTAMP + """ + ).strip() + + "\n", + "models/orders.sql": dedent( + """ + MODEL (description "Orders by timestamp cursor.", + materialized incremental, + incremental_strategy append, + cursor ordered_at, + cursor_type timestamp, + cursor_grain second, + ); + + SELECT id, ordered_at FROM __seed("raw_orders") + """ + ).strip() + + "\n", + "models/daily_orders.sql": dedent( + """ + MODEL (description "Daily orders rebuilt over a lookback.", + materialized incremental, + incremental_strategy delete_insert, + cursor ordered_at, + cursor_type timestamp, + cursor_grain second, + lookback 1d, + ); + + SELECT id, ordered_at FROM __seed("raw_orders") + WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() + """ + ).strip() + + "\n", + "models/order_count.sql": dedent( + """ + MODEL (description "Order count without a cursor.", materialized table); + + SELECT COUNT(*) AS order_count FROM __seed("raw_orders") + """ + ).strip() + + "\n", + "models/refunds.sql": dedent( + """ + MODEL (description "Refunds.", materialized table); + + SELECT refund_id, amount_cents FROM __source("raw_refunds") + """ + ).strip() + + "\n", + "sources/raw.yml": _SOURCES.format(expression=_EXPRESSION) + "\n", +} +_SELECTION: list[str] = [ + "model:daily_orders", + "model:order_count", + "model:orders", + "model:refunds", +] +_UNKNOWN: list[str] = [ + "source freshness", + "retention", + "column rename support", + "sources outside mirrored schemas", +] +_REPLAYED: str = ( + "cursor window of daily_orders (the run starts it away from the last build's cursor)" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PlanPreviewTestCase( + description="matching observations plan like the live warehouse", + edited_files={}, + injected_sql=(), + expected_window_models=("orders",), + expected_unplanned_cursor_models=(), + expected_added_unknowns=(_REPLAYED,), + ), + PlanPreviewTestCase( + description="a cursor_max from no successful build is not a cursor position", + edited_files={}, + injected_sql=( + "INSERT INTO main._sqlbuild_observations VALUES ('injected-orders', 'cursor_max', " + "'orders', 'timestamp', '2099-01-01 00:00:00', 'injected-run', " + "'2099-01-01 00:00:00')", + ), + expected_window_models=(), + expected_unplanned_cursor_models=("orders",), + expected_added_unknowns=( + "cursor position of orders (no successful build recorded it)", + _REPLAYED, + ), + ), + PlanPreviewTestCase( + description="an edited expression plans its declared columns and says so", + edited_files={ + "sources/raw.yml": _SOURCES.format(expression=f"{_EXPRESSION} WHERE true") + "\n" + }, + injected_sql=(), + expected_window_models=("orders",), + expected_unplanned_cursor_models=(), + expected_added_unknowns=( + _REPLAYED, + "query columns of expression source raw_refunds (planned as declared)", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_synced_location_when_a_plan_is_requested_then_it_plans_from_the_copy_without_the_warehouse( + test_case: PlanPreviewTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files(project_dir=project, files=_PROJECT_FILES) + built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + with duckdb.connect(project / "warehouse.db") as warehouse: + for statement in test_case.injected_sql: + _ = warehouse.execute(statement) + write_files(project_dir=project, files=test_case.edited_files) + live_run: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION) + assert live_run.returncode == 0, live_run.stderr + live: dict[str, Any] = json.loads(live_run.stdout) + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + started: datetime = datetime.now(UTC) + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + _ = wait_for_warehouse_sync_health( + api, + {"failure": None, "tables": ["catalog", "consumption", "fingerprints", "observations"]}, + location, + log, + after=started, + ) + lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db") + try: + refused: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION) + request_id: str = request_plan(control, _SELECTION) + result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) + finally: + lock.close() + + assert refused.returncode != 0 + outcome: dict[str, Any] = result["result_json"]["outcome"] + assert outcome["kind"] == "plan", outcome + preview: dict[str, Any] = outcome["document"] + assert "synced_copy" in preview, log.read_text() + synced: dict[str, Any] = preview.pop("synced_copy") + assert synced["synced_at"] is not None + assert synced["unknown_until_run"] == [*_UNKNOWN, *test_case.expected_added_unknowns] + live_models: dict[str, dict[str, Any]] = {model["name"]: model for model in live.pop("models")} + preview_models: dict[str, dict[str, Any]] = { + model["name"]: model for model in preview.pop("models") + } + live_replayed: dict[str, Any] = live_models.pop("daily_orders") + preview_replayed: dict[str, Any] = preview_models.pop("daily_orders") + assert live_replayed["cursor"]["resolved_bounds"]["start"] is not None + assert (preview_replayed["action"], preview_replayed["reason"]) == ( + live_replayed["action"], + live_replayed["reason"], + ) + live_freshness: dict[str, Any] = live["metadata"].pop("direct_source_freshness") + preview_freshness: dict[str, Any] = preview["metadata"].pop("direct_source_freshness") + assert live_freshness["observed_source_names"] == ["raw_refunds"] + assert preview_freshness["unknown_source_details"] == [ + { + "message": "a plan preview does not observe source freshness", + "reason": "error", + "source_name": "raw_refunds", + } + ] + assert synced["cursor_window_starts"] == { + name: live_models[name]["cursor_bounds"]["start"] + for name in test_case.expected_window_models + } + for name in (*test_case.expected_window_models, *test_case.expected_unplanned_cursor_models): + for model in (live_models[name], preview_models.pop(name)): + assert model["action"] == "incremental_append" + assert model["reason"] == "normal_incremental" + _ = live_models.pop(name) + assert preview_models == live_models + assert preview == live diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index e59d7ec479..488c18d903 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -57,6 +57,7 @@ append_microbatch_history, append_migration_history, append_node_result_history, + append_observation_history, append_old_name_history, append_retried_fingerprints, location_process, @@ -162,6 +163,22 @@ "external-run", (), ), + ( + "duckdb", + "database", + "observation_history", + append_observation_history, + "external-run", + (), + ), + ( + "sqlite", + "path", + "observation_history", + append_observation_history, + "external-run", + (), + ), ( "duckdb", "database", @@ -331,6 +348,22 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and "external-run", (), ), + ( + "duckdb", + "database", + "observation_history", + append_observation_history, + "external-run", + (), + ), + ( + "sqlite", + "path", + "observation_history", + append_observation_history, + "external-run", + (), + ), ] ], ids=lambda case: case.description, diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index 5930b21bc8..78965a49c0 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -34,7 +34,8 @@ export function planResultOf(outcome: Wire['LocationWorkOutcome']): PlanResult { migrations: list(document.migrations), column_migrations: list(document.column_migrations), table_type_conversions: list(document.table_type_conversions), - retention: list(document.retention) + retention: list(document.retention), + synced_copy: document.synced_copy ?? null }, error: null }; diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index 57d5ff7ead..c03592d390 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -538,16 +538,28 @@ export class MockOperate { if (check.error) return this.#ctx.reply({ plan: null, error: check.error }); if (check.verb !== 'build') return this.#ctx.reply({ plan: null, error: `sqb plan previews builds; sqb ${check.verb} cannot be planned` }); const reason = command.includes('--full-refresh') ? 'full_refresh' : 'no_change'; - const models = check.selection.filter((key) => key.startsWith('model:')).map((key) => ({ - name: key.slice('model:'.length), - materialization_type: 'table', - action: 'create_table', - reason, - backfill: { action: reason === 'full_refresh' ? 'full' : 'forward', duration: null }, - qualified_name: `main.${key.slice('model:'.length)}` - })); + const modelKinds = new Set(['view', 'table', 'incremental']); + const models = check.selection.filter((key) => modelKinds.has(snapshotNode(key)?.kind ?? '')).map((key) => { + const incremental = snapshotNode(key)?.kind === 'incremental'; + return { + name: key, + materialization_type: incremental ? 'incremental' : 'table', + action: incremental ? 'incremental_append' : 'create_table', + reason: incremental && reason === 'no_change' ? 'normal_incremental' : reason, + backfill: { action: reason === 'full_refresh' ? 'full' : 'forward', duration: null }, + qualified_name: `main.${key}`, + ...(incremental ? { incremental_strategy: 'append', cursor_column: 'updated_at', cursor_type: 'timestamp' } : {}) + }; + }); + // Locations plan from the server's synced copy: incremental windows start at the synced cursor. + const windowStart = new Date(Date.now() - 3_600_000).toISOString().slice(0, 19).replace('T', ' '); + const synced_copy = { + synced_at: new Date(Date.now() - 300_000).toISOString(), + cursor_window_starts: Object.fromEntries(models.filter((m) => m.materialization_type === 'incremental' && reason !== 'full_refresh').map((m) => [m.name, windowStart])), + unknown_until_run: ['source freshness', 'retention', 'column rename support', 'sources outside mirrored schemas'] + }; return this.#ctx.reply({ - plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [] }, + plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [], synced_copy }, error: null }); } diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 72bc25ccd7..1b41832ec0 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -529,6 +529,18 @@ export type PlanDocument = { column_migrations: { model: string; origin_column: string; destination_column: string; decision: string }[]; table_type_conversions: { model: string; actual_type?: string; desired_type?: string }[]; retention: { model: string; action?: string }[]; + /** What a location's preview planned from instead of the warehouse; `null` for a live plan. */ + synced_copy: PlanSyncedCopy | null; +}; + +/** The server's synced copy of warehouse metadata a location's plan preview planned from. */ +export type PlanSyncedCopy = { + /** When every table the preview reads last synced; `null` until all of them have. */ + synced_at: string | null; + /** Each incremental model's window start; its end is the latest data when the run starts. */ + cursor_window_starts: Record; + /** What the preview could not know, which the run plans live. */ + unknown_until_run: string[]; }; /** The location's plan of a build command, or why it could not plan it. */ diff --git a/ui/src/lib/run/plan-report.svelte b/ui/src/lib/run/plan-report.svelte index 044b2a33fe..23d93fa2bf 100644 --- a/ui/src/lib/run/plan-report.svelte +++ b/ui/src/lib/run/plan-report.svelte @@ -3,6 +3,8 @@ // then the routine models. import { to } from '$lib/paths'; import type { PlanDocument } from '$lib/api'; + import { formatAgo } from '$lib/format/time'; + import { app } from '$lib/state/app.svelte'; import { planSections, type PlanEntry } from './plan-text'; let { plan }: { plan: PlanDocument } = $props(); @@ -17,6 +19,21 @@
+ {#if plan.synced_copy} + {@const copy = plan.synced_copy} +
+

+ Planned from warehouse metadata {#if copy.synced_at}synced {:else}that has not fully synced yet{/if}. + The run plans again against the warehouse. +

+ {#if copy.unknown_until_run.length} +
+ Unknown until the run +
    {#each copy.unknown_until_run as item (item)}
  • {item}
  • {/each}
+
+ {/if} +
+ {/if} {#each sections as s (s.title)} {@const shown = expanded[s.title] ? s.entries : s.entries.slice(0, limit)}
@@ -172,6 +189,37 @@ .warning.err { border-left-color: var(--sb-error); } + .preview { + display: flex; + flex-direction: column; + gap: 6px; + padding: 8px 10px; + border: 1px solid var(--border); + border-left: 2px solid var(--primary); + border-radius: 2px; + background: var(--sb-inset); + color: var(--muted-foreground); + } + .preview p { + margin: 0; + } + .preview time { + color: var(--foreground); + } + .unknown span { + color: var(--sb-text-faint); + font-size: 11.5px; + } + .unknown ul { + display: flex; + flex-wrap: wrap; + gap: 4px 14px; + margin: 2px 0 0; + padding: 0; + list-style: none; + font-family: var(--font-mono); + font-size: 11px; + } code { color: var(--sb-text-faint); font-family: var(--font-mono); diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index 1ec8b747ba..dd73c21e65 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -22,6 +22,7 @@ const plan = (models: PlanModel[], over: Partial = {}): PlanDocume column_migrations: [], table_type_conversions: [], retention: [], + synced_copy: null, ...over }); @@ -57,6 +58,18 @@ describe('plan text, as sqb plan prints it', () => { ]); }); + it('shows a preview window from the synced cursor to the run, and keeps a range the plan resolved', () => { + const incremental = { materialization_type: 'incremental', incremental_strategy: 'append', cursor_type: 'timestamp', cursor_column: 'placed_at', action: 'incremental_append', reason: 'normal_incremental' } as const; + const resolved = { start: '2026-01-02 00:00:00', end: '2026-01-03 00:00:00' }; + const preview = plan([model('lines', incremental), model('events', { ...incremental, cursor_bounds: resolved })], { + synced_copy: { synced_at: '2026-01-03T00:05:00Z', cursor_window_starts: { lines: '2026-01-01 01:00:00', events: '2025-12-31 00:00:00' }, unknown_until_run: ['retention'] } + }); + expect(planSections(preview)[0].entries.map((e) => e.detail)).toEqual([ + ['cursor: placed_at (timestamp)', 'range: 2026-01-01 01:00:00 → latest data when the run starts'], + ['cursor: placed_at (timestamp)', 'range: 2026-01-02 00:00:00 → 2026-01-03 00:00:00'] + ]); + }); + it('collapses a plan that is all full refresh into counts by materialization', () => { const full = plan([model('a', { reason: 'full_refresh' }), model('b', { reason: 'full_refresh' }), model('v', { reason: 'full_refresh', materialization_type: 'view' })]); expect(planSections(full)).toEqual([{ title: 'Full refresh', changed: true, entries: [{ name: 'table', value: '2 models', detail: [] }, { name: 'view', value: '1 model', detail: [] }] }]); diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index f5c87c8cfd..5ebd9bd18c 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -45,7 +45,8 @@ export function actionText(m: PlanModel): string { return 'continue forward'; } -function cursorLines(m: PlanModel, showRange: boolean): string[] { +/** The cursor lines; a preview's `windowStart` stands in for a range it could not bound. */ +function cursorLines(m: PlanModel, showRange: boolean, windowStart?: string): string[] { const column = m.cursor_column; if (!column) return []; const lines = [`cursor: ${column}${m.cursor_type ? ` (${m.cursor_type})` : ''}`]; @@ -53,6 +54,7 @@ function cursorLines(m: PlanModel, showRange: boolean): string[] { if (requested && (requested.start || requested.end)) lines.push(`requested: ${requested.start ?? 'earliest available'} → ${requested.end ?? 'latest available'}`); const range = m.cursor?.resolved_bounds ?? m.cursor_bounds; if (showRange && range && (range.start || range.end)) lines.push(`range: ${range.start ?? '…'} → ${range.end ?? '…'}`); + else if (showRange && windowStart) lines.push(`range: ${windowStart} → latest data when the run starts`); return lines; } @@ -68,13 +70,13 @@ const SCHEMA_SIGNS: Record = { column_type_changed: { sign: '~', label: 'type changed' } }; -function changedEntry(m: PlanModel, reason: string): PlanEntry { - if (reason === 'first_run') return { name: m.name, value: materializationLabel(m), detail: cursorLines(m, true) }; +function changedEntry(m: PlanModel, reason: string, windowStart?: string): PlanEntry { + if (reason === 'first_run') return { name: m.name, value: materializationLabel(m), detail: cursorLines(m, true, windowStart) }; return { name: m.name, value: actionText(m), detail: [ - ...cursorLines(m, reason !== 'full_refresh'), + ...cursorLines(m, reason !== 'full_refresh', windowStart), ...policyLine(m), ...(m.changed_functions?.length ? [`cause: function ${m.changed_functions.join(', ')} changed`] : []) ], @@ -119,8 +121,9 @@ export function planSections(plan: PlanDocument): PlanSection[] { for (const m of active) counts.set(m.materialization_type, (counts.get(m.materialization_type) ?? 0) + 1); add('Full refresh', [...counts].map(([kind, n]) => ({ name: kind, value: `${n} ${n === 1 ? 'model' : 'models'}`, detail: [] })), true); } else { - for (const [reason, title] of REASON_GROUPS) add(title, active.filter((m) => m.reason === reason).map((m) => changedEntry(m, reason)), true); - add('Models', active.filter((m) => ROUTINE.has(m.reason)).map((m) => ({ name: m.name, value: materializationLabel(m), detail: cursorLines(m, true) }))); + const starts = plan.synced_copy?.cursor_window_starts ?? {}; + for (const [reason, title] of REASON_GROUPS) add(title, active.filter((m) => m.reason === reason).map((m) => changedEntry(m, reason, starts[m.name])), true); + add('Models', active.filter((m) => ROUTINE.has(m.reason)).map((m) => ({ name: m.name, value: materializationLabel(m), detail: cursorLines(m, true, starts[m.name]) }))); } add('Functions', plan.functions.filter((f) => !f.reason || f.reason === 'no_change').map(functionEntry)); const seedLabel: Record = { first_run: 'first_run', config_changed: 'seed_changed', no_change: 'current' }; diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index 2d68318502..92bf838f3b 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -1,7 +1,8 @@