From 261453369198a2438dab299fd55857d837f89db6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:41:23 +0000 Subject: [PATCH 01/17] feat(state): add the display-only observations state table --- .../src/janitor_planning/constants.rs | 2 +- crates/sqlbuild-core/src/constants.rs | 5 +- .../src/bindings/_helpers/cli/registration.rs | 1 + .../_helpers/exported_constants/constants.rs | 8 +- .../_helpers/janitor/janitor_facts.rs | 11 ++- .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/observations_state.rs | 82 +++++++++++++++++++ crates/sqlbuild-state/src/lib.rs | 1 + .../src/observations/constants.rs | 22 +++++ .../src/observations/main/mod.rs | 1 + .../main/prune_observations_sql.rs | 13 +++ crates/sqlbuild-state/src/observations/mod.rs | 2 + fensu.toml | 1 + src/sqlbuild/_native.pyi | 15 ++++ .../adapter/contract/classes/base_adapter.py | 24 ++++++ .../adapter/contract/classes/observations.py | 37 +++++++++ .../contract/classes/strict_adapter.py | 16 ++++ .../bigquery/classes/bigquery_adapter.py | 5 +- .../clickhouse/classes/clickhouse_adapter.py | 10 ++- .../databricks/classes/databricks_adapter.py | 5 +- .../adapters/duckdb/classes/duckdb_adapter.py | 3 +- .../motherduck/classes/motherduck_adapter.py | 3 +- .../postgres/classes/postgres_adapter.py | 5 +- .../snowflake/classes/snowflake_adapter.py | 14 +++- .../adapters/sqlite/classes/sqlite_adapter.py | 5 +- .../sqlserver/classes/sqlserver_adapter.py | 16 +++- 26 files changed, 292 insertions(+), 16 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs create mode 100644 crates/sqlbuild-state/src/observations/constants.rs create mode 100644 crates/sqlbuild-state/src/observations/main/mod.rs create mode 100644 crates/sqlbuild-state/src/observations/main/prune_observations_sql.rs create mode 100644 crates/sqlbuild-state/src/observations/mod.rs create mode 100644 src/sqlbuild/adapter/contract/classes/observations.py diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index c68aa2bfc7..533e9fefe6 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -3,7 +3,7 @@ use sqlbuild_core::constants::STATE_TABLE_NAMES; /// SQLBuild state tables the janitor never acts on. -pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 10] = STATE_TABLE_NAMES; +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 11] = STATE_TABLE_NAMES; /// Prefix of every strict janitor archive name. pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; /// Separator between archive name segments. diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 7eeb30d732..1dc5865b99 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -38,8 +38,10 @@ pub const MICROBATCH_TABLE_NAME: &str = "_sqlbuild_microbatches"; pub const AUDIT_RESULTS_TABLE_NAME: &str = "_sqlbuild_audit_results"; /// State table of janitor archive and drop events. pub const JANITOR_EVENTS_TABLE_NAME: &str = "_sqlbuild_janitor_events"; +/// State table of display-only observations, which plan previews read and planning never does. +pub const OBSERVATIONS_TABLE_NAME: &str = "_sqlbuild_observations"; /// Every state table SQLBuild writes in a target's state schema. -pub const STATE_TABLE_NAMES: [&str; 10] = [ +pub const STATE_TABLE_NAMES: [&str; 11] = [ FINGERPRINT_TABLE_NAME, MICROBATCH_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, @@ -50,6 +52,7 @@ pub const STATE_TABLE_NAMES: [&str; 10] = [ MIGRATION_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, + OBSERVATIONS_TABLE_NAME, ]; /// A zero-length cursor duration. pub const ZERO_DAY_DURATION: &str = "0d"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 21e77b8f61..9367712a19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -29,6 +29,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { plan_command::registration::register(module)?; freshness::registration::register(module)?; crate::bindings::_helpers::native_host::consumption_state::register(module)?; + crate::bindings::_helpers::native_host::observations_state::register(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; execution_history::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 13a47eadb0..c7ecfe0ed6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -20,9 +20,10 @@ use sqlbuild_core::command_output::constants::{ use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, - NODE_RESULTS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, - PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, - STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, + NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, + PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SETTINGS_SECTION, + SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, + ZERO_DAY_DURATION, }; use sqlbuild_core::diagnostic_routing::constants::{ SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD, @@ -93,6 +94,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("NODE_RESULTS_TABLE_NAME", NODE_RESULTS_TABLE_NAME), ("AUDIT_RESULTS_TABLE_NAME", AUDIT_RESULTS_TABLE_NAME), ("JANITOR_EVENTS_TABLE_NAME", JANITOR_EVENTS_TABLE_NAME), + ("OBSERVATIONS_TABLE_NAME", OBSERVATIONS_TABLE_NAME), ("COLUMN_NODE_TYPE", COLUMN_NODE_TYPE), ("COLUMN_NODE_NAME", COLUMN_NODE_NAME), ("COLUMN_TARGET_DATABASE", COLUMN_TARGET_DATABASE), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index fceeda9567..9378dea579 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -25,7 +25,9 @@ const EXECUTION_LOGGER: &str = "sqlbuild.execution"; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; -use sqlbuild_core::constants::{CONSUMPTION_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_core::constants::{ + CONSUMPTION_TABLE_NAME, OBSERVATIONS_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, +}; /// The adapter and its open connection. pub(crate) struct Warehouse<'py> { @@ -475,7 +477,7 @@ pub(crate) fn gather_warehouse_facts<'py>( }) } -/// Fingerprint, source freshness and consumption history prune candidates for target schemas. +/// Fingerprint, source freshness, consumption and observation history prune candidates for target schemas. pub(crate) fn direct_state_prune_candidates<'py>( warehouse: &Warehouse<'py>, target_schemas: &[SchemaKey], @@ -495,6 +497,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( SOURCE_FRESHNESS_TABLE_NAME, )); locations.push((database.clone(), schema.clone(), CONSUMPTION_TABLE_NAME)); + locations.push((database.clone(), schema.clone(), OBSERVATIONS_TABLE_NAME)); } let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut candidates: Vec> = Vec::new(); @@ -512,6 +515,10 @@ pub(crate) fn direct_state_prune_candidates<'py>( CONSUMPTION_TABLE_NAME, "render_prune_consumption_history_sql", ), + ( + OBSERVATIONS_TABLE_NAME, + "render_prune_observations_history_sql", + ), ] { if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 3fb073463c..016fab107c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod interrupt; pub(crate) mod model_lookup; +pub(crate) mod observations_state; pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; pub(crate) mod orchestration_sensors; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs new file mode 100644 index 0000000000..5b64d4c19f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs @@ -0,0 +1,82 @@ +//! DDL and pruning of the display-only observations table through the adapter. + +use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::constants::OBSERVATIONS_TABLE_NAME; +use sqlbuild_state::observations::constants::{ + NULLABLE_OBSERVATION_COLUMNS, OBSERVATION_COLUMN_TYPES, +}; +use sqlbuild_state::observations::main::prune_observations_sql::prune_observations_sql; + +use crate::bindings::_helpers::native_host::consumption_state::state_table_name; +use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; + +/// Create-if-missing DDL for the observations table, with adapter-rendered column types. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] +fn build_observations_create_table_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + render_framework_type: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = database.py(); + let Some(table) = state_table_name( + render_qualified_name, + (database, schema), + OBSERVATIONS_TABLE_NAME, + )? + else { + return Ok(String::new()); + }; + let columns: Bound<'_, PyAny> = PyTuple::new( + py, + OBSERVATION_COLUMN_TYPES.iter().map(|(column, _)| *column), + )? + .into_any(); + let types: Bound<'_, PyDict> = PyDict::new(py); + for (column, declared) in OBSERVATION_COLUMN_TYPES { + types.set_item(column, declared)?; + } + let required: Vec<&str> = OBSERVATION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .filter(|column| !NULLABLE_OBSERVATION_COLUMNS.contains(column)) + .collect(); + state_table_create_sql( + &table, + ( + &columns, + &types.into_any(), + &PyFrozenSet::new(py, &required)?.into_any(), + ), + render_framework_type, + ) +} + +/// SQL deleting observations beyond the newest `retain_versions` per kind and node. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] +fn build_observations_prune_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + retain_versions: i64, +) -> PyResult { + Ok(state_table_name( + render_qualified_name, + (database, schema), + OBSERVATIONS_TABLE_NAME, + )? + .map(|table| prune_observations_sql(&table, retain_versions)) + .unwrap_or_default()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_observations_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_observations_prune_sql, module)?) +} diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index d0fcba2598..3fc9db2953 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -7,4 +7,5 @@ pub mod events; pub mod execution_history; pub mod freshness; pub mod microbatches; +pub mod observations; pub mod state_tables; diff --git a/crates/sqlbuild-state/src/observations/constants.rs b/crates/sqlbuild-state/src/observations/constants.rs new file mode 100644 index 0000000000..8f41fc7db8 --- /dev/null +++ b/crates/sqlbuild-state/src/observations/constants.rs @@ -0,0 +1,22 @@ +//! Columns and kinds of the display-only observations state table. + +use crate::freshness::constants::{STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE}; + +/// The column the warehouse clock fills on insert. +pub const OBSERVED_AT_COLUMN: &str = "observed_at"; +/// Columns in declaration order with their state value types; `observed_at` is last. +pub const OBSERVATION_COLUMN_TYPES: [(&str, &str); 7] = [ + ("observation_id", STRING_STATE_VALUE_TYPE), + ("kind", STRING_STATE_VALUE_TYPE), + ("node", STRING_STATE_VALUE_TYPE), + ("cursor_type", STRING_STATE_VALUE_TYPE), + ("value", STRING_STATE_VALUE_TYPE), + ("run_id", STRING_STATE_VALUE_TYPE), + (OBSERVED_AT_COLUMN, TEXT_TIMESTAMP_STATE_VALUE_TYPE), +]; +/// Columns that may be NULL: only `cursor_max` observations have a cursor type. +pub const NULLABLE_OBSERVATION_COLUMNS: [&str; 1] = ["cursor_type"]; +/// Observation of a model's cursor `MAX` after a successful run. +pub const CURSOR_MAX_KIND: &str = "cursor_max"; +/// Observation of the columns a live `DESCRIBE` returned for an expression source. +pub const SOURCE_COLUMNS_KIND: &str = "source_columns"; diff --git a/crates/sqlbuild-state/src/observations/main/mod.rs b/crates/sqlbuild-state/src/observations/main/mod.rs new file mode 100644 index 0000000000..6e65780594 --- /dev/null +++ b/crates/sqlbuild-state/src/observations/main/mod.rs @@ -0,0 +1 @@ +pub mod prune_observations_sql; diff --git a/crates/sqlbuild-state/src/observations/main/prune_observations_sql.rs b/crates/sqlbuild-state/src/observations/main/prune_observations_sql.rs new file mode 100644 index 0000000000..5f835804a0 --- /dev/null +++ b/crates/sqlbuild-state/src/observations/main/prune_observations_sql.rs @@ -0,0 +1,13 @@ +//! Prune old observations from one observations table. + +use crate::observations::constants::OBSERVED_AT_COLUMN; + +/// Delete every observation ranked beyond the newest `retain` per kind and node. +pub fn prune_observations_sql(qualified_name: &str, retain: i64) -> String { + format!( + "DELETE FROM {qualified_name} WHERE observation_id IN (SELECT observation_id FROM \ +(SELECT observation_id, ROW_NUMBER() OVER (PARTITION BY kind, node ORDER BY \ +{OBSERVED_AT_COLUMN} DESC, observation_id DESC) AS observation_rank FROM {qualified_name}) \ +AS ranked WHERE observation_rank > {retain})" + ) +} diff --git a/crates/sqlbuild-state/src/observations/mod.rs b/crates/sqlbuild-state/src/observations/mod.rs new file mode 100644 index 0000000000..aae420f1a5 --- /dev/null +++ b/crates/sqlbuild-state/src/observations/mod.rs @@ -0,0 +1,2 @@ +pub mod constants; +pub mod main; diff --git a/fensu.toml b/fensu.toml index e9afc99dd8..e65d291d22 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2832,6 +2832,7 @@ module_layers = [ "native_host=infra", "native_host/consumption_state=front", "native_host/model_lookup=project", + "native_host/observations_state=front", "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", "native_host/warehouse_metadata=commands", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a77ff62373..d758ee9ae0 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -190,6 +190,7 @@ COLUMN_MIGRATION_TABLE_NAME: str OLD_NAME_VIEW_TABLE_NAME: str MICROBATCH_TABLE_NAME: str NODE_RESULTS_TABLE_NAME: str +OBSERVATIONS_TABLE_NAME: str AUDIT_RESULTS_TABLE_NAME: str JANITOR_EVENTS_TABLE_NAME: str COLUMN_NODE_TYPE: str @@ -5502,6 +5503,20 @@ def build_consumption_prune_sql( render_qualified_name: Callable[..., str | None], retain_versions: int, ) -> str: ... +def build_observations_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def build_observations_prune_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + retain_versions: int, +) -> str: ... def build_source_freshness_create_table_sql( *, database: str | None, diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 030a92b905..7366fe01c9 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2301,6 +2301,18 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s render_framework_type=self.render_framework_type, ) + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the observations table when it is missing.""" + + from sqlbuild._native import build_observations_create_table_sql + + return build_observations_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_create_consumption_index_sqls( self, *, @@ -2476,6 +2488,18 @@ def render_prune_consumption_history_sql( del database, schema, retain_versions raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def render_prune_observations_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old observations.""" + + del database, schema, retain_versions + raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def sql_analysis_dialect(self) -> str | None: """Return the configured SQL analysis dialect name, if any.""" diff --git a/src/sqlbuild/adapter/contract/classes/observations.py b/src/sqlbuild/adapter/contract/classes/observations.py new file mode 100644 index 0000000000..91bcc7d7cf --- /dev/null +++ b/src/sqlbuild/adapter/contract/classes/observations.py @@ -0,0 +1,37 @@ +"""Default adapter implementations for the display-only observations state table.""" + +from __future__ import annotations + +from typing import Any, cast + + +class ObservationsMixin: + """Provide portable observations-table DDL and pruning for first-class warehouse adapters.""" + + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the observations table when it is missing.""" + + from sqlbuild._native import build_observations_create_table_sql + + adapter: Any = cast(Any, self) + return build_observations_create_table_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) + + def render_prune_observations_history_sql( + self, *, database: str | None, schema: str, retain_versions: int + ) -> str: + """Render SQL deleting observations beyond the newest `retain_versions` per node.""" + + from sqlbuild._native import build_observations_prune_sql + + adapter: Any = cast(Any, self) + return build_observations_prune_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + retain_versions=retain_versions, + ) diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 3504c8f4c4..1e40f0a464 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -1039,6 +1039,11 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s """Render DDL that creates the consumption table when it is missing.""" ... + @abstractmethod + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the observations table when it is missing.""" + ... + @abstractmethod def render_create_consumption_index_sqls( self, @@ -1147,6 +1152,17 @@ def render_prune_consumption_history_sql( """Render SQL that prunes old consumption executions.""" ... + @abstractmethod + def render_prune_observations_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old observations.""" + ... + @abstractmethod def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index bfc30561b6..4658bb818c 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -38,6 +38,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -151,7 +152,9 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class BigQueryAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class BigQueryAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """BigQuery adapter backed by google-cloud-bigquery.""" def get_columns_for_relations( diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 4155286972..cc8d367733 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -32,6 +32,7 @@ from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_snapshot_sql import render_is_distinct_from from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -154,7 +155,9 @@ ) -class ClickHouseAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class ClickHouseAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """ClickHouse adapter that builds tables aside and rewrites whole partitions.""" adapter_name: ClassVar[str] = BuiltinAdapter.CLICKHOUSE.value @@ -1926,6 +1929,11 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s super().render_create_consumption_table_sql(database=database, schema=schema) ) + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_observations_table_sql(database=database, schema=schema) + ) + def render_create_node_result_table_sql(self, *, database: str | None, schema: str) -> str: return _state_table_ddl( super().render_create_node_result_table_sql(database=database, schema=schema) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 6a4533c9da..b5a1c03c8a 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -39,6 +39,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -144,7 +145,9 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class DatabricksAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class DatabricksAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """Databricks adapter backed by databricks-sql-connector.""" def get_columns_for_relations( diff --git a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py index faf2a9effb..820c5d6447 100644 --- a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py +++ b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py @@ -10,11 +10,12 @@ from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.microbatches.constants import MICROBATCH_GENERATION_COMMENT_PREFIX -class DuckDbAdapter(MicrobatchMixin, ConsumptionMixin, DuckDbBackedAdapter): +class DuckDbAdapter(MicrobatchMixin, ConsumptionMixin, ObservationsMixin, DuckDbBackedAdapter): """DuckDB adapter allowing implicit main for targetless and named local targets.""" adapter_name: ClassVar[str] = BuiltinAdapter.DUCKDB.value diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index c3ef1b029d..6419cae9f0 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -11,6 +11,7 @@ from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.types import BuiltinAdapter, LoaderLogicalType from sqlbuild.adapters.motherduck.constants import ( @@ -22,7 +23,7 @@ ) -class MotherDuckAdapter(MicrobatchMixin, ConsumptionMixin, DuckDbBackedAdapter): +class MotherDuckAdapter(MicrobatchMixin, ConsumptionMixin, ObservationsMixin, DuckDbBackedAdapter): """MotherDuck adapter backed by DuckDB's MotherDuck connection support.""" adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index b0d22ccdc4..7907c13f32 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -39,6 +39,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -122,7 +123,9 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class PostgresAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class PostgresAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """PostgreSQL adapter backed by psycopg.""" def get_columns_for_relations( diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index b8fb21158e..a0469a709a 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -40,6 +40,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -197,7 +198,9 @@ _BULK_COLUMN_RELATION_CHUNK_SIZE: int = 2_000 -class SnowflakeAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class SnowflakeAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """Snowflake adapter backed by snowflake-connector-python.""" adapter_name: ClassVar[str] = BuiltinAdapter.SNOWFLAKE.value @@ -660,6 +663,15 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s schema=schema, ) + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + return self._with_state_table_retention( + create_sql=super().render_create_observations_table_sql( + database=database, schema=schema + ), + database=database, + schema=schema, + ) + def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: return self._with_state_table_retention( create_sql=super().render_create_microbatch_state_table_sql( diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 89ecddecf3..3ae20ff28b 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -44,6 +44,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -166,7 +167,9 @@ _FALSE_SEED_VALUES: frozenset[str] = frozenset({"false", "f", "no", "n", "0"}) -class SqliteAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class SqliteAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """SQLite adapter: one database file, SQLBuild schemas stored as `schema__name` tables.""" adapter_name: ClassVar[str] = BuiltinAdapter.SQLITE.value diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 0a7af7476b..f6ecbc9cf5 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -40,6 +40,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.observations import ObservationsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -133,7 +134,9 @@ def _render_tsql_distinct_condition(*, left: str, right: str) -> str: ) -class SqlServerAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): +class SqlServerAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter +): """Microsoft SQL Server adapter backed by pymssql.""" def get_columns_for_relations( @@ -504,6 +507,17 @@ def render_create_consumption_table_sql(self, *, database: str | None, schema: s key_columns=("target_schema", "target_name"), ) + def render_create_observations_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import OBSERVATIONS_TABLE_NAME + + return self._keyed_state_table_sql( + create_sql=super().render_create_observations_table_sql( + database=database, schema=schema + ), + table=(database, schema, OBSERVATIONS_TABLE_NAME), + key_columns=("kind", "node"), + ) + def render_create_consumption_index_sqls( self, *, database: str | None, schema: str ) -> tuple[str, ...]: From a54e9bcba95e90d8882ea1b75264de8e23b14161 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:04:31 +0000 Subject: [PATCH 02/17] feat(state): record cursor observations after successful builds After a build, each successful incremental model's cursor MAX is read with the planner's own cursor SQL and appended to the observations table in its state schema. Failures only warn. --- Makefile | 1 + .../_helpers/build_command/execution.rs | 10 ++ .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/native_host/consumption_state.rs | 10 +- .../native_host/observations_state.rs | 79 ++++++++- .../src/bindings/_helpers/observations/mod.rs | 1 + .../_helpers/observations/publication.rs | 167 ++++++++++++++++++ .../_helpers/planner_warehouse/cursors.rs | 15 +- fensu.toml | 1 + .../cli/commands/main/build/_test_types.py | 9 + .../main/build/test_cursor_observations.py | 119 +++++++++++++ 11 files changed, 399 insertions(+), 14 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py diff --git a/Makefile b/Makefile index 9e646384b7..6ed94078e8 100644 --- a/Makefile +++ b/Makefile @@ -147,6 +147,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_empty_cursor_inputs.py \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 824b6119f8..8bac241503 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -41,6 +41,7 @@ use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::consumption_state::recorder_callbacks; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::results::ReadResult; +use crate::bindings::_helpers::observations::publication::append_cursor_observations; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::fields::field as native_field; @@ -483,6 +484,15 @@ pub(crate) fn execute_build_plan<'py>( &connection_config, &native_field(&project, "run_id")?, )?; + for warning in append_cursor_observations( + (&project, &successful_models), + (&adapter, &connection_config), + &native_field(&project, "run_id")? + .str()? + .extract::()?, + ) { + emit(&context.stream, &format!("warning: {warning}\n"))?; + } lifecycle.finalize(py)?; Ok(Outcome { result: native_result, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 1b33b89fce..213941ba16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -67,6 +67,7 @@ pub(crate) mod microbatches; pub(crate) mod model_config; pub(crate) mod native_host; pub(crate) mod native_threads; +pub(crate) mod observations; pub(crate) mod output_capture; pub(crate) mod plan_command; pub(crate) mod plan_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index 4586fd655d..d385970b39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; /// Rows per `INSERT`; SQL Server accepts at most 1,000 rows in one `VALUES` list. -const ROWS_PER_STATEMENT: usize = 500; +pub(crate) const ROWS_PER_STATEMENT: usize = 500; /// Warehouse timestamp texts with a UTC offset, as drivers render aware values. const OFFSET_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f%:z", "%Y-%m-%d %H:%M:%S%.f%:z"]; /// Warehouse timestamp texts without an offset, which the consumption table stores as UTC. @@ -46,7 +46,7 @@ impl Location { } } - fn keywords<'py>(&self, py: Python<'py>) -> Vec<(&'static str, Bound<'py, PyAny>)> { + pub(crate) fn keywords<'py>(&self, py: Python<'py>) -> Vec<(&'static str, Bound<'py, PyAny>)> { vec![ ( "database", @@ -158,7 +158,7 @@ fn build_consumption_prune_sql( .unwrap_or_default()) } -fn execute<'py>( +pub(crate) fn execute<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, sql: &str, @@ -314,7 +314,7 @@ fn read_row(row: &Bound<'_, PyAny>) -> PyResult { }) } -fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { +pub(crate) fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { let value: Bound<'_, PyAny> = value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)); render_state_sql_literal(py, &value, STRING_STATE_VALUE_TYPE, None) @@ -392,7 +392,7 @@ impl ConsumptionConnection { } } - fn connected<'py>(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn connected<'py>(&self, py: Python<'py>) -> PyResult> { let mut open = self .connection .lock() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs index 5b64d4c19f..d2f68fa321 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs @@ -1,6 +1,6 @@ //! DDL and pruning of the display-only observations table through the adapter. -use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::OBSERVATIONS_TABLE_NAME; @@ -8,8 +8,14 @@ use sqlbuild_state::observations::constants::{ NULLABLE_OBSERVATION_COLUMNS, OBSERVATION_COLUMN_TYPES, }; use sqlbuild_state::observations::main::prune_observations_sql::prune_observations_sql; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; +use uuid::Uuid; -use crate::bindings::_helpers::native_host::consumption_state::state_table_name; +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::native_host::consumption_state::{ + Location, ROWS_PER_STATEMENT, execute, literal, state_table_name, +}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; /// Create-if-missing DDL for the observations table, with adapter-rendered column types. @@ -73,6 +79,75 @@ fn build_observations_prune_sql( .unwrap_or_default()) } +/// One observation to append; the warehouse clock fills `observed_at`. +pub(crate) struct ObservationAppend { + pub(crate) kind: &'static str, + pub(crate) node: String, + pub(crate) cursor_type: Option, + pub(crate) value: String, + pub(crate) run_id: String, +} + +/// Create the schema and observations table in `location` if missing, then append `rows`. +pub(crate) fn append_observations( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, + rows: &[ObservationAppend], +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + let keywords = location.keywords(py); + let Some(table) = state_table_name( + &native_field(adapter, "render_qualified_name")?, + (&keywords[0].1, &keywords[1].1), + OBSERVATIONS_TABLE_NAME, + )? + else { + return Ok(()); + }; + for statement in + call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? + { + execute(adapter, connection, &statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(adapter, "render_create_observations_table_sql")?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + execute(adapter, connection, &create)?; + } + let observed_at: String = + call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; + let columns: Vec<&str> = OBSERVATION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + for chunk in rows.chunks(ROWS_PER_STATEMENT) { + let cells: Vec> = chunk + .iter() + .map(|row| { + Ok(vec![ + literal(py, Some(&Uuid::now_v7().to_string()))?, + literal(py, Some(row.kind))?, + literal(py, Some(&row.node))?, + literal(py, row.cursor_type.as_deref())?, + literal(py, Some(&row.value))?, + literal(py, Some(&row.run_id))?, + observed_at.clone(), + ]) + }) + .collect::>()?; + execute( + adapter, + connection, + &values_insert_sql(&table, &columns, &cells), + )?; + } + Ok(()) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( build_observations_create_table_sql, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs new file mode 100644 index 0000000000..933500a237 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs @@ -0,0 +1 @@ +pub(crate) mod publication; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs new file mode 100644 index 0000000000..de7bbcef22 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs @@ -0,0 +1,167 @@ +//! Record each successful incremental model's cursor `MAX` after a build, for plan previews. + +use std::collections::{BTreeMap, HashSet}; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; +use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; +use sqlbuild_state::observations::constants::CURSOR_MAX_KIND; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::{model_reads, project_reads}; +use crate::bindings::_helpers::consumption::relation_reads::relation; +use crate::bindings::_helpers::native_host::consumption_state::{ConsumptionConnection, Location}; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::observations_state::{ + ObservationAppend, append_observations, +}; +use crate::bindings::_helpers::planner_phases::objects::config_str; +use crate::bindings::_helpers::planner_warehouse::constants::RUN_BOUNDED; +use crate::bindings::_helpers::planner_warehouse::cursors::{adapter_cursor_sql, bound_task}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Most concurrent cursor reads after a build, whatever the adapter allows. +const MAX_OBSERVATION_READS: i64 = 4; + +/// One successful incremental model's cursor read: its name, state location, cursor type and SQL. +struct CursorRead { + model: String, + location: Location, + cursor_type: String, + sql: String, +} + +/// The cursor read of every successful incremental model, rendered as the planner renders it. +fn cursor_reads( + project: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, + successful_models: &[String], +) -> PyResult> { + let succeeded: HashSet<&str> = successful_models.iter().map(String::as_str).collect(); + let mut reads: Vec = Vec::new(); + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let name: String = model_reads::name_object(&model)?.extract()?; + if !succeeded.contains(name.as_str()) + || config_str(&model, "materialized")?.as_deref() != Some("incremental") + { + continue; + } + let Some(cursor_column) = config_str(&model, "cursor")? else { + continue; + }; + let destination: Bound<'_, PyAny> = model_reads::destination(&model)?; + let qualified: Bound<'_, PyAny> = native_field(&destination, "qualified_name")?; + if qualified.is_none() || native_field(&destination, "schema")?.is_none() { + continue; + } + let Some(query) = cursor_query_groups(&[CursorTagRequest { + relation: qualified.extract()?, + cursor_column, + cursor_type: config_str(&model, "cursor_type")?, + min_tag: None, + max_tag: name.clone(), + }]) + .into_iter() + .next() else { + continue; + }; + reads.push(CursorRead { + location: Location::of(&relation(&destination)?), + sql: adapter_cursor_sql(adapter, &query)?, + cursor_type: query.cursor_type, + model: name, + }); + } + Ok(reads) +} + +/// Read each cursor `MAX` and append it, by state location; returns one warning per failure. +fn observe<'py>( + reads: &[CursorRead], + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + run_id: &str, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let execute: Bound<'py, PyAny> = native_field(adapter, "execute")?; + let tasks: Vec> = reads + .iter() + .map(|read| bound_task(py, read.sql.clone(), (connection, &execute))) + .collect::>()?; + let concurrency: i64 = native_field(adapter, "metadata_inspection_concurrency")? + .extract::()? + .clamp(1, MAX_OBSERVATION_READS); + let outcomes: Vec> = call_with( + &host::entry(py, RUN_BOUNDED)?, + &[ + ("tasks", pyo3::types::PyTuple::new(py, tasks)?.into_any()), + ("concurrency", concurrency.into_pyobject(py)?.into_any()), + ], + )? + .try_iter()? + .collect::>()?; + let mut warnings: Vec = Vec::new(); + let mut rows: BTreeMap> = BTreeMap::new(); + for (read, outcome) in reads.iter().zip(outcomes) { + let error: Bound<'py, PyAny> = outcome.get_item(2)?; + if !error.is_none() { + warnings.push(format!( + "could not read the cursor of model {}: {}; plan previews show it as unknown", + read.model, + error.str()? + )); + continue; + } + let row: Bound<'py, PyAny> = outcome.get_item(0)?; + if row.is_none() || row.get_item(0)?.is_none() { + continue; + } + rows.entry(read.location.clone()) + .or_default() + .push(ObservationAppend { + kind: CURSOR_MAX_KIND, + node: read.model.clone(), + cursor_type: Some(read.cursor_type.clone()), + value: row.get_item(0)?.str()?.extract()?, + run_id: run_id.to_owned(), + }); + } + for (location, rows) in rows { + if let Err(error) = append_observations(adapter, connection, &location, &rows) { + warnings.push(format!( + "could not record cursor observations in schema {}: {error}; plan previews show those cursors as unknown", + location.schema + )); + } + } + Ok(warnings) +} + +/// Record the cursor `MAX` of each successful incremental model; failures only warn. +pub(crate) fn append_cursor_observations( + (project, successful_models): (&Bound<'_, PyAny>, &[String]), + (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + run_id: &str, +) -> Vec { + let recorded: PyResult> = (|| { + let reads: Vec = cursor_reads(project, adapter, successful_models)?; + if reads.is_empty() { + return Ok(Vec::new()); + } + let py: Python<'_> = adapter.py(); + let connection = ConsumptionConnection::new(adapter, connection_config); + let observed: PyResult> = connection + .connected(py) + .and_then(|handle| observe(&reads, (adapter, &handle), run_id)); + let closed: PyResult<()> = connection.close(py); + let warnings: Vec = observed?; + closed?; + Ok(warnings) + })(); + recorded.unwrap_or_else(|error| { + vec![format!( + "could not record cursor observations: {error}; plan previews show cursors as unknown" + )] + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 32ff170938..b1eada0557 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -620,7 +620,7 @@ fn report_cursor_query( ) } -fn bound_task<'py>( +pub(crate) fn bound_task<'py>( py: Python<'py>, sql: String, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -697,12 +697,12 @@ fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResul } /// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. -fn adapter_cursor_sql( - request: &SnapshotRequest<'_>, +pub(crate) fn adapter_cursor_sql( + adapter: &Bound<'_, PyAny>, query: &PhysicalCursorQuery, ) -> PyResult { - let py = request.adapter.py(); - let render = native_field(&request.adapter, CURSOR_TEXT_METHOD)?; + let py = adapter.py(); + let render = native_field(adapter, CURSOR_TEXT_METHOD)?; let failure: RefCell> = RefCell::new(None); let sql: String = cursor_query_sql(query, &|expression: &str| { call_with( @@ -740,7 +740,8 @@ fn execute_queries<'py>( if !on_progress.is_none() { on_progress.call1((format!("Inspecting cursor bounds {identity}..."),))?; } - let outcome = read_cursor_query(py, &adapter_cursor_sql(request, query)?, handles)?; + let outcome = + read_cursor_query(py, &adapter_cursor_sql(&request.adapter, query)?, handles)?; report_cursor_query(py, query, (&outcome, &identity), on_progress)?; outcomes.push(outcome); } @@ -756,7 +757,7 @@ fn execute_queries<'py>( for query in queries { tasks.push(bound_task( py, - adapter_cursor_sql(request, query)?, + adapter_cursor_sql(&request.adapter, query)?, handles, )?); } diff --git a/fensu.toml b/fensu.toml index e65d291d22..391b437ff7 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2837,6 +2837,7 @@ module_layers = [ "native_host/orchestration_triggers=project", "native_host/warehouse_metadata=commands", "native_threads=infra", + "observations=plan", "output_capture=output", "plan_command=commands", "plan_output=plan", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index a6152e378e..ef7d3553d1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -363,6 +363,15 @@ class ModelBackedCursorBuildE2ETestCase: expected_full_refresh_counts: tuple[tuple[str, int], ...] = field(default_factory=tuple) +@dataclass(frozen=True) +class CursorObservationBuildE2ETestCase: + """Builds whose successful incremental models must record their cursor MAX as observations.""" + + description: str + builds: int + expected_observations: tuple[tuple[str, str, str, int], ...] + + @dataclass(frozen=True) class AppendCursorBuildE2ETestCase: """Test case for append cursor lower-bound behavior across reruns.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py new file mode 100644 index 0000000000..04ea20e6ec --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py @@ -0,0 +1,119 @@ +"""A build records the cursor MAX of each successful incremental model for plan previews.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + CursorObservationBuildE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": dedent( + """ + name = "cursor_observation_project" + adapter = "duckdb" + + [connection] + database = "observations.duckdb" + """ + ).strip() + + "\n", + "seeds/raw_orders.csv": "id,ordered_at\n1,2026-01-01 00:00:00\n2,2026-01-01 01:00:00\n", + "seeds/raw_orders.yml": dedent( + """ + seeds: + - name: raw_orders + description: Raw orders. + columns: + - name: id + type: INTEGER + - name: ordered_at + type: TIMESTAMP + """ + ).strip() + + "\n", + "models/orders.sql": dedent( + """ + MODEL (description "Orders by timestamp cursor.", + materialized incremental, + incremental_strategy append, + cursor ordered_at, + cursor_type timestamp, + cursor_grain second, + ); + + SELECT id, ordered_at + FROM __seed("raw_orders") + """ + ).strip() + + "\n", + "models/order_ids.sql": dedent( + """ + MODEL (description "Orders by integer cursor.", + materialized incremental, + incremental_strategy append, + cursor id, + cursor_type integer, + ); + + SELECT id FROM __seed("raw_orders") + """ + ).strip() + + "\n", + "models/order_count.sql": dedent( + """ + MODEL (description "Order count without a cursor.", materialized table); + + SELECT COUNT(*) AS order_count FROM __seed("raw_orders") + """ + ).strip() + + "\n", +} + + +@pytest.mark.parametrize( + "test_case", + [ + CursorObservationBuildE2ETestCase( + description="each build appends one cursor_max per incremental model", + builds=2, + expected_observations=( + ("order_ids", "integer", "2", 2), + ("orders", "timestamp", "2026-01-01 01:00:00", 2), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_incremental_models_when_building_then_cursor_max_observations_are_recorded( + tmp_path: Path, test_case: CursorObservationBuildE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="cursor_observations", repo_files=_PROJECT_FILES + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=("build",), project_dir=project_dir) for _ in range(test_case.builds) + ] + observations: list[tuple[object, ...]] = query_duckdb( + db_path=project_dir / "observations.duckdb", + sql=( + "SELECT node, cursor_type, value, COUNT(DISTINCT run_id) " + "FROM main._sqlbuild_observations WHERE kind = 'cursor_max' " + "GROUP BY node, cursor_type, value ORDER BY node" + ), + ) + + assert [result.returncode for result in results] == [0] * test_case.builds, results[-1].stderr + assert "cursor observations" not in "".join(result.stderr for result in results) + assert tuple(observations) == test_case.expected_observations From 66ad5e5f527232d5410a527e48866515e9e9b77e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:16:15 +0000 Subject: [PATCH 03/17] feat(state): record described source columns after builds A build keeps the columns planning described for each type-enforced expression source and appends them as source_columns observations in the state schema of every model reading the source. Plans record nothing. --- .../_helpers/build_command/command.rs | 13 ++- .../_helpers/build_command/execution.rs | 4 +- .../src/bindings/_helpers/observations/mod.rs | 1 + .../_helpers/observations/publication.rs | 84 ++++++++++++++++--- .../_helpers/observations/source_capture.rs | 32 +++++++ .../planner_warehouse/source_columns.rs | 10 ++- .../cli/commands/main/build/_test_types.py | 9 ++ .../main/build/test_cursor_observations.py | 63 +++++++++++++- 8 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 3f45271352..15c8a81788 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -21,6 +21,9 @@ use crate::bindings::_helpers::build_command::planning::{ use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::observations::source_capture::{ + arm_source_capture, take_source_capture, +}; use crate::bindings::_helpers::records::fields::field as native_field; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { @@ -56,8 +59,16 @@ fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) - within(&cache_scope, |_| run_cached_build(context, providers)) } -/// Plan and run the build, sharing one relation metadata cache from planning to the run's end. +/// Plan and run the build, capturing the source columns planning describes for its observations. fn run_cached_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { + arm_source_capture(); + let built: PyResult = run_captured_build(context, providers); + let _ = take_source_capture(); + built +} + +/// Plan and run the build, sharing one relation metadata cache from planning to the run's end. +fn run_captured_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; if let Some(worker) = &context.worker { worker.enforce_plan_writers(&pipeline)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 8bac241503..82ce827200 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -41,7 +41,7 @@ use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::consumption_state::recorder_callbacks; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::results::ReadResult; -use crate::bindings::_helpers::observations::publication::append_cursor_observations; +use crate::bindings::_helpers::observations::publication::append_build_observations; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::fields::field as native_field; @@ -484,7 +484,7 @@ pub(crate) fn execute_build_plan<'py>( &connection_config, &native_field(&project, "run_id")?, )?; - for warning in append_cursor_observations( + for warning in append_build_observations( (&project, &successful_models), (&adapter, &connection_config), &native_field(&project, "run_id")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs index 933500a237..eac8f7714f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs @@ -1 +1,2 @@ pub(crate) mod publication; +pub(crate) mod source_capture; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs index de7bbcef22..9dc3055a7b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs @@ -1,12 +1,13 @@ -//! Record each successful incremental model's cursor `MAX` after a build, for plan previews. +//! Record what a build observed, cursor `MAX` values and described source columns, for plan previews. -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use pyo3::IntoPyObject; +use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; -use sqlbuild_state::observations::constants::CURSOR_MAX_KIND; +use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_views::{model_reads, project_reads}; @@ -16,11 +17,14 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::observations_state::{ ObservationAppend, append_observations, }; +use crate::bindings::_helpers::observations::source_capture::take_source_capture; use crate::bindings::_helpers::planner_phases::objects::config_str; use crate::bindings::_helpers::planner_warehouse::constants::RUN_BOUNDED; use crate::bindings::_helpers::planner_warehouse::cursors::{adapter_cursor_sql, bound_task}; use crate::bindings::_helpers::records::fields::field as native_field; +/// Dependency kind of a source in a model's `dep_rows`. +const SOURCE_DEPENDENCY: &str = "source"; /// Most concurrent cursor reads after a build, whatever the adapter allows. const MAX_OBSERVATION_READS: i64 = 4; @@ -77,9 +81,65 @@ fn cursor_reads( Ok(reads) } -/// Read each cursor `MAX` and append it, by state location; returns one warning per failure. +/// One `source_columns` row per described source, in the state schema of each model reading it. +fn source_column_rows( + project: &Bound<'_, PyAny>, + described: &BTreeMap>, + run_id: &str, +) -> PyResult>> { + let mut readers: BTreeMap> = BTreeMap::new(); + if described.is_empty() { + return Ok(BTreeMap::new()); + } + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let destination: Bound<'_, PyAny> = model_reads::destination(&model)?; + if native_field(&destination, "schema")?.is_none() { + continue; + } + let sources: Vec = model_reads::dep_rows(&model)? + .into_iter() + .filter(|(kind, name)| kind == SOURCE_DEPENDENCY && described.contains_key(name)) + .map(|(_, name)| name) + .collect(); + if !sources.is_empty() { + readers + .entry(Location::of(&relation(&destination)?)) + .or_default() + .extend(sources); + } + } + let mut rows: BTreeMap> = BTreeMap::new(); + for (location, sources) in readers { + for source in sources { + let row: ObservationAppend = source_columns_row(source, described, run_id)?; + rows.entry(location.clone()).or_default().push(row); + } + } + Ok(rows) +} + +/// The `source_columns` observation of one described source, its column names as a JSON array. +fn source_columns_row( + source: String, + described: &BTreeMap>, + run_id: &str, +) -> PyResult { + let value: String = serde_json::to_string(&described[&source]) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + Ok(ObservationAppend { + kind: SOURCE_COLUMNS_KIND, + node: source, + cursor_type: None, + value, + run_id: run_id.to_owned(), + }) +} + +/// Read each cursor `MAX`, then append it with `rows`, by state location; one warning per failure. fn observe<'py>( reads: &[CursorRead], + mut rows: BTreeMap>, (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), run_id: &str, ) -> PyResult> { @@ -102,7 +162,6 @@ fn observe<'py>( .try_iter()? .collect::>()?; let mut warnings: Vec = Vec::new(); - let mut rows: BTreeMap> = BTreeMap::new(); for (read, outcome) in reads.iter().zip(outcomes) { let error: Bound<'py, PyAny> = outcome.get_item(2)?; if !error.is_none() { @@ -130,7 +189,7 @@ fn observe<'py>( for (location, rows) in rows { if let Err(error) = append_observations(adapter, connection, &location, &rows) { warnings.push(format!( - "could not record cursor observations in schema {}: {error}; plan previews show those cursors as unknown", + "could not record observations in schema {}: {error}; plan previews show them as unknown", location.schema )); } @@ -138,22 +197,25 @@ fn observe<'py>( Ok(warnings) } -/// Record the cursor `MAX` of each successful incremental model; failures only warn. -pub(crate) fn append_cursor_observations( +/// Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. +pub(crate) fn append_build_observations( (project, successful_models): (&Bound<'_, PyAny>, &[String]), (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), run_id: &str, ) -> Vec { + let described: BTreeMap> = take_source_capture().into_iter().collect(); let recorded: PyResult> = (|| { let reads: Vec = cursor_reads(project, adapter, successful_models)?; - if reads.is_empty() { + let sources: BTreeMap> = + source_column_rows(project, &described, run_id)?; + if reads.is_empty() && sources.is_empty() { return Ok(Vec::new()); } let py: Python<'_> = adapter.py(); let connection = ConsumptionConnection::new(adapter, connection_config); let observed: PyResult> = connection .connected(py) - .and_then(|handle| observe(&reads, (adapter, &handle), run_id)); + .and_then(|handle| observe(&reads, sources, (adapter, &handle), run_id)); let closed: PyResult<()> = connection.close(py); let warnings: Vec = observed?; closed?; @@ -161,7 +223,7 @@ pub(crate) fn append_cursor_observations( })(); recorded.unwrap_or_else(|error| { vec![format!( - "could not record cursor observations: {error}; plan previews show cursors as unknown" + "could not record observations: {error}; plan previews show them as unknown" )] }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs new file mode 100644 index 0000000000..5006cdd805 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs @@ -0,0 +1,32 @@ +//! The expression-source columns a build's planning described, kept for its observations. + +use std::cell::RefCell; + +/// Each described source's name and column names, in the order planning described them. +pub(crate) type DescribedSources = Vec<(String, Vec)>; + +thread_local! { + /// Per thread, since a build plans and records on the thread that runs it; `None` when not armed. + static CAPTURED: RefCell> = const { RefCell::new(None) }; +} + +/// Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. +pub(crate) fn arm_source_capture() { + CAPTURED.with(|slot| *slot.borrow_mut() = Some(Vec::new())); +} + +/// Keep one source's described column names when a build armed the capture. +pub(crate) fn capture_source_columns(source: &str, columns: &[String]) { + CAPTURED.with(|slot| { + if let Some(captured) = slot.borrow_mut().as_mut() { + captured.push((source.to_owned(), columns.to_vec())); + } + }); +} + +/// The captured source columns, disarming the capture. +pub(crate) fn take_source_capture() -> DescribedSources { + CAPTURED + .with(|slot| slot.borrow_mut().take()) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index b99fd5cebe..613b1842d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::observations::source_capture::capture_source_columns; use crate::bindings::_helpers::planner_phases::objects::{frozenset, text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ COLUMN_INFO, METADATA_NAME_FILTER_LIMIT, RELATION_LOOKUP, @@ -97,13 +98,18 @@ pub(crate) fn source_inspection<'py>( &[("connection", connection.clone()), ("sql", expression)], )?; let mut columns: Vec<_> = Vec::new(); + let mut described: Vec = Vec::new(); for name in names.try_iter()? { + let name: Bound<'py, PyAny> = name?; + described.push(name.extract()?); columns.push(call_with( &column_info, - &[("name", name?), ("type", text(py, ""))], + &[("name", name), ("type", text(py, ""))], )?); } - result.set_item(native_field(entry, "name")?, tuple(py, columns)?)?; + let source: Bound<'py, PyAny> = native_field(entry, "name")?; + capture_source_columns(&source.extract::()?, &described); + result.set_item(source, tuple(py, columns)?)?; } continue; } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index ef7d3553d1..c1cf13ba76 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -372,6 +372,15 @@ class CursorObservationBuildE2ETestCase: expected_observations: tuple[tuple[str, str, str, int], ...] +@dataclass(frozen=True) +class SourceColumnObservationE2ETestCase: + """Commands after which each build, and no plan, must have recorded described source columns.""" + + description: str + commands: tuple[tuple[str, ...], ...] + expected_observations: tuple[tuple[str, str, int], ...] + + @dataclass(frozen=True) class AppendCursorBuildE2ETestCase: """Test case for append cursor lower-bound behavior across reruns.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py index 04ea20e6ec..9403f216a8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py @@ -1,4 +1,4 @@ -"""A build records the cursor MAX of each successful incremental model for plan previews.""" +"""A build records cursor MAX values and described source columns for plan previews.""" from __future__ import annotations @@ -10,6 +10,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( CursorObservationBuildE2ETestCase, + SourceColumnObservationE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( prepare_inline_project, @@ -70,6 +71,30 @@ """ ).strip() + "\n", + "sources/raw.yml": dedent( + """ + sources: + - name: raw_refunds + description: Refunds described from an inline query. + expression: | + SELECT * FROM (VALUES (1, 100)) AS refunds(refund_id, amount_cents) + type_enforcement: true + columns: + - name: refund_id + type: INTEGER + - name: amount_cents + type: INTEGER + """ + ).strip() + + "\n", + "models/refunds.sql": dedent( + """ + MODEL (description "Refunds.", materialized table); + + SELECT refund_id, amount_cents FROM __source("raw_refunds") + """ + ).strip() + + "\n", "models/order_count.sql": dedent( """ MODEL (description "Order count without a cursor.", materialized table); @@ -117,3 +142,39 @@ def test_given_incremental_models_when_building_then_cursor_max_observations_are assert [result.returncode for result in results] == [0] * test_case.builds, results[-1].stderr assert "cursor observations" not in "".join(result.stderr for result in results) assert tuple(observations) == test_case.expected_observations + + +@pytest.mark.parametrize( + "test_case", + [ + SourceColumnObservationE2ETestCase( + description="builds record described source columns and plans record none", + commands=(("build",), ("plan",), ("build",)), + expected_observations=(("raw_refunds", '["refund_id","amount_cents"]', 2),), + ), + ], + ids=lambda case: case.description, +) +def test_given_expression_source_when_building_then_source_column_observations_are_recorded( + tmp_path: Path, test_case: SourceColumnObservationE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="source_observations", repo_files=_PROJECT_FILES + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=command, project_dir=project_dir) for command in test_case.commands + ] + observations: list[tuple[object, ...]] = query_duckdb( + db_path=project_dir / "observations.duckdb", + sql=( + "SELECT node, value, COUNT(DISTINCT run_id) " + "FROM main._sqlbuild_observations WHERE kind = 'source_columns' " + "GROUP BY node, value ORDER BY node" + ), + ) + + assert [result.returncode for result in results] == [0] * len(test_case.commands), results[ + -1 + ].stderr + assert tuple(observations) == test_case.expected_observations From 8cdf18fc7fc31168f3e4374149bcc755ee08f5b6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:41:26 +0000 Subject: [PATCH 04/17] feat(orchestration): mirror build observations into the control store Location sync reads _sqlbuild_observations as its own history kind and the server keeps the rows in sqb_warehouse_observations, keyed by the observation identity, with kind and node columns for latest-value reads. --- .../orchestration/observation_history.json | 53 +++++++++++++++++ clients/typescript/src/schema.d.ts | 2 +- .../_helpers/resources/warehouse_metadata.rs | 2 + .../src/store/schema.sql | 15 +++++ .../main/ingest/ingest_observations.rs | 57 +++++++++++++++++++ .../src/warehouse_metadata/main/ingest/mod.rs | 1 + .../src/warehouse_metadata/models.rs | 29 +++++++++- .../tests/api/test_warehouse_metadata.rs | 5 ++ .../api/warehouse_metadata/events/helpers.rs | 3 +- .../native_host/warehouse_metadata.rs | 25 ++++++-- .../compiled_project/publication/helpers.py | 34 +++++++++++ .../publication/test_warehouse_metadata.py | 33 +++++++++++ 12 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/observation_history.json create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/observation_history.json b/benchmarks/duckshop/edge_cases/orchestration/observation_history.json new file mode 100644 index 0000000000..8d89110ffe --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/observation_history.json @@ -0,0 +1,53 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "observation_id": "observation-000", + "kind": "cursor_max", + "node": "orders_🧾", + "cursor_type": "timestamp", + "value": "2026-02-03 04:05:06.123456", + "run_id": "external-run", + "observed_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "page_lengths": [ + 100, + 7 + ], + "later_index": 106, + "source_count": 107, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id", + "retained_result": "observation-103", + "kind": "observations", + "source_key": "observation_id", + "timestamp_key": "observed_at", + "id_prefix": "observation", + "later_record": { + "observation_id": "observation-103", + "kind": "source_columns", + "node": "shop.orders_🧾", + "cursor_type": null, + "value": "[\"order_id\", \"amount_€\"]", + "observed_at": "2026-02-03T04:05:06.000001Z" + }, + "last_record": { + "observation_id": "observation-103", + "kind": "source_columns", + "node": "shop.orders_🧾", + "cursor_type": null, + "value": "[\"order_id\", \"amount_€\"]", + "observed_at": "2026-02-03T04:05:06.000001Z" + }, + "record_overrides": { + "observation-103": { + "kind": "source_columns", + "node": "shop.orders_🧾", + "cursor_type": null, + "value": "[\"order_id\", \"amount_€\"]", + "observed_at": "2026-02-03T04:05:06.000001Z" + } + } +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ccc5e5cc8a..8f20e063b6 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -3082,7 +3082,7 @@ export interface components { * @description Catalog observations require full scans; only immutable history supports a watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "catalog" | "statistics"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "observations" | "catalog" | "statistics"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs index 21aeb82684..302a7e4b5f 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs @@ -17,6 +17,7 @@ use crate::warehouse_metadata::main::ingest::ingest_janitor_events::ingest_janit use crate::warehouse_metadata::main::ingest::ingest_microbatches::ingest_microbatches; use crate::warehouse_metadata::main::ingest::ingest_migrations::ingest_migrations; use crate::warehouse_metadata::main::ingest::ingest_node_results::ingest_node_results; +use crate::warehouse_metadata::main::ingest::ingest_observations::ingest_observations; use crate::warehouse_metadata::main::ingest::ingest_old_name_views::ingest_old_name_views; use crate::warehouse_metadata::main::ingest::ingest_source_freshness::ingest_source_freshness; use crate::warehouse_metadata::main::ingest::ingest_statistics::ingest_statistics; @@ -56,6 +57,7 @@ pub(crate) async fn report( } WarehouseHistoryKind::NodeResults => ingest_node_results(&store, scope, &batch).await, WarehouseHistoryKind::Consumption => ingest_consumption(&store, scope, &batch).await, + WarehouseHistoryKind::Observations => ingest_observations(&store, scope, &batch).await, WarehouseHistoryKind::Catalog => ingest_catalog(&store, scope, &batch).await, WarehouseHistoryKind::Statistics => ingest_statistics(&store, scope, &batch).await, }?; diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql index 215b451ebb..eb7b972977 100644 --- a/crates/sqlbuild-orchestration/src/store/schema.sql +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -502,6 +502,21 @@ CREATE TABLE sqb_warehouse_consumption ( scan_id {uuid} ); CREATE INDEX sqb_consumption_namespace ON sqb_warehouse_consumption (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_observations ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + kind TEXT NOT NULL, + node TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_observations_namespace ON sqb_warehouse_observations (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE INDEX sqb_observations_latest ON sqb_warehouse_observations (location_id,target,kind,node,observed_at,event_id); CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( location_id, target, (record_json ->> 'source_name'), (record_json ->> 'target_database'), (record_json ->> 'target_schema'), diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs new file mode 100644 index 0000000000..c76912e03d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs @@ -0,0 +1,57 @@ +//! Mirror display-only build observations; planning reads the warehouse, never these rows. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseObservationRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_observations( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: record.observation_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.observed_at, + columns: vec![ + ("kind", Parameter::Text(record.kind.clone())), + ("node", Parameter::Text(record.node.clone())), + ], + record_json: json!(record), + }) + .collect(), + records + .iter() + .flat_map(|record| { + [ + &record.observation_id, + &record.state_schema, + &record.kind, + &record.node, + &record.run_id, + ] + }) + .any(|value| value.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs index aa95f66a37..e54d3083df 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs @@ -7,6 +7,7 @@ pub mod ingest_janitor_events; pub mod ingest_microbatches; pub mod ingest_migrations; pub mod ingest_node_results; +pub mod ingest_observations; pub mod ingest_old_name_views; pub mod ingest_source_freshness; pub mod ingest_statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 5281ecee88..f3d9e27fa6 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -7,7 +7,8 @@ use uuid::Uuid; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, - NODE_RESULTS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, }; use crate::store::errors::ControlError; @@ -270,6 +271,21 @@ pub struct WarehouseConsumptionRecord { pub recorded_at: DateTime, } +/// A display-only value a build observed, such as a cursor `MAX`; planning never reads it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseObservationRecord { + pub state_database: Option, + pub state_schema: String, + pub observation_id: String, + pub kind: String, + pub node: String, + pub cursor_type: Option, + pub value: String, + pub run_id: String, + pub observed_at: DateTime, +} + /// Catalog observations require full scans; only immutable history supports a watermark. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] @@ -284,13 +300,14 @@ pub enum WarehouseHistoryKind { ColumnMigrations, NodeResults, Consumption, + Observations, Catalog, Statistics, } impl WarehouseHistoryKind { /// Catalog and statistics stay last: a failed statistics read rewinds one kind, to catalog. - pub const ALL: [Self; 12] = [ + pub const ALL: [Self; 13] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, @@ -301,6 +318,7 @@ impl WarehouseHistoryKind { Self::ColumnMigrations, Self::NodeResults, Self::Consumption, + Self::Observations, Self::Catalog, Self::Statistics, ]; @@ -358,6 +376,11 @@ impl WarehouseHistoryKind { "sqb_warehouse_consumption", Some(CONSUMPTION_TABLE_NAME), ), + Self::Observations => ( + "observations", + "sqb_warehouse_observations", + Some(OBSERVATIONS_TABLE_NAME), + ), Self::Catalog => ("catalog", "sqb_warehouse_catalog", None), Self::Statistics => ("statistics", "sqb_warehouse_statistics", None), } @@ -540,6 +563,7 @@ pub enum WarehouseHistoryRead { ColumnMigrations(StateEventsRead), NodeResults(NodeResultsRead), Consumption(ConsumptionRead), + Observations(StateEventsRead), Catalog(CatalogRead), Statistics(StatisticsRead), } @@ -557,6 +581,7 @@ pub enum WarehouseHistoryRecord { ColumnMigration(Box), NodeResult(Box), Consumption(Box), + Observation(Box), Catalog(WarehouseCatalogRecord), Statistics(WarehouseStatisticsRecord), } diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 56536e0d3f..a367bd80d1 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -402,6 +402,11 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit "consumption_history", "execution_id", ), + ( + WarehouseHistoryKind::Observations, + "observation_history", + "observation_id", + ), ] { crate::api::tests::warehouse_events_helpers::event_history( store, diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs index c8f58887a4..6925eac3df 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs @@ -137,7 +137,8 @@ pub(crate) async fn event_history( table: kind.mirror_table(), operation: "INSERT", condition: "WHEN 'rejected-result' IN (NEW.record_json ->> 'event_id', NEW.record_json ->> 'result_id', \ - NEW.record_json ->> 'node_name', NEW.record_json ->> 'execution_id')", + NEW.record_json ->> 'node_name', NEW.record_json ->> 'execution_id', \ + NEW.record_json ->> 'observation_id')", sqlite_body: "SELECT RAISE(ABORT, 'injected audit failure')", postgres_body: "RAISE EXCEPTION 'injected audit failure'", }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index ec29ddfbb0..18b2a1ef0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -11,7 +11,8 @@ use sqlbuild_config::command_connection::constants::{ use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, - NODE_RESULTS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, }; use sqlbuild_core::models::SqlValue; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; @@ -26,6 +27,7 @@ use sqlbuild_state::events::constants::{ use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, }; +use sqlbuild_state::observations::constants::OBSERVATION_COLUMN_TYPES; use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; use sqlbuild_state::state_tables::models::HistoryRead; @@ -148,7 +150,8 @@ pub(crate) fn read_history<'py>( | WarehouseHistoryRead::Migrations(read) | WarehouseHistoryRead::OldNameViews(read) | WarehouseHistoryRead::Microbatches(read) - | WarehouseHistoryRead::ColumnMigrations(read) => &read.location, + | WarehouseHistoryRead::ColumnMigrations(read) + | WarehouseHistoryRead::Observations(read) => &read.location, WarehouseHistoryRead::NodeResults(read) => &read.location, WarehouseHistoryRead::Consumption(read) => &read.location, }; @@ -181,6 +184,7 @@ fn read_page<'py>( let (adapter, connection) = (&session.adapter, &session.connection); let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); let consumption_columns = CONSUMPTION_COLUMN_TYPES.map(|(name, _)| name); + let observation_columns = OBSERVATION_COLUMN_TYPES.map(|(name, _)| name); let event_columns: Vec = match request { WarehouseHistoryRead::AuditResults(_) => py .import("sqlbuild.executor.audit_results.constants")? @@ -294,6 +298,15 @@ fn read_page<'py>( consumption_columns.as_slice(), RECORDED_AT_COLUMN, ), + WarehouseHistoryRead::Observations(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + OBSERVATIONS_TABLE_NAME, + observation_columns.as_slice(), + OBSERVED_AT_COLUMN, + ), }; let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); if limit == 0 || limit > MAX_HISTORY_RECORDS { @@ -406,6 +419,7 @@ fn read_page<'py>( "input_version", RECORDED_AT_COLUMN, ]), + WarehouseHistoryRead::Observations(_) => Some(&["observation_id", OBSERVED_AT_COLUMN]), _ => None, }; let mut cursor_indexes: Vec = columns @@ -564,6 +578,7 @@ fn stream_page<'py>( WarehouseHistoryRecord::ColumnMigration(record) => record.created_at, WarehouseHistoryRecord::NodeResult(record) => record.ts, WarehouseHistoryRecord::Consumption(record) => record.recorded_at, + WarehouseHistoryRecord::Observation(record) => record.observed_at, }; if stream.since.is_none_or(|since| observed >= since) { records.push(record); @@ -635,7 +650,8 @@ fn unchanged_token(request: &WarehouseHistoryRead) -> Option<&str> { | WarehouseHistoryRead::Migrations(read) | WarehouseHistoryRead::OldNameViews(read) | WarehouseHistoryRead::Microbatches(read) - | WarehouseHistoryRead::ColumnMigrations(read) => read.unchanged_token.as_deref(), + | WarehouseHistoryRead::ColumnMigrations(read) + | WarehouseHistoryRead::Observations(read) => read.unchanged_token.as_deref(), WarehouseHistoryRead::NodeResults(read) => read.unchanged_token.as_deref(), WarehouseHistoryRead::Consumption(read) => read.unchanged_token.as_deref(), WarehouseHistoryRead::Catalog(_) | WarehouseHistoryRead::Statistics(_) => None, @@ -694,7 +710,8 @@ fn decode_record( ), WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::Migrations(read) - | WarehouseHistoryRead::ColumnMigrations(read) => { + | WarehouseHistoryRead::ColumnMigrations(read) + | WarehouseHistoryRead::Observations(read) => { return decode_event_record(&read.location, row, (columns, &[timestamp])); } WarehouseHistoryRead::NodeResults(NodeResultsRead { location, .. }) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index aa7f70188d..c9bb7df1c8 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -40,6 +40,7 @@ OldNameViewEventType, StateSqlValueType, build_consumption_create_table_sql, + build_observations_create_table_sql, render_state_sql_literal, write_fingerprint, write_migration_state_event, @@ -317,6 +318,39 @@ def append_consumption_history(adapter: Any, connection: Any, fixture: dict[str, ) +def append_observation_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Observation rows in the real table layout; retried appends repeat identical rows.""" + _ = adapter.execute( + connection=connection, + sql=build_observations_create_table_sql( + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ), + ) + columns: list[str] = list(fixture["record"])[2:] + types: dict[str, StateSqlValueType] = dict.fromkeys(columns, StateSqlValueType.STRING) | { + "observed_at": StateSqlValueType.TEXT_TIMESTAMP + } + rows: list[dict[str, Any]] = [ + row | {"observed_at": datetime.fromisoformat(row["observed_at"])} + for row in identified_rows(fixture) + ] + for row in rows + [rows[0]] * fixture["duplicate_rows"]: + values: list[str] = [ + render_state_sql_literal( + value=row[column], declared_type=types[column], timestamp_type="TIMESTAMP" + ) + for column in columns + ] + _ = adapter.execute( + connection=connection, + sql=f"INSERT INTO main._sqlbuild_observations ({', '.join(columns)}) " + f"VALUES ({', '.join(values)})", + ) + + def identified_rows(fixture: dict[str, Any]) -> list[dict[str, Any]]: """The fixture record once per source identity, with its per-identity overrides.""" return [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index e59d7ec479..488c18d903 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -57,6 +57,7 @@ append_microbatch_history, append_migration_history, append_node_result_history, + append_observation_history, append_old_name_history, append_retried_fingerprints, location_process, @@ -162,6 +163,22 @@ "external-run", (), ), + ( + "duckdb", + "database", + "observation_history", + append_observation_history, + "external-run", + (), + ), + ( + "sqlite", + "path", + "observation_history", + append_observation_history, + "external-run", + (), + ), ( "duckdb", "database", @@ -331,6 +348,22 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and "external-run", (), ), + ( + "duckdb", + "database", + "observation_history", + append_observation_history, + "external-run", + (), + ), + ( + "sqlite", + "path", + "observation_history", + append_observation_history, + "external-run", + (), + ), ] ], ids=lambda case: case.description, From 925fdd75242ae3058fea2f9bc564b3e00af1206e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:28:12 +0000 Subject: [PATCH 05/17] test(policy): keep build observations out of planning and build decisions Add Fensu rule XSB078: Rust code may name observation state only in the files that write, prune, mirror or display it. An E2E build ignores a wrong cursor_max observation and appends from the live MAX. --- fensu.toml | 16 ++++- scripts/fensu_policy/_helpers/path_checks.py | 10 +++ scripts/fensu_policy/_helpers/unused_state.py | 17 ++++- scripts/fensu_policy/constants.py | 12 ++++ .../rules/observation_boundary.py | 61 ++++++++++++++++++ .../cli/commands/main/build/_test_types.py | 13 ++++ .../main/build/test_cursor_observations.py | 56 +++++++++++++++++ .../scripts/fensu_policy/_test_types.py | 9 +++ .../scripts/fensu_policy/helpers.py | 58 +++++++++++++++++ .../fensu_policy/test_observation_boundary.py | 62 +++++++++++++++++++ 10 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 scripts/fensu_policy/rules/observation_boundary.py create mode 100644 tests/integration/scripts/fensu_policy/helpers.py create mode 100644 tests/integration/scripts/fensu_policy/test_observation_boundary.py diff --git a/fensu.toml b/fensu.toml index 391b437ff7..964d82b43d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1630,9 +1630,10 @@ tooling = [] rule_packs = ["rust"] rule_modules = [ "scripts.fensu_policy.rules.native_boundary", + "scripts.fensu_policy.rules.observation_boundary", "scripts.fensu_policy.rules.unused_state", ] -select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077"] +select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078"] [targets.rust.rule_options.XSB076] schema_paths = ["crates/sqlbuild-orchestration/src/store"] @@ -1646,6 +1647,19 @@ allowed = [ "sqb_warehouse_migrations.decision=Reader comes with the UI epic (D132), which filters on it.", ] +[targets.rust.rule_options.XSB078] +allowed_paths = [ + "crates/sqlbuild-core/src/constants.rs=Names the observation state table.", + "crates/sqlbuild-orchestration=Mirrors observations into the control store and serves them for display.", + "crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs=Registers the observation state bindings.", + "crates/sqlbuild-python/src/bindings/_helpers/exported_constants=Exports the table name to Python adapters, which create it.", + "crates/sqlbuild-python/src/bindings/_helpers/janitor=Prunes observation history.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs=Declares the observation state bindings module.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs=Renders observation state SQL for adapters.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs=Reads observations for the control-store mirror.", + "crates/sqlbuild-python/src/bindings/_helpers/observations=Writes observations after a build.", +] + [targets.rust.rule_options.XSB077] construction_required = [ "Action=A permission is live only when a route requires it; role tables and classifiers do not count.", diff --git a/scripts/fensu_policy/_helpers/path_checks.py b/scripts/fensu_policy/_helpers/path_checks.py index 3f4ff27c48..e61f0e43db 100644 --- a/scripts/fensu_policy/_helpers/path_checks.py +++ b/scripts/fensu_policy/_helpers/path_checks.py @@ -32,3 +32,13 @@ def is_adapter_implementation_path(*, path: str) -> bool: "/adapter/contract/classes/duckdb_backed_adapter.py", ) ) + + +def allowed_prefix(*, path: str, prefixes: list[str]) -> str | None: + """The allowed file or directory entry that covers a repository-relative path.""" + + prefix: str + for prefix in prefixes: + if path == prefix or path.startswith(prefix.rstrip("/") + "/"): + return prefix + return None diff --git a/scripts/fensu_policy/_helpers/unused_state.py b/scripts/fensu_policy/_helpers/unused_state.py index 0571284126..20303010e8 100644 --- a/scripts/fensu_policy/_helpers/unused_state.py +++ b/scripts/fensu_policy/_helpers/unused_state.py @@ -1,4 +1,4 @@ -"""Find control-store state nothing reads and Rust enum variants nothing names.""" +"""Find control-store state nothing reads, Rust enum variants nothing names, and state uses.""" from __future__ import annotations @@ -14,6 +14,7 @@ BRACKET_OPEN, LIST_SEPARATOR, NATIVE_PATH_SEPARATOR, + OBSERVATION_STATE_PATTERN, PARENTHESIS_CLOSE, PARENTHESIS_OPEN, REASON_SEPARATOR, @@ -23,6 +24,7 @@ RUST_ENUM_PATTERN, RUST_FROM_ATTRIBUTE_PATTERN, RUST_IMPL_HEADER_PATTERN, + RUST_LINE_COMMENT, RUST_LINE_CONTINUATION, RUST_LITERAL_PATTERN, RUST_LOOKBEHIND, @@ -66,6 +68,7 @@ ) from scripts.fensu_policy.models import EnumVariant, StateDefinition, StateReads +_OBSERVATION_STATE: re.Pattern[str] = re.compile(OBSERVATION_STATE_PATTERN) _LEXEMES: re.Pattern[str] = re.compile(RUST_LITERAL_PATTERN, re.DOTALL) _CONTINUATION: re.Pattern[str] = re.compile(RUST_LINE_CONTINUATION) _TEST_MODULE: re.Pattern[str] = re.compile(RUST_TEST_MODULE_PATTERN) @@ -589,3 +592,15 @@ def _state_key(*, definition: StateDefinition) -> str: if definition.column is None: return definition.table return f"{definition.table}{SQL_QUALIFIER}{definition.column}" + + +def observation_reference_lines(*, source: str) -> list[int]: + """One-based lines whose code, not a line comment, names display-only observation state.""" + + lines: list[int] = [] + number: int + line: str + for number, line in enumerate(source.splitlines(), start=1): + if _OBSERVATION_STATE.search(line.partition(RUST_LINE_COMMENT)[0]): + lines.append(number) + return lines diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index e6f5bcab4e..6bcecb4572 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -476,3 +476,15 @@ description="`table.column=reason` entries for state read outside production code.", ) POLICY_CONFIG_FILE: str = "fensu.toml" +OBSERVATION_ALLOWED_PATHS: RuleOption[tuple[str, ...]] = RuleOption.string_list( + name="allowed_paths", + default=(), + description=( + "`path=reason` entries for the Rust files or directories that may name observation state." + ), +) +OBSERVATION_STATE_PATTERN: str = ( + r"\b(?:observations_state|OBSERVATIONS_TABLE_NAME|WarehouseObservationRecord" + r"|WarehouseHistoryKind::Observations|_sqlbuild_observations|sqb_warehouse_observations)\b" +) +RUST_LINE_COMMENT: str = "//" diff --git a/scripts/fensu_policy/rules/observation_boundary.py b/scripts/fensu_policy/rules/observation_boundary.py new file mode 100644 index 0000000000..54361ac133 --- /dev/null +++ b/scripts/fensu_policy/rules/observation_boundary.py @@ -0,0 +1,61 @@ +"""Rules keeping build observations out of planning and build decisions.""" + +from __future__ import annotations + +from fensu import AnalyzerId, Family, Fault, Project, RuleContext, rule + +from scripts.fensu_policy._helpers.path_checks import allowed_prefix +from scripts.fensu_policy._helpers.unused_state import ( + is_rust_test_path, + observation_reference_lines, + reasoned_entries, +) +from scripts.fensu_policy.constants import OBSERVATION_ALLOWED_PATHS, POLICY_CONFIG_FILE + + +@rule( + code="XSB078", + family=Family.CUSTOM, + slug="observation-display-only", + message="build observations are display-only and must not reach planning or build decisions", + remediation=( + "Read the live warehouse fact instead; only the files in allowed_paths, which write, " + "prune, mirror or display observations, may name observation state." + ), + cacheable=False, + analyzers=(AnalyzerId.RUST,), + options=(OBSERVATION_ALLOWED_PATHS,), +) +def observation_display_only(*, project: Project, ctx: RuleContext) -> list[Fault]: + del project + reasons: dict[str, str] + malformed: list[str] + reasons, malformed = reasoned_entries(entries=ctx.option(OBSERVATION_ALLOWED_PATHS)) + prefixes: list[str] = list(reasons) + used: set[str] = set() + faults: list[Fault] = [] + for facts in ctx.rust.files: + path: str = facts.file.path.value + if facts.test or is_rust_test_path(path=path): + continue + lines: list[int] = observation_reference_lines(source=facts.source) + if not lines: + continue + prefix: str | None = allowed_prefix(path=path, prefixes=prefixes) + if prefix is not None: + used.add(prefix) + continue + faults.extend( + ctx.fault_for(path=ctx.repo_root / path, line=line, column=0) for line in lines + ) + problems: list[str] = [ + f"allowed_paths entry {entry!r} needs `path=reason`" for entry in malformed + ] + [ + f"allowed path {prefix} no longer names observation state; remove the entry" + for prefix in prefixes + if prefix not in used + ] + return [ + ctx.fault_for(path=ctx.repo_root / POLICY_CONFIG_FILE, line=1, column=0, message=problem) + for problem in problems + ] + faults diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index c1cf13ba76..e9449ca7af 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -372,6 +372,19 @@ class CursorObservationBuildE2ETestCase: expected_observations: tuple[tuple[str, str, str, int], ...] +@dataclass(frozen=True) +class DisplayOnlyObservationE2ETestCase: + """A wrong cursor observation that plan and build must ignore in favour of the live MAX. + + A misled build would append nothing; the live bound is inclusive, so it re-reads the MAX row. + """ + + description: str + injected_observations: tuple[tuple[str, str, str], ...] + added_seed_rows: str + expected_row_counts: tuple[tuple[str, int], ...] + + @dataclass(frozen=True) class SourceColumnObservationE2ETestCase: """Commands after which each build, and no plan, must have recorded described source columns.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py index 9403f216a8..73eb80b25d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py @@ -10,9 +10,12 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( CursorObservationBuildE2ETestCase, + DisplayOnlyObservationE2ETestCase, SourceColumnObservationE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + normalize_cli_output, prepare_inline_project, query_duckdb, run_sqb, @@ -178,3 +181,56 @@ def test_given_expression_source_when_building_then_source_column_observations_a -1 ].stderr assert tuple(observations) == test_case.expected_observations + + +@pytest.mark.parametrize( + "test_case", + [ + DisplayOnlyObservationE2ETestCase( + description="cursor_max past the live MAX still appends from the inclusive live bound", + injected_observations=( + ("order_ids", "integer", "999"), + ("orders", "timestamp", "2099-01-01 00:00:00"), + ), + added_seed_rows="3,2026-01-01 02:00:00\n", + expected_row_counts=(("order_ids", 4), ("orders", 4)), + ), + ], + ids=lambda case: case.description, +) +def test_given_wrong_cursor_observation_when_planning_and_building_then_live_max_is_used( + tmp_path: Path, test_case: DisplayOnlyObservationE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="display_only_observations", repo_files=_PROJECT_FILES + ) + database: Path = project_dir / "observations.duckdb" + first: subprocess.CompletedProcess[str] = run_sqb(command=("build",), project_dir=project_dir) + assert first.returncode == 0, first.stderr + seed: Path = project_dir / "seeds/raw_orders.csv" + seed.write_text(seed.read_text() + test_case.added_seed_rows) + + honest: subprocess.CompletedProcess[str] = run_sqb(command=("plan",), project_dir=project_dir) + node: str + cursor_type: str + value: str + for node, cursor_type, value in test_case.injected_observations: + execute_duckdb( + db_path=database, + sql=( + "INSERT INTO main._sqlbuild_observations VALUES " + f"('injected-{node}', 'cursor_max', '{node}', '{cursor_type}', '{value}', " + "'injected-run', '2099-01-01 00:00:00')" + ), + ) + misled: subprocess.CompletedProcess[str] = run_sqb(command=("plan",), project_dir=project_dir) + build: subprocess.CompletedProcess[str] = run_sqb(command=("build",), project_dir=project_dir) + counts: tuple[tuple[str, int], ...] = tuple( + (name, query_duckdb(db_path=database, sql=f"SELECT COUNT(*) FROM main.{name}")[0][0]) + for name, _ in test_case.expected_row_counts + ) + + assert honest.returncode == misled.returncode == 0, misled.stderr + assert normalize_cli_output(misled.stdout) == normalize_cli_output(honest.stdout) + assert build.returncode == 0, build.stderr + assert counts == test_case.expected_row_counts diff --git a/tests/integration/scripts/fensu_policy/_test_types.py b/tests/integration/scripts/fensu_policy/_test_types.py index ef287d1b65..a4ae42b48c 100644 --- a/tests/integration/scripts/fensu_policy/_test_types.py +++ b/tests/integration/scripts/fensu_policy/_test_types.py @@ -8,3 +8,12 @@ class NativeBoundaryTestCase: description: str metadata: str expected_fault_count: int + + +@dataclass(frozen=True) +class ObservationBoundaryTestCase: + description: str + allowed: tuple[str, ...] + planner: str + expected_returncode: int + expected_faults: tuple[str, ...] diff --git a/tests/integration/scripts/fensu_policy/helpers.py b/tests/integration/scripts/fensu_policy/helpers.py new file mode 100644 index 0000000000..45e8140bac --- /dev/null +++ b/tests/integration/scripts/fensu_policy/helpers.py @@ -0,0 +1,58 @@ +"""Build a minimal repository and run the real Fensu CLI on its Rust target.""" + +import shutil +import subprocess +import sys +from pathlib import Path + + +def write_observation_project(*, root: Path, allowed: tuple[str, ...], planner: str) -> None: + """A crate whose observations module may name observation state, plus the given planner.""" + + repository: Path = Path(__file__).resolve().parents[4] + shutil.copytree( + repository / "scripts/fensu_policy", + root / "scripts/fensu_policy", + ignore=shutil.ignore_patterns("__pycache__"), + ) + (root / "scripts/__init__.py").write_text("") + (root / "Cargo.toml").write_text( + '[workspace]\nmembers = ["crates/sqlbuild-python"]\nresolver = "2"\n' + ) + crate: Path = root / "crates/sqlbuild-python" + (crate / "src/observations").mkdir(parents=True) + (crate / "Cargo.toml").write_text( + '[package]\nname = "sqlbuild-python"\nversion = "0.1.0"\nedition = "2021"\n' + ) + (crate / "src/lib.rs").write_text("mod observations;\nmod planner;\n") + (crate / "src/observations/mod.rs").write_text( + "pub fn table() -> &'static str { OBSERVATIONS_TABLE_NAME }\n" + ) + (crate / "src/planner.rs").write_text(planner) + entries: str = ", ".join(f'"{path}=Writes observations."' for path in allowed) + (root / "fensu.toml").write_text( + '[targets.rust]\nanalyzer = "rust"\nroots = ["crates"]\n' + 'rule_modules = ["scripts.fensu_policy.rules.observation_boundary"]\n' + 'select = ["XSB078"]\n\n' + f"[targets.rust.rule_options.XSB078]\nallowed_paths = [{entries}]\n" + ) + + +def check_rust(*, root: Path) -> subprocess.CompletedProcess[str]: + """Run `fensu check` on the Rust target without a cache.""" + + return subprocess.run( + [ + str(Path(sys.executable).with_name("fensu")), + "check", + "--target", + "rust", + "--no-cache", + "--no-color", + ], + cwd=root, + capture_output=True, + text=True, + timeout=30, + check=False, + ) diff --git a/tests/integration/scripts/fensu_policy/test_observation_boundary.py b/tests/integration/scripts/fensu_policy/test_observation_boundary.py new file mode 100644 index 0000000000..98b6be983d --- /dev/null +++ b/tests/integration/scripts/fensu_policy/test_observation_boundary.py @@ -0,0 +1,62 @@ +"""Exercise the observation display-only policy through Fensu's real Rust analyzer.""" + +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.scripts.fensu_policy._test_types import ObservationBoundaryTestCase +from tests.integration.scripts.fensu_policy.helpers import check_rust, write_observation_project + + +@pytest.mark.parametrize( + "test_case", + [ + ObservationBoundaryTestCase( + description="planner mentioning observations only in a comment", + allowed=("crates/sqlbuild-python/src/observations",), + planner="// OBSERVATIONS_TABLE_NAME is display-only\npub fn plan() {}\n", + expected_returncode=0, + expected_faults=(), + ), + ObservationBoundaryTestCase( + description="planner matching the observations history kind", + allowed=("crates/sqlbuild-python/src/observations",), + planner=( + "pub fn plan() {}\n" + "pub fn mirrored(kind: Kind) -> bool { kind == WarehouseHistoryKind::Observations }\n" + ), + expected_returncode=1, + expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:2:0",), + ), + ObservationBoundaryTestCase( + description="planner reading the observation table by its SQL name", + allowed=("crates/sqlbuild-python/src/observations",), + planner='pub const READ: &str = "SELECT value FROM _sqlbuild_observations";\n', + expected_returncode=1, + expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:1:0",), + ), + ObservationBoundaryTestCase( + description="allowed path that no longer names observation state", + allowed=( + "crates/sqlbuild-python/src/observations", + "crates/sqlbuild-python/src/planner.rs", + ), + planner="pub fn plan() {}\n", + expected_returncode=1, + expected_faults=("allowed path crates/sqlbuild-python/src/planner.rs no longer names",), + ), + ], + ids=lambda case: case.description, +) +def test_given_rust_naming_observation_state_when_checking_then_only_unallowed_code_faults( + tmp_path: Path, test_case: ObservationBoundaryTestCase +) -> None: + write_observation_project(root=tmp_path, allowed=test_case.allowed, planner=test_case.planner) + + result: subprocess.CompletedProcess[str] = check_rust(root=tmp_path) + + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert result.stdout.count("XSB078") == len(test_case.expected_faults), result.stdout + assert all(fault in result.stdout for fault in test_case.expected_faults), result.stdout + assert "src/observations/mod.rs" not in result.stdout From 967bb8ab8bad30da0184c275a0187e5632acb057 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:38:04 +0000 Subject: [PATCH 06/17] fix(observations): record the described SQL hash with source columns A source_columns value is now {sql_hash, columns}, the SHA-256 of the exact expression SQL planning described, so a preview can tell when the expression changed since the observation and report it as unknown. --- .../orchestration/observation_history.json | 6 ++-- .../_helpers/observations/publication.rs | 30 ++++++------------- .../_helpers/observations/source_capture.rs | 25 ++++++++++++---- .../planner_warehouse/source_columns.rs | 3 +- .../src/observations/constants.rs | 4 +++ .../main/build/test_cursor_observations.py | 20 +++++++++++-- 6 files changed, 56 insertions(+), 32 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/observation_history.json b/benchmarks/duckshop/edge_cases/orchestration/observation_history.json index 8d89110ffe..e195f33703 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/observation_history.json +++ b/benchmarks/duckshop/edge_cases/orchestration/observation_history.json @@ -30,7 +30,7 @@ "kind": "source_columns", "node": "shop.orders_🧾", "cursor_type": null, - "value": "[\"order_id\", \"amount_€\"]", + "value": "{\"sql_hash\":\"928300f876a1cfe48302513f8c25ca06ec449c95f3480c06b84c694a39b6be0d\",\"columns\":[\"order_id\",\"amount_€\"]}", "observed_at": "2026-02-03T04:05:06.000001Z" }, "last_record": { @@ -38,7 +38,7 @@ "kind": "source_columns", "node": "shop.orders_🧾", "cursor_type": null, - "value": "[\"order_id\", \"amount_€\"]", + "value": "{\"sql_hash\":\"928300f876a1cfe48302513f8c25ca06ec449c95f3480c06b84c694a39b6be0d\",\"columns\":[\"order_id\",\"amount_€\"]}", "observed_at": "2026-02-03T04:05:06.000001Z" }, "record_overrides": { @@ -46,7 +46,7 @@ "kind": "source_columns", "node": "shop.orders_🧾", "cursor_type": null, - "value": "[\"order_id\", \"amount_€\"]", + "value": "{\"sql_hash\":\"928300f876a1cfe48302513f8c25ca06ec449c95f3480c06b84c694a39b6be0d\",\"columns\":[\"order_id\",\"amount_€\"]}", "observed_at": "2026-02-03T04:05:06.000001Z" } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs index 9dc3055a7b..1f495e5b93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs @@ -3,7 +3,6 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use pyo3::IntoPyObject; -use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; @@ -84,7 +83,7 @@ fn cursor_reads( /// One `source_columns` row per described source, in the state schema of each model reading it. fn source_column_rows( project: &Bound<'_, PyAny>, - described: &BTreeMap>, + described: &BTreeMap, run_id: &str, ) -> PyResult>> { let mut readers: BTreeMap> = BTreeMap::new(); @@ -112,30 +111,19 @@ fn source_column_rows( let mut rows: BTreeMap> = BTreeMap::new(); for (location, sources) in readers { for source in sources { - let row: ObservationAppend = source_columns_row(source, described, run_id)?; + let row: ObservationAppend = ObservationAppend { + kind: SOURCE_COLUMNS_KIND, + value: described[&source].clone(), + node: source, + cursor_type: None, + run_id: run_id.to_owned(), + }; rows.entry(location.clone()).or_default().push(row); } } Ok(rows) } -/// The `source_columns` observation of one described source, its column names as a JSON array. -fn source_columns_row( - source: String, - described: &BTreeMap>, - run_id: &str, -) -> PyResult { - let value: String = serde_json::to_string(&described[&source]) - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - Ok(ObservationAppend { - kind: SOURCE_COLUMNS_KIND, - node: source, - cursor_type: None, - value, - run_id: run_id.to_owned(), - }) -} - /// Read each cursor `MAX`, then append it with `rows`, by state location; one warning per failure. fn observe<'py>( reads: &[CursorRead], @@ -203,7 +191,7 @@ pub(crate) fn append_build_observations( (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), run_id: &str, ) -> Vec { - let described: BTreeMap> = take_source_capture().into_iter().collect(); + let described: BTreeMap = take_source_capture().into_iter().collect(); let recorded: PyResult> = (|| { let reads: Vec = cursor_reads(project, adapter, successful_models)?; let sources: BTreeMap> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs index 5006cdd805..b33192b5fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs @@ -2,8 +2,14 @@ use std::cell::RefCell; -/// Each described source's name and column names, in the order planning described them. -pub(crate) type DescribedSources = Vec<(String, Vec)>; +use serde_json::{Map, Value}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; +use sqlbuild_state::observations::constants::{ + SOURCE_COLUMNS_NAMES_KEY, SOURCE_COLUMNS_SQL_HASH_KEY, +}; + +/// Each described source's name and observation value, in the order planning described them. +pub(crate) type DescribedSources = Vec<(String, String)>; thread_local! { /// Per thread, since a build plans and records on the thread that runs it; `None` when not armed. @@ -15,11 +21,20 @@ pub(crate) fn arm_source_capture() { CAPTURED.with(|slot| *slot.borrow_mut() = Some(Vec::new())); } -/// Keep one source's described column names when a build armed the capture. -pub(crate) fn capture_source_columns(source: &str, columns: &[String]) { +/// Keep one source's described SQL hash and column names when a build armed the capture. +pub(crate) fn capture_source_columns(source: &str, sql: &str, columns: &[String]) { CAPTURED.with(|slot| { if let Some(captured) = slot.borrow_mut().as_mut() { - captured.push((source.to_owned(), columns.to_vec())); + let mut value: Map = Map::new(); + value.insert( + SOURCE_COLUMNS_SQL_HASH_KEY.to_owned(), + Value::String(sha256_hex(sql.as_bytes())), + ); + value.insert( + SOURCE_COLUMNS_NAMES_KEY.to_owned(), + Value::from(columns.to_vec()), + ); + captured.push((source.to_owned(), Value::Object(value).to_string())); } }); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index 613b1842d5..5ec08e1c83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -93,6 +93,7 @@ pub(crate) fn source_inspection<'py>( let expression = native_field(entry, "expression")?; if !expression.is_none() { if native_field(entry, "type_enforcement")?.is_truthy()? { + let sql: String = expression.extract()?; let names = call_with( &native_field(adapter, "query_column_names")?, &[("connection", connection.clone()), ("sql", expression)], @@ -108,7 +109,7 @@ pub(crate) fn source_inspection<'py>( )?); } let source: Bound<'py, PyAny> = native_field(entry, "name")?; - capture_source_columns(&source.extract::()?, &described); + capture_source_columns(&source.extract::()?, &sql, &described); result.set_item(source, tuple(py, columns)?)?; } continue; diff --git a/crates/sqlbuild-state/src/observations/constants.rs b/crates/sqlbuild-state/src/observations/constants.rs index 8f41fc7db8..009848b760 100644 --- a/crates/sqlbuild-state/src/observations/constants.rs +++ b/crates/sqlbuild-state/src/observations/constants.rs @@ -20,3 +20,7 @@ pub const NULLABLE_OBSERVATION_COLUMNS: [&str; 1] = ["cursor_type"]; pub const CURSOR_MAX_KIND: &str = "cursor_max"; /// Observation of the columns a live `DESCRIBE` returned for an expression source. pub const SOURCE_COLUMNS_KIND: &str = "source_columns"; +/// Key of a `source_columns` value holding the SHA-256 of the SQL that was described. +pub const SOURCE_COLUMNS_SQL_HASH_KEY: &str = "sql_hash"; +/// Key of a `source_columns` value holding the described column names in order. +pub const SOURCE_COLUMNS_NAMES_KEY: &str = "columns"; diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py index 73eb80b25d..9fe47fd8f2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py @@ -2,6 +2,8 @@ from __future__ import annotations +import hashlib +import json import subprocess from pathlib import Path from textwrap import dedent @@ -151,9 +153,23 @@ def test_given_incremental_models_when_building_then_cursor_max_observations_are "test_case", [ SourceColumnObservationE2ETestCase( - description="builds record described source columns and plans record none", + description="builds record the described SQL hash and columns and plans record none", commands=(("build",), ("plan",), ("build",)), - expected_observations=(("raw_refunds", '["refund_id","amount_cents"]', 2),), + expected_observations=( + ( + "raw_refunds", + json.dumps( + { + "sql_hash": hashlib.sha256( + b"SELECT * FROM (VALUES (1, 100)) AS refunds(refund_id, amount_cents)\n" + ).hexdigest(), + "columns": ["refund_id", "amount_cents"], + }, + separators=(",", ":"), + ), + 2, + ), + ), ), ], ids=lambda case: case.description, From a2d5307e6decb688ebf30c94a49c2fe9d81673ae Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:01:16 +0000 Subject: [PATCH 07/17] feat(server): serve the mirrored copy a plan preview plans from Add POST .../metadata/preview-copy for a location's fenced reader. It returns the target's catalog relations, each node's newest fingerprint with its definition, newest succeeded run and newest observation per kind, and synced_at: the oldest of the newest clean syncs of catalog, fingerprints and consumption, or null until all three synced cleanly. --- clients/typescript/src/schema.d.ts | 127 ++++++++ .../api/_helpers/resources/location_health.rs | 2 +- .../src/api/_helpers/resources/router.rs | 4 + .../_helpers/resources/warehouse_metadata.rs | 24 +- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/platform/_helpers/work_client.rs | 14 +- .../src/platform/main/location_health.rs | 5 +- .../_helpers/mirror_pages.rs | 152 +++++++++- .../src/warehouse_metadata/constants.rs | 8 + .../main/location_reads/mod.rs | 1 + .../main/location_reads/read_preview_copy.rs | 36 +++ .../src/warehouse_metadata/models.rs | 30 ++ .../tests/api/test_contracts.rs | 2 + .../tests/api/test_warehouse_metadata.rs | 6 + .../preview_copy/helpers.rs | 274 ++++++++++++++++++ 15 files changed, 677 insertions(+), 9 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 8f20e063b6..38cff562b5 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1010,6 +1010,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/preview-copy": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["preview_copy"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/statistics-due": { parameters: { query?: never; @@ -2308,6 +2324,30 @@ export interface components { */ scheduler_lease_expires_at?: string | null; }; + /** @description What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. */ + PreviewCopy: { + fingerprints: components["schemas"]["WarehouseFingerprintRecord"][]; + latest_runs: components["schemas"]["PreviewLatestRun"][]; + observations: components["schemas"]["WarehouseObservationRecord"][]; + relations: components["schemas"]["WarehouseCatalogRecord"][]; + /** Format: date-time */ + synced_at?: string | null; + }; + /** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */ + PreviewCopyRead: { + /** Format: int64 */ + generation: number; + /** Format: uuid */ + session_id: string; + }; + /** @description The newest successful build of a node, which a `cursor_max` observation must come from. */ + PreviewLatestRun: { + node_name: string; + node_type: string; + run_id: string; + state_database?: string | null; + state_schema: string; + }; /** @description A changed subject, such as `model:orders`, or `role:analyst` (`role:` for those who can't assign roles). */ PublishedChange: { change_kind: string; @@ -3078,6 +3118,25 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */ + WarehouseFingerprintRecord: { + definition: string; + definition_hash: string; + event_id?: string | null; + metadata_json: string; + node_name: string; + node_type: string; + run_id: string; + schema_fingerprint: string; + state_database?: string | null; + state_schema: string; + target_database?: string | null; + target_name?: string | null; + target_schema?: string | null; + /** Format: date-time */ + ts: string; + version_hash: string; + }; /** * @description Catalog observations require full scans; only immutable history supports a watermark. * @enum {string} @@ -3131,6 +3190,19 @@ export interface components { /** Format: date-time */ observed_at: string; }; + /** @description A display-only value a build observed, such as a cursor `MAX`; planning never reads it. */ + WarehouseObservationRecord: { + cursor_type?: string | null; + kind: string; + node: string; + observation_id: string; + /** Format: date-time */ + observed_at: string; + run_id: string; + state_database?: string | null; + state_schema: string; + value: string; + }; /** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */ WarehouseStateLocation: { database?: string | null; @@ -5889,6 +5961,61 @@ export interface operations { }; }; }; + preview_copy: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PreviewCopyRead"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PreviewCopy"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; statistics_due: { parameters: { query?: never; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs index 24a318b3f9..adeed404f4 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs @@ -41,5 +41,5 @@ pub(crate) async fn recent( Path(location_id): Path, ) -> Result>, ApiError> { require_location(&identity, location_id)?; - Ok(Json(location_health(&store, location_id).await?)) + Ok(Json(location_health(&store, (location_id, None)).await?)) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index 5df6d1402d..eb4365481d 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -35,6 +35,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(request): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + read_preview_copy(&store, (location_id, &target), &request).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 54d45356d9..77bba7b6f7 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -96,6 +96,7 @@ use crate::api::_helpers::resources::{ trigger_reads::events, trigger_reads::ticks, warehouse_metadata::report, + warehouse_metadata::preview_copy, warehouse_metadata::statistics_due, warehouse_metadata::sync_due, warehouse_metadata::watermark diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index eaf407d5aa..a44a93c5b3 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,8 +10,8 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - MetadataBatch, StatisticsDueRead, StatisticsRead, WarehouseSyncDue, WarehouseSyncDueRead, - WarehouseSyncResponse, WarehouseWatermarkRead, + MetadataBatch, PreviewCopy, PreviewCopyRead, StatisticsDueRead, StatisticsRead, + WarehouseSyncDue, WarehouseSyncDueRead, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -54,6 +54,16 @@ impl LocationWorkClient { .await } + /// The mirrored copy a plan preview plans from, so previewing never opens the warehouse. + pub async fn preview_copy( + &self, + target: &str, + request: &PreviewCopyRead, + ) -> Result { + self.post_metadata((target, Some("preview-copy")), request) + .await + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_metadata( &self, diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs index 148e6fb77f..a50785fabe 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/location_health.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs @@ -8,14 +8,15 @@ use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, TransactionMode}; +/// Every target's observations, or only `target`'s. pub async fn location_health( store: &ControlStore, - location_id: Uuid, + (location_id, target): (Uuid, Option<&str>), ) -> Result, ControlError> { let (transaction, observations) = recent_health( begin(store, TransactionMode::Read).await?, location_id, - None, + target, ) .await?; commit(transaction).await?; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs index e016900106..e9e4048b2e 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs @@ -1,9 +1,41 @@ -//! Mirror history pages share database scoping and the (observed_at, event_id) keyset cursor. +//! Mirror reads share database scoping and the keyset cursor; previews read each node's newest facts. + +use std::collections::{BTreeMap, BTreeSet}; use chrono::{DateTime, Utc}; +use serde::de::DeserializeOwned; +use serde_json::Value; use uuid::Uuid; -use crate::store::models::Parameter; +use crate::platform::models::{LocationHealth, LocationHealthObservation}; +use crate::store::errors::ControlError; +use crate::store::main::decompress::decompress; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::types::Row; +use crate::warehouse_metadata::models::{ + PreviewLatestRun, WarehouseCatalogRecord, WarehouseConsumptionRecord, + WarehouseFingerprintRecord, WarehouseHistoryKind, WarehouseObservationRecord, +}; + +const RELATIONS_SQL: &str = "SELECT record_json FROM sqb_warehouse_catalog \ + WHERE location_id=$1 AND target=$2 ORDER BY state_database, state_schema, event_id"; +const FINGERPRINTS_SQL: &str = "SELECT f.record_json, t.body FROM (SELECT record_json, definition_text_hash, \ + ROW_NUMBER() OVER (PARTITION BY state_database, state_schema, node_type, node_name, target_database, \ + target_schema, target_name ORDER BY observed_at DESC, event_id DESC) AS newest \ + FROM sqb_warehouse_fingerprints WHERE location_id=$1 AND target=$2) f \ + JOIN sqb_source_texts t ON t.text_hash=f.definition_text_hash WHERE f.newest=1 \ + ORDER BY f.record_json ->> 'node_type', f.record_json ->> 'node_name'"; +const LATEST_RUNS_SQL: &str = "SELECT record_json FROM (SELECT record_json, ROW_NUMBER() OVER (PARTITION BY \ + state_database, state_schema, record_json ->> 'node_type', record_json ->> 'node_name' \ + ORDER BY observed_at DESC, event_id DESC) AS newest FROM sqb_warehouse_consumption \ + WHERE location_id=$1 AND target=$2 AND record_json ->> 'phase'='succeeded' \ + AND record_json ->> 'run_id' IS NOT NULL) runs WHERE newest=1 \ + ORDER BY record_json ->> 'node_type', record_json ->> 'node_name'"; +const OBSERVATIONS_SQL: &str = "SELECT record_json FROM (SELECT record_json, kind, node, ROW_NUMBER() OVER \ + (PARTITION BY state_database, state_schema, kind, node ORDER BY observed_at DESC, event_id DESC) AS newest \ + FROM sqb_warehouse_observations WHERE location_id=$1 AND target=$2) observations WHERE newest=1 \ + ORDER BY kind, node"; /// Adds scope, cursor and page size to the parameters, with the SQL after the caller's conditions. pub(crate) fn page_tail( @@ -34,3 +66,119 @@ pub(crate) fn page_tail( ); (parameters, tail) } + +fn scope_parameters((location_id, target): (Uuid, &str)) -> Vec { + vec![Parameter::Id(location_id), Parameter::Text(target.into())] +} + +fn decoded(mut row: Row) -> Result { + serde_json::from_value(row.remove("record_json").unwrap_or(Value::Null)) + .map_err(|error| ControlError::Invalid(format!("invalid mirrored record: {error}"))) +} + +async fn mirrored( + transaction: ControlTransaction, + sql: &str, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, sql, scope_parameters(scope)).await?; + let mut records: Vec = Vec::with_capacity(rows.len()); + for row in rows { + records.push(decoded(row)?); + } + Ok((transaction, records)) +} + +/// Every relation of the target's newest complete catalog scans. +pub(crate) async fn preview_relations( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + mirrored(transaction, RELATIONS_SQL, scope).await +} + +/// Each node's newest fingerprint, with its definition restored from the shared text store. +pub(crate) async fn preview_fingerprints( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, FINGERPRINTS_SQL, scope_parameters(scope)).await?; + let mut fingerprints: Vec = Vec::with_capacity(rows.len()); + for mut row in rows { + let body: Vec = decompress(&row.remove("body").unwrap_or(Value::Null))?; + let definition: String = + String::from_utf8(body).map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut record: Value = row.remove("record_json").unwrap_or(Value::Null); + if let Value::Object(object) = &mut record { + object.insert("definition".into(), Value::String(definition)); + } + fingerprints.push(serde_json::from_value(record).map_err(|error| { + ControlError::Invalid(format!("invalid mirrored fingerprint: {error}")) + })?); + } + Ok((transaction, fingerprints)) +} + +/// Each node's newest succeeded build that names its node and run. +pub(crate) async fn preview_latest_runs( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, consumption): (ControlTransaction, Vec) = + mirrored(transaction, LATEST_RUNS_SQL, scope).await?; + let mut runs: Vec = Vec::with_capacity(consumption.len()); + for record in consumption { + if let (Some(node_type), Some(node_name), Some(run_id)) = + (record.node_type, record.node_name, record.run_id) + { + runs.push(PreviewLatestRun { + state_database: record.state_database, + state_schema: record.state_schema, + node_type, + node_name, + run_id, + }); + } + } + Ok((transaction, runs)) +} + +/// Each node's newest observation of each kind. +pub(crate) async fn preview_observations( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + mirrored(transaction, OBSERVATIONS_SQL, scope).await +} + +/// The oldest of each required table's newest clean sync; `None` until every table synced cleanly. +pub(crate) fn synced_at( + health: &[LocationHealthObservation], + tables: &[WarehouseHistoryKind], +) -> Option> { + let mut newest: BTreeMap<(Option<&str>, &str, &str), DateTime> = BTreeMap::new(); + for observation in health { + let LocationHealth::WarehouseSync { + location, + table, + failure: None, + .. + } = &observation.report.health + else { + continue; + }; + if !tables.contains(table) { + continue; + } + let key = ( + location.database.as_deref(), + location.schema.as_str(), + table.as_str(), + ); + let at = newest.entry(key).or_insert(observation.received_at); + *at = (*at).max(observation.received_at); + } + let synced: BTreeSet<&str> = newest.keys().map(|(_, _, table)| *table).collect(); + let complete: bool = tables.iter().all(|table| synced.contains(table.as_str())); + complete.then(|| newest.values().min().copied()).flatten() +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 5ff4f0a6f6..7a15ec7660 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -1,7 +1,15 @@ //! Limits bound each authenticated warehouse observation transaction. +use crate::warehouse_metadata::models::WarehouseHistoryKind; + pub const MAX_HISTORY_RECORDS: usize = 100; pub(crate) const CATALOG_MIRROR: &str = "sqb_warehouse_catalog"; pub(crate) const DEFAULT_STATISTICS_RETENTION_DAYS: u16 = 90; /// Column-lineage node definitions read per query, well under every backend's bound parameter limit. pub(crate) const LINEAGE_DEFINITION_BATCH: usize = 1000; +/// Mirrored tables a plan preview needs synced cleanly; observations only refine it into knowns. +pub(crate) const PREVIEW_REQUIRED_TABLES: [WarehouseHistoryKind; 3] = [ + WarehouseHistoryKind::Catalog, + WarehouseHistoryKind::Fingerprints, + WarehouseHistoryKind::Consumption, +]; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs index 05bf3ab152..d54f5f9745 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs @@ -1,3 +1,4 @@ +pub mod read_preview_copy; pub mod read_statistics_due; pub mod read_sync_due; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs new file mode 100644 index 0000000000..61e5fcd90e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs @@ -0,0 +1,36 @@ +//! A plan preview plans from the mirror, so the copy carries only what its facade may answer. + +use uuid::Uuid; + +use crate::platform::main::location_health::location_health; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; +use crate::warehouse_metadata::_helpers::mirror_pages::{ + preview_fingerprints, preview_latest_runs, preview_observations, preview_relations, synced_at, +}; +use crate::warehouse_metadata::constants::PREVIEW_REQUIRED_TABLES; +use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; + +pub async fn read_preview_copy( + store: &ControlStore, + scope: (Uuid, &str), + request: &PreviewCopyRead, +) -> Result { + let transaction = + begin_ingestion(store, scope, (request.session_id, request.generation)).await?; + let (transaction, relations) = preview_relations(transaction, scope).await?; + let (transaction, fingerprints) = preview_fingerprints(transaction, scope).await?; + let (transaction, latest_runs) = preview_latest_runs(transaction, scope).await?; + let (transaction, observations) = preview_observations(transaction, scope).await?; + commit(transaction).await?; + let health = location_health(store, (scope.0, Some(scope.1))).await?; + Ok(PreviewCopy { + synced_at: synced_at(&health, &PREVIEW_REQUIRED_TABLES), + relations, + fingerprints, + latest_runs, + observations, + }) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index f3d9e27fa6..71de712368 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -429,6 +429,36 @@ pub struct WarehouseSyncDue { pub now: DateTime, } +/// A location asks for the mirrored copy a plan preview plans from, as its fenced reader. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct PreviewCopyRead { + pub session_id: Uuid, + pub generation: i64, +} + +/// The newest successful build of a node, which a `cursor_max` observation must come from. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct PreviewLatestRun { + pub state_database: Option, + pub state_schema: String, + pub node_type: String, + pub node_name: String, + pub run_id: String, +} + +/// What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct PreviewCopy { + pub synced_at: Option>, + pub relations: Vec, + pub fingerprints: Vec, + pub latest_runs: Vec, + pub observations: Vec, +} + /// Private interpreter input; cursor values are data and receive adapter literal rendering. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 24b70b4ab0..a8e01d7cd0 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -106,6 +106,8 @@ mod warehouse_events_helpers; pub(crate) mod warehouse_history_helpers; #[path = "warehouse_metadata/helpers.rs"] mod warehouse_metadata_helpers; +#[path = "warehouse_metadata/preview_copy/helpers.rs"] +mod warehouse_preview_copy_helpers; #[path = "warehouse_metadata/reconciliation/helpers.rs"] mod warehouse_reconciliation_helpers; #[path = "work/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index a367bd80d1..7ec23aab2c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -423,6 +423,12 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit (&client, &url, &token.bearer), ) .await; + crate::api::tests::warehouse_preview_copy_helpers::preview_copy_scope( + &outbound, + (session_id, preview_session), + (&client, &url, &token.bearer), + ) + .await; stop_api(api).await; close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs new file mode 100644 index 0000000000..d73f7c0828 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs @@ -0,0 +1,274 @@ +//! A preview copy holds each node's newest mirrored facts, dated by the stalest required clean sync. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::platform::models::{LocationHealthReport, LocationWorkClient}; +use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; + +const NODE: &str = "preview_copy_orders"; + +async fn post(client: &Client, (url, bearer): (&str, &str), body: Value) { + let response = client + .post(url) + .bearer_auth(bearer) + .json(&body) + .send() + .await + .expect("metadata batch"); + assert_eq!(response.status(), StatusCode::OK, "{body}"); +} + +fn fingerprint(run_id: &str, definition: &str, ts: &str) -> Value { + json!({"state_database": null, "state_schema": "main", "node_type": "model", "node_name": NODE, + "target_database": null, "target_schema": "main", "target_name": NODE, "run_id": run_id, + "definition_hash": format!("definition-{run_id}"), "version_hash": format!("version-{run_id}"), + "schema_fingerprint": "id:INTEGER", "definition": definition, "metadata_json": "{}", "ts": ts}) +} + +fn consumption(execution_id: &str, (phase, run_id): (&str, &str), recorded_at: &str) -> Value { + json!({"state_database": null, "state_schema": "main", "execution_id": execution_id, "phase": phase, + "run_id": run_id, "node_type": "model", "node_name": NODE, "target_database": null, + "target_schema": "main", "target_name": NODE, "version_hash": "version", "input_count": 0, + "input_kind": null, "input_name": null, "input_version": null, "recorded_at": recorded_at}) +} + +fn observation( + id: &str, + (kind, node, value): (&str, &str, &str), + (run_id, at): (&str, &str), +) -> Value { + json!({"state_database": null, "state_schema": "main", "observation_id": id, "kind": kind, + "node": node, "cursor_type": (kind == "cursor_max").then_some("integer"), "value": value, + "run_id": run_id, "observed_at": at}) +} + +fn sync_report(session: Uuid, table: &str, failure: Option<&str>) -> LocationHealthReport { + serde_json::from_value(json!({ + "event_id": Uuid::now_v7(), "session_id": session, "target": "preview", + "health": {"kind": "warehouse_sync", "generation": 1, "location": {"database": null, "schema": "main"}, + "table": table, "failure": failure, "records_sent": 1, "elapsed_millis": 1}, + })) + .expect("sync health") +} + +/// The preview target's copy: newest fingerprint, succeeded run and observation per node, and `synced_at`. +pub(crate) async fn preview_copy_scope( + outbound: &LocationWorkClient, + (session_id, preview_session): (Uuid, Uuid), + (client, url, bearer): (&Client, &str, &str), +) { + let metadata = url.replace("/development/", "/preview/"); + let batch = |kind: &str, records: Vec| json!({"kind": kind, "session_id": preview_session, "generation": 1, "records": records}); + post( + client, + (&metadata, bearer), + batch( + "fingerprints", + vec![ + fingerprint("copy-run-2", "SELECT 2 AS id", "2026-03-01T11:00:00Z"), + fingerprint("copy-run-1", "SELECT 1 AS id", "2026-03-01T10:00:00Z"), + ], + ), + ) + .await; + post( + client, + (&metadata, bearer), + batch( + "consumption", + vec![ + consumption( + "copy-execution-1", + ("succeeded", "copy-run-1"), + "2026-03-01T10:00:00Z", + ), + consumption( + "copy-execution-2", + ("succeeded", "copy-run-2"), + "2026-03-01T11:00:00Z", + ), + consumption( + "copy-execution-3", + ("started", "copy-run-3"), + "2026-03-01T12:00:00Z", + ), + ], + ), + ) + .await; + post( + client, + (&metadata, bearer), + batch( + "observations", + vec![ + observation( + "copy-observation-2", + ("cursor_max", NODE, "2"), + ("copy-run-2", "2026-03-01T11:00:01Z"), + ), + observation( + "copy-observation-1", + ("cursor_max", NODE, "1"), + ("copy-run-1", "2026-03-01T10:00:01Z"), + ), + observation( + "copy-observation-3", + ("source_columns", "raw_copy", "{}"), + ("copy-run-1", "2026-03-01T10:00:02Z"), + ), + ], + ), + ) + .await; + let mut catalog = batch( + "catalog", + vec![json!({"state_database": null, "state_schema": "main", + "name": NODE, "relation_type": "table", "columns": [{"name": "id", "data_type": "INTEGER"}]})], + ); + catalog["scan"] = json!({"scan_id": Uuid::now_v7(), "location": {"database": null, "schema": "main"}, "complete": true}); + post(client, (&metadata, bearer), catalog).await; + + let read = PreviewCopyRead { + session_id: preview_session, + generation: 1, + }; + let copy = || async { + outbound + .preview_copy("preview", &read) + .await + .expect("preview copy") + }; + let ours = |copy: &PreviewCopy| { + ( + copy.fingerprints + .iter() + .filter(|f| f.node_name == NODE) + .map(|f| (f.run_id.clone(), f.definition.clone())) + .collect::>(), + copy.latest_runs + .iter() + .filter(|r| r.node_name == NODE) + .map(|r| r.run_id.clone()) + .collect::>(), + copy.observations + .iter() + .filter(|o| o.observation_id.starts_with("copy-")) + .map(|o| (o.kind.clone(), o.value.clone(), o.run_id.clone())) + .collect::>(), + copy.relations + .iter() + .filter(|r| r.name == NODE) + .map(|r| r.columns.len()) + .collect::>(), + ) + }; + let unsynced = copy().await; + assert_eq!(unsynced.synced_at, None, "nothing synced cleanly yet"); + assert_eq!( + ours(&unsynced), + ( + vec![("copy-run-2".into(), "SELECT 2 AS id".into())], + vec!["copy-run-2".into()], + vec![ + ("cursor_max".into(), "2".into(), "copy-run-2".into()), + ("source_columns".into(), "{}".into(), "copy-run-1".into()), + ], + vec![1], + ), + "newest fingerprint with its definition, newest succeeded run, newest observation per kind" + ); + + let fingerprints_clean = sync_report(preview_session, "fingerprints", None); + for report in [ + sync_report(preview_session, "catalog", None), + fingerprints_clean.clone(), + sync_report(preview_session, "catalog", None), + sync_report(preview_session, "consumption", Some("read")), + ] { + assert!( + outbound + .report_health(&report) + .await + .expect("report health") + ); + } + assert_eq!( + copy().await.synced_at, + None, + "consumption never synced cleanly" + ); + for report in [ + sync_report(preview_session, "consumption", None), + sync_report(preview_session, "fingerprints", Some("read")), + sync_report(preview_session, "observations", Some("read")), + ] { + assert!( + outbound + .report_health(&report) + .await + .expect("report health") + ); + } + let health_url = format!( + "{}/health", + metadata.split("/targets/").next().expect("location URL") + ); + let observations: Vec = client + .get(&health_url) + .bearer_auth(bearer) + .send() + .await + .expect("health") + .json() + .await + .expect("health reply"); + let fingerprints_at = observations + .iter() + .find(|observation| observation["report"]["event_id"] == json!(fingerprints_clean.event_id)) + .and_then(|observation| observation["received_at"].as_str()) + .map(|at| at.parse::>().expect("time")); + assert_eq!( + copy().await.synced_at, + fingerprints_at, + "the stalest required table's newest clean sync; failures and observations do not count" + ); + + let development = outbound + .preview_copy( + "development", + &PreviewCopyRead { + session_id, + generation: 1, + }, + ) + .await + .expect("development copy"); + assert_eq!( + ours(&development), + (vec![], vec![], vec![], vec![]), + "another target's mirror never leaks" + ); + + let refused = |session: Uuid, generation: i64, target: &str| { + let endpoint = format!( + "{}/preview-copy", + metadata.replace("/preview/", &format!("/{target}/")) + ); + let request = client + .post(endpoint) + .bearer_auth(bearer) + .json(&json!({"session_id": session, "generation": generation})); + async move { request.send().await.expect("refused copy").status() } + }; + assert_eq!( + refused(preview_session, 2, "preview").await, + StatusCode::CONFLICT + ); + assert_eq!( + refused(session_id, 1, "preview").await, + StatusCode::BAD_REQUEST + ); +} From c948bd07fa1089ffb93e44db059ff43a7ccbbf52 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:51:49 +0000 Subject: [PATCH 08/17] feat(plan): plan location previews from the synced warehouse copy A location's requested plan now fetches the server's synced copy and runs sqb plan against an adapter that answers listings, columns and fingerprints from it, never connecting to the warehouse. Cursor targets use the newest successful build's cursor_max, inputs are read when the run starts, and expression sources use the columns described for the same SQL. The plan reports when the copy synced and what stays unknown until the run, and refuses --defer-to. Fensu XSB079 confines the synced copy to sqb plan. --- crates/sqlbuild-cache/src/digest/constants.rs | 3 + .../src/digest/main/base64_text.rs | 27 + crates/sqlbuild-cache/src/digest/main/mod.rs | 1 + .../native_commands/main/_location_plan.rs | 42 +- .../native_commands/main/_location_work.rs | 7 +- .../src/plan_display/constants.rs | 8 + .../_helpers/execution_history/sqlite.rs | 5 +- .../_helpers/execution_history/statements.rs | 5 +- .../src/bindings/_helpers/native_host/mod.rs | 2 +- .../_helpers/native_host/registration.rs | 5 + .../_helpers/native_host/sqlite_connection.rs | 96 +++- .../_helpers/native_host/sqlite_cursor.rs | 99 ---- .../_helpers/native_host/synced_copy.rs | 511 ++++++++++++++++++ .../bindings/_helpers/plan_command/plan.rs | 106 +++- .../_helpers/plan_output/plan_json.rs | 7 +- .../_helpers/planner_warehouse/cursors.rs | 46 +- .../planner_warehouse/source_columns.rs | 42 +- .../src/bindings/_helpers/recall/constants.rs | 3 - .../src/bindings/_helpers/recall/rendering.rs | 29 +- fensu.toml | 12 +- scripts/fensu_policy/_helpers/unused_state.py | 14 +- scripts/fensu_policy/constants.py | 9 + .../rules/observation_boundary.py | 59 +- src/sqlbuild/_native.pyi | 1 + .../main/build/test_cursor_observations.py | 4 +- .../scripts/fensu_policy/_test_types.py | 1 + .../scripts/fensu_policy/helpers.py | 11 +- .../fensu_policy/test_observation_boundary.py | 33 +- .../publication/_test_types.py | 10 + .../publication/catalog/helpers.py | 25 + .../publication/test_plan_preview.py | 246 +++++++++ 31 files changed, 1288 insertions(+), 181 deletions(-) create mode 100644 crates/sqlbuild-cache/src/digest/main/base64_text.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_cursor.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py diff --git a/crates/sqlbuild-cache/src/digest/constants.rs b/crates/sqlbuild-cache/src/digest/constants.rs index b2f4bd45bc..c059f2f237 100644 --- a/crates/sqlbuild-cache/src/digest/constants.rs +++ b/crates/sqlbuild-cache/src/digest/constants.rs @@ -51,3 +51,6 @@ pub(crate) const DIRECT_URL_FILE: &str = "direct_url.json"; /// Inputs at least this large are hashed across the rayon pool; the digest is the same either way. pub(crate) const PARALLEL_HASH_BYTES: usize = 1 << 20; +/// The standard base64 alphabet. +pub const BASE64_ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; diff --git a/crates/sqlbuild-cache/src/digest/main/base64_text.rs b/crates/sqlbuild-cache/src/digest/main/base64_text.rs new file mode 100644 index 0000000000..69c91e1268 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/base64_text.rs @@ -0,0 +1,27 @@ +//! Standard padded base64 text of bytes, as Python's `base64.b64encode(data).decode()` gives. + +use crate::digest::constants::BASE64_ALPHABET; + +/// Standard padded base64 of `data`. +#[must_use] +pub fn base64_text(data: &[u8]) -> String { + let mut encoded = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let bytes = [ + chunk[0], + *chunk.get(1).unwrap_or(&0), + *chunk.get(2).unwrap_or(&0), + ]; + let triple = u32::from(bytes[0]) << 16 | u32::from(bytes[1]) << 8 | u32::from(bytes[2]); + for index in 0..4 { + if index <= chunk.len() { + encoded.push(char::from( + BASE64_ALPHABET[(triple >> (18 - 6 * index) & 63) as usize], + )); + } else { + encoded.push('='); + } + } + } + encoded +} diff --git a/crates/sqlbuild-cache/src/digest/main/mod.rs b/crates/sqlbuild-cache/src/digest/main/mod.rs index 3d1fb78930..d9de0036e4 100644 --- a/crates/sqlbuild-cache/src/digest/main/mod.rs +++ b/crates/sqlbuild-cache/src/digest/main/mod.rs @@ -1,3 +1,4 @@ +pub mod base64_text; pub mod bytes_digest; pub mod compiled_code_identity; pub mod content_digest; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs index 0f6ed68932..bf7c4e627c 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -1,14 +1,18 @@ -//! A requested plan runs as an ordinary `sqb plan --json` child on the server-frozen node keys. +//! A requested plan runs as a `sqb plan --json` child on frozen node keys, from the synced copy. use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Stdio; use serde_json::Value; +use sqlbuild_core::plan_display::constants::PLAN_SYNCED_COPY_ENV; +use sqlbuild_orchestration::platform::models::LocationWorkClient; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::warehouse_metadata::models::PreviewCopyRead; use tokio::io::AsyncReadExt; use tokio::process::Child; +use uuid::Uuid; use crate::argv::main::accepted_settings_arguments::accepted_settings_arguments; use crate::native_commands::_helpers::execution::location_child; @@ -21,9 +25,39 @@ use crate::native_commands::models::LocationServe; pub(super) async fn plan_location( (options, script): (&LocationServe, &Path), (selection, settings): (&BTreeSet, &BTreeMap), + (client, session_id, generation): (&LocationWorkClient, Uuid, i64), stopped: impl Future>, ) -> Result, ControlError> { - let mut child = match plan_child((options, script), (selection, settings)) { + let read = PreviewCopyRead { + session_id, + generation, + }; + let copy = match client.preview_copy(&options.target, &read).await { + Ok(copy) => copy, + Err(error) => { + return Ok(Some(plan_failure(&format!( + "the synced warehouse copy is unavailable: {error}" + )))); + } + }; + let path: PathBuf = + std::env::temp_dir().join(format!("sqlbuild-plan-copy-{}.json", Uuid::now_v7())); + std::fs::write( + &path, + serde_json::to_vec(©).map_err(std::io::Error::other)?, + )?; + let planned = run_plan((options, script), (selection, settings), &path, stopped).await; + drop(std::fs::remove_file(&path)); + planned +} + +async fn run_plan( + (options, script): (&LocationServe, &Path), + (selection, settings): (&BTreeSet, &BTreeMap), + copy: &Path, + stopped: impl Future>, +) -> Result, ControlError> { + let mut child = match plan_child((options, script), (selection, settings), copy) { Ok(child) => child?, Err(message) => return Ok(Some(plan_failure(&message))), }; @@ -57,6 +91,7 @@ pub(super) async fn plan_location( fn plan_child( (options, script): (&LocationServe, &Path), (selection, settings): (&BTreeSet, &BTreeMap), + copy: &Path, ) -> Result, String> { let flags = accepted_settings_arguments(PLAN_COMMAND, settings)?; let mut command = location_child(options, script); @@ -78,6 +113,7 @@ fn plan_child( Ok(command .env_remove("SQLBUILD_LOCATION_TOKEN") .env_remove("SQLBUILD_RUN_TOKEN") + .env(PLAN_SYNCED_COPY_ENV, copy) .stdout(Stdio::piped()) .spawn()) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index 41b65faff0..cc51433faa 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::platform::models::{ }; use sqlbuild_orchestration::store::errors::ControlError; use tokio::time::Instant; +use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::check_location_connection; @@ -56,7 +57,8 @@ pub(super) async fn run_location_work( () = tokio::time::sleep_until(deadline) => Ok(()), } }; - let checked = perform_work((options, script), claim.request.work, expires).await?; + let copy = (client, poll.session_id, claim.generation); + let checked = perform_work((options, script), claim.request.work, copy, expires).await?; let Some((outcome, local_files)) = checked else { drop(write_output(&[( Stream::Stderr, @@ -122,6 +124,7 @@ pub(super) async fn run_location_work( async fn perform_work( (options, script): (&LocationServe, &Path), work: LocationWork, + copy: (&LocationWorkClient, Uuid, i64), expires: impl Future>, ) -> Result)>, ControlError> { let message = match &work { @@ -150,7 +153,7 @@ async fn perform_work( selection, settings, .. - } => plan_location((options, script), (&selection, &settings), expires) + } => plan_location((options, script), (&selection, &settings), copy, expires) .await? .map(|document| (LocationWorkOutcome::Plan { document }, Vec::new())), }) diff --git a/crates/sqlbuild-core/src/plan_display/constants.rs b/crates/sqlbuild-core/src/plan_display/constants.rs index aa6211db6f..19fb27808d 100644 --- a/crates/sqlbuild-core/src/plan_display/constants.rs +++ b/crates/sqlbuild-core/src/plan_display/constants.rs @@ -35,3 +35,11 @@ pub(crate) const MIGRATION_INCOMPATIBLE: &str = "incompatible"; pub(crate) const WATERMARK_STRATEGY: &str = "watermark"; pub(crate) const QUERY_DIFF_INDENT: &str = " "; pub(crate) const UNIFIED_DIFF_HEADER_PREFIXES: [&str; 3] = ["---", "+++", "@@"]; +/// Names the synced-copy file a location writes for a plan preview; set, `sqb plan` never connects. +pub const PLAN_SYNCED_COPY_ENV: &str = "SQLBUILD_PLAN_SYNCED_COPY"; +/// The plan JSON key describing the synced copy a preview planned from. +pub const SYNCED_COPY_KEY: &str = "synced_copy"; +/// Why a preview refuses `--defer-to`: deferral reads another target's live state. +pub const SYNCED_COPY_DEFER_REFUSAL: &str = + "plan previews from the synced copy cannot use --defer-to"; +pub const SYNCED_COPY_DEFER_HELP: &str = "Run re-plans live, including deferral."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs index ea38f66ca7..2fcc1fd162 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs @@ -52,8 +52,9 @@ use crate::bindings::_helpers::execution_history::statements::{ column, execute, execute_plain, execute_query, fetch_one, tuple_of, }; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, sqlite_params}; -use crate::bindings::_helpers::native_host::sqlite_cursor::SqliteCursor; +use crate::bindings::_helpers::native_host::sqlite_connection::{ + SqliteConnection, SqliteCursor, sqlite_params, +}; use crate::bindings::_helpers::native_threads::reentrant_lock::ReentrantLock; use crate::bindings::_helpers::records::fields::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs index cbb9f83ccf..a595566ade 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs @@ -5,8 +5,9 @@ use pyo3::types::PyTuple; use sqlbuild_state::execution_history::models::{Query, SqlParam}; use crate::bindings::_helpers::execution_history::conversion::python_params; -use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, sqlite_params}; -use crate::bindings::_helpers::native_host::sqlite_cursor::{SqliteCursor, SqliteRow}; +use crate::bindings::_helpers::native_host::sqlite_connection::{ + SqliteConnection, SqliteCursor, SqliteRow, sqlite_params, +}; /// `target.execute(sql, params)`. pub(crate) fn execute<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 016fab107c..59eca3d642 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -15,8 +15,8 @@ pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; -pub(crate) mod sqlite_cursor; pub(crate) mod state_column_evolution; +pub(crate) mod synced_copy; pub(crate) mod warehouse_catalog; pub(crate) mod warehouse_metadata; pub(crate) mod warehouse_statistics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs index 95ebe934fd..0439de3ce2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs @@ -6,11 +6,16 @@ use pyo3::wrap_pyfunction; use sqlbuild_host::entries::main::host_entry_labels::host_entry_labels; use crate::bindings::_helpers::native_host::orchestration_declarations::declaration_site; +use crate::bindings::_helpers::native_host::synced_copy::WarehouseAccessRefusedError; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add( "HOST_ENTRIES", PyTuple::new(module.py(), host_entry_labels())?, )?; + module.add( + "WarehouseAccessRefusedError", + module.py().get_type::(), + )?; module.add_function(wrap_pyfunction!(declaration_site, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs index c82f67e182..d2132bea74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_connection.rs @@ -1,18 +1,19 @@ -//! Rust-owned SQLite connection; Python views exist only for connection overrides. +//! Rust-owned SQLite connection and its rows; Python views exist only for connection overrides. use std::collections::VecDeque; use std::path::PathBuf; use std::sync::{Arc, Mutex, PoisonError}; -use pyo3::exceptions::PyValueError; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::{PyIndexError, PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; +use pyo3::types::PyBytes; use rusqlite::types::Value; use rusqlite::{Connection, Error, ErrorCode, OpenFlags, params_from_iter}; use sqlbuild_state::execution_history::constants::MEMORY_PATH; use sqlbuild_state::execution_history::models::SqlParam; use crate::bindings::_helpers::boundary::python_calls::imported; -use crate::bindings::_helpers::native_host::sqlite_cursor::{SqliteCursor, SqliteRow}; /// Preserve DB-API exception classes at the public storage-error boundary. fn database_error(py: Python<'_>, error: Error) -> PyErr { @@ -187,3 +188,92 @@ impl SqliteConnection { Ok(()) } } + +#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteRow")] +pub(crate) struct SqliteRow { + pub(crate) columns: Arc>, + pub(crate) values: Vec, +} + +impl SqliteRow { + pub(crate) fn value<'py>(&self, py: Python<'py>, index: usize) -> PyResult> { + match self.values.get(index) { + Some(Value::Null) => Ok(py.None().into_bound(py)), + Some(Value::Integer(value)) => value.into_bound_py_any(py), + Some(Value::Real(value)) => value.into_bound_py_any(py), + Some(Value::Text(value)) => value.into_bound_py_any(py), + Some(Value::Blob(value)) => Ok(PyBytes::new(py, value).into_any()), + None => Err(PyIndexError::new_err("tuple index out of range")), + } + } + + pub(crate) fn column<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let index: usize = self + .columns + .iter() + .position(|column| column.eq_ignore_ascii_case(name)) + .ok_or_else(|| PyIndexError::new_err("No item with that key"))?; + self.value(py, index) + } +} + +#[pymethods] +impl SqliteRow { + fn __getitem__<'py>(&self, key: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(index) = key.extract::() { + let index = if index < 0 { + self.values.len().checked_add_signed(index) + } else { + Some(index as usize) + }; + return self.value(key.py(), index.unwrap_or(usize::MAX)); + } + if let Ok(name) = key.extract::() { + return self.column(key.py(), &name); + } + Err(PyTypeError::new_err("Index must be int or string")) + } +} + +#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteCursor")] +pub(crate) struct SqliteCursor { + pub(crate) rows: Mutex>, + pub(crate) last_row_id: i64, +} + +impl SqliteCursor { + pub(crate) fn next_row(&self) -> Option { + self.rows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .pop_front() + } +} + +#[pymethods] +impl SqliteCursor { + fn __iter__(slf: Bound<'_, Self>) -> Bound<'_, Self> { + slf + } + + fn __next__(&self) -> Option { + self.next_row() + } + + fn fetchone(&self) -> Option { + self.next_row() + } + + fn fetchall(&self) -> Vec { + self.rows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .drain(..) + .collect() + } + + #[getter] + fn lastrowid(&self) -> i64 { + self.last_row_id + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_cursor.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_cursor.rs deleted file mode 100644 index 2a9796015c..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/sqlite_cursor.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Native SQLite rows, with a DB-API view for the existing connection override boundary. - -use std::collections::VecDeque; -use std::sync::{Arc, Mutex, PoisonError}; - -use pyo3::IntoPyObjectExt; -use pyo3::exceptions::{PyIndexError, PyTypeError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; -use pyo3::types::PyBytes; -use rusqlite::types::Value; - -#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteRow")] -pub(crate) struct SqliteRow { - pub(crate) columns: Arc>, - pub(crate) values: Vec, -} - -impl SqliteRow { - pub(crate) fn value<'py>(&self, py: Python<'py>, index: usize) -> PyResult> { - match self.values.get(index) { - Some(Value::Null) => Ok(py.None().into_bound(py)), - Some(Value::Integer(value)) => value.into_bound_py_any(py), - Some(Value::Real(value)) => value.into_bound_py_any(py), - Some(Value::Text(value)) => value.into_bound_py_any(py), - Some(Value::Blob(value)) => Ok(PyBytes::new(py, value).into_any()), - None => Err(PyIndexError::new_err("tuple index out of range")), - } - } - - pub(crate) fn column<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { - let index: usize = self - .columns - .iter() - .position(|column| column.eq_ignore_ascii_case(name)) - .ok_or_else(|| PyIndexError::new_err("No item with that key"))?; - self.value(py, index) - } -} - -#[pymethods] -impl SqliteRow { - fn __getitem__<'py>(&self, key: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(index) = key.extract::() { - let index = if index < 0 { - self.values.len().checked_add_signed(index) - } else { - Some(index as usize) - }; - return self.value(key.py(), index.unwrap_or(usize::MAX)); - } - if let Ok(name) = key.extract::() { - return self.column(key.py(), &name); - } - Err(PyTypeError::new_err("Index must be int or string")) - } -} - -#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteCursor")] -pub(crate) struct SqliteCursor { - pub(crate) rows: Mutex>, - pub(crate) last_row_id: i64, -} - -impl SqliteCursor { - pub(crate) fn next_row(&self) -> Option { - self.rows - .lock() - .unwrap_or_else(PoisonError::into_inner) - .pop_front() - } -} - -#[pymethods] -impl SqliteCursor { - fn __iter__(slf: Bound<'_, Self>) -> Bound<'_, Self> { - slf - } - - fn __next__(&self) -> Option { - self.next_row() - } - - fn fetchone(&self) -> Option { - self.next_row() - } - - fn fetchall(&self) -> Vec { - self.rows - .lock() - .unwrap_or_else(PoisonError::into_inner) - .drain(..) - .collect() - } - - #[getter] - fn lastrowid(&self) -> i64 { - self.last_row_id - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs new file mode 100644 index 0000000000..b089d388af --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs @@ -0,0 +1,511 @@ +//! A plan preview's adapter: warehouse reads come from the location's mirrored copy, nothing connects. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Mutex, PoisonError}; + +use chrono::SecondsFormat; +use pyo3::create_exception; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use serde::Deserialize; +use sqlbuild_analysis::fingerprint_storage::constants::FINGERPRINT_COLUMNS; +use sqlbuild_analysis::fingerprint_storage::main::read_latest_sql::read_latest_sql; +use sqlbuild_cache::digest::main::base64_text::base64_text; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; +use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, STATE_TABLE_NAMES}; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_orchestration::warehouse_metadata::models::{PreviewCopy, WarehouseCatalogRecord}; +use sqlbuild_state::execution_history::constants::MEMORY_PATH; +use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; + +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, SqliteCursor}; +use crate::bindings::_helpers::records::fields::field as native_field; + +create_exception!( + sqlbuild._native, + WarehouseAccessRefusedError, + PyRuntimeError, + "Raised when a plan preview reaches past the synced copy for the warehouse." +); + +/// Adapter methods that only describe the dialect, never reaching the warehouse. +const PASSTHROUGH_NAMES: [&str; 11] = [ + "adapter_name", + "default_database", + "default_schema", + "default_table_promotion_mode", + "expression_inference_profile", + "metadata_inspection_concurrency", + "python_functions_inherit_default_namespace", + "requires_derived_table_aliases", + "sql_analysis_dialect", + "sql_analysis_dialect_name", + "star_exclude_keyword", +]; +/// Rendering and capability methods pass through; every other unknown attribute is refused. +const PASSTHROUGH_PREFIXES: [&str; 2] = ["render_", "supports_"]; +/// Local table holding the copy's fingerprints; previews read it through per-location views. +const FINGERPRINT_STORE: &str = "__sqlbuild_synced_fingerprints"; +const FINGERPRINT_STORE_COLUMNS: [&str; 2] = ["state_database", "state_schema"]; +const MODEL_NODE_TYPE: &str = "model"; + +/// Items a preview reports as known only once the run plans live. +pub(crate) const PREVIEW_UNKNOWNS: [&str; 4] = [ + "source freshness", + "retention", + "column rename support", + "sources outside mirrored schemas", +]; + +/// A source's described columns, keyed to the exact SQL that was described. +#[derive(Deserialize)] +struct DescribedColumns { + sql_hash: String, + columns: Vec, +} + +fn refused(access: &str) -> pyo3::PyErr { + WarehouseAccessRefusedError::new_err(format!( + "plan previews read only the synced copy; refused warehouse access: {access}" + )) +} + +fn same(left: Option<&str>, right: Option<&str>) -> bool { + left.unwrap_or_default().to_lowercase() == right.unwrap_or_default().to_lowercase() +} + +/// A listing call's database, schemas and optional relation names. +type ListingScope = (Option, Option>, Option>); + +/// The `database`, `schemas` and optional `names` of a listing call; its connection goes unused. +fn scope( + py: Python<'_>, + function: &str, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + function, + options, + (&["connection", "database", "schemas"], &["names"]), + )?; + Ok(( + arguments.value("database")?.extract()?, + arguments.value("schemas")?.extract()?, + arguments.value("names")?.extract()?, + )) +} + +/// Whether `wanted` is unset or holds `value`, compared as the warehouse folds case. +fn any_same(wanted: Option<&[String]>, value: &str) -> bool { + let Some(wanted) = wanted else { + return true; + }; + for item in wanted { + if same(Some(value), Some(item)) { + return true; + } + } + false +} + +fn literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +/// A connection that refuses every use; like the adapter, it lives in the adapter contract's module. +#[pyclass( + frozen, + module = "sqlbuild.adapter.contract", + name = "SyncedCopyConnection" +)] +pub(crate) struct SyncedCopyConnection; + +#[pymethods] +impl SyncedCopyConnection { + fn __getattr__(&self, name: &str) -> PyResult<()> { + Err(refused(&format!("connection.{name}"))) + } +} + +/// Answers listing, column and fingerprint reads from the copy; dialect methods use `adapter`. +#[pyclass( + frozen, + module = "sqlbuild.adapter.contract", + name = "SyncedCopyAdapter" +)] +pub(crate) struct SyncedCopyAdapter { + adapter: Py, + copy: PreviewCopy, + store: SqliteConnection, + views: Mutex, String)>>, + unknowns: Mutex>, + windows: Mutex>, +} + +impl SyncedCopyAdapter { + /// The facade over `adapter` answering from the copy written at `path`. + pub(crate) fn load(py: Python<'_>, adapter: Bound<'_, PyAny>, path: &str) -> PyResult { + let text: String = std::fs::read_to_string(path)?; + let copy: PreviewCopy = serde_json::from_str(&text) + .map_err(|error| PyRuntimeError::new_err(format!("invalid synced copy: {error}")))?; + let store: SqliteConnection = SqliteConnection::open(py, MEMORY_PATH)?; + let columns: Vec<&str> = FINGERPRINT_STORE_COLUMNS + .iter() + .chain(FINGERPRINT_COLUMNS.iter()) + .copied() + .collect(); + store.execute_native( + py, + &format!("CREATE TABLE {FINGERPRINT_STORE} ({})", columns.join(", ")), + &[], + )?; + let placeholders: String = vec!["?"; columns.len()].join(", "); + for record in ©.fingerprints { + let values: [Option; 14] = [ + record.state_database.clone(), + Some(record.state_schema.clone()), + Some(record.node_type.clone()), + Some(record.node_name.clone()), + record.target_database.clone(), + record.target_schema.clone(), + record.target_name.clone(), + Some(record.run_id.clone()), + Some(record.definition_hash.clone()), + Some(record.version_hash.clone()), + Some(record.schema_fingerprint.clone()), + Some(base64_text(record.definition.as_bytes())), + Some(base64_text(record.metadata_json.as_bytes())), + Some(record.ts.to_rfc3339_opts(SecondsFormat::AutoSi, true)), + ]; + let values: Vec = values + .into_iter() + .map(|value| { + value.map_or(rusqlite::types::Value::Null, rusqlite::types::Value::Text) + }) + .collect(); + store.execute_native( + py, + &format!("INSERT INTO {FINGERPRINT_STORE} VALUES ({placeholders})"), + &values, + )?; + } + Ok(Self { + adapter: adapter.unbind(), + copy, + store, + views: Mutex::new(Vec::new()), + unknowns: Mutex::new(BTreeSet::new()), + windows: Mutex::new(BTreeMap::new()), + }) + } + + /// When the copy was synced, as ISO 8601 text; `None` before every required table synced. + pub(crate) fn synced_at(&self) -> Option { + self.copy + .synced_at + .map(|at| at.to_rfc3339_opts(SecondsFormat::AutoSi, true)) + } + + /// What the preview could not know, in a stable order. + pub(crate) fn unknowns(&self) -> Vec { + let mut unknowns: Vec = PREVIEW_UNKNOWNS + .iter() + .map(|item| (*item).to_owned()) + .collect(); + unknowns.extend( + self.unknowns + .lock() + .unwrap_or_else(PoisonError::into_inner) + .iter() + .cloned(), + ); + unknowns + } + + /// Note one item the preview reports as unknown until the run plans live. + pub(crate) fn mark_unknown(&self, item: String) { + self.unknowns + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(item); + } + + /// Note that `model` reads from `start` up to the latest data when the run starts. + pub(crate) fn mark_window(&self, model: &str, start: &str) { + self.windows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(model.to_owned(), start.to_owned()); + } + + /// Each model's window start, read up to the latest data when the run starts. + pub(crate) fn windows(&self) -> BTreeMap { + self.windows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// The model's cursor `MAX` text, only when observed by its newest successful build. + pub(crate) fn target_max(&self, model: &str) -> Option<&str> { + for run in &self.copy.latest_runs { + if run.node_type != MODEL_NODE_TYPE || run.node_name != model { + continue; + } + for observation in &self.copy.observations { + if observation.kind == CURSOR_MAX_KIND + && observation.node == model + && observation.run_id == run.run_id + && observation.state_schema == run.state_schema + && observation.state_database == run.state_database + { + return Some(observation.value.as_str()); + } + } + } + None + } + + /// The source's columns as last described, only when it described exactly `sql`. + pub(crate) fn described_columns( + &self, + source: &str, + sql: &str, + ) -> PyResult>> { + let hash: String = sha256_hex(sql.as_bytes()); + for observation in &self.copy.observations { + if observation.kind != SOURCE_COLUMNS_KIND || observation.node != source { + continue; + } + let described: DescribedColumns = + serde_json::from_str(&observation.value).map_err(|error| { + PyRuntimeError::new_err(format!( + "invalid source_columns observation for {source}: {error}" + )) + })?; + if described.sql_hash == hash { + return Ok(Some(described.columns)); + } + } + Ok(None) + } + + fn listed( + &self, + database: Option<&str>, + schemas: Option<&[String]>, + names: Option<&[String]>, + ) -> Vec<&WarehouseCatalogRecord> { + let mut listed: Vec<&WarehouseCatalogRecord> = Vec::new(); + for record in &self.copy.relations { + if (STATE_TABLE_NAMES.contains(&record.name.as_str()) + && record.name != FINGERPRINT_TABLE_NAME) + || !(database.is_none() + || record.state_database.is_none() + || same(record.state_database.as_deref(), database)) + || !any_same(schemas, &record.state_schema) + || !any_same(names, &record.name) + { + continue; + } + listed.push(record); + } + listed + } + + fn relation_info<'py>( + &self, + py: Python<'py>, + record: &WarehouseCatalogRecord, + ) -> PyResult> { + call_with( + &host::entry(py, AdapterHost::RelationInfo)?, + &[ + ( + "database", + optional_text(py, record.state_database.as_deref()), + ), + ("schema", PyString::new(py, &record.state_schema).into_any()), + ("name", PyString::new(py, &record.name).into_any()), + ( + "relation_type", + PyString::new(py, &record.relation_type).into_any(), + ), + ], + ) + } + + fn columns<'py>( + &self, + py: Python<'py>, + record: &WarehouseCatalogRecord, + ) -> PyResult> { + let column_info: Bound<'py, PyAny> = host::entry(py, AdapterHost::ColumnInfo)?; + let mut columns: Vec> = Vec::with_capacity(record.columns.len()); + for column in &record.columns { + columns.push(call_with( + &column_info, + &[ + ("name", PyString::new(py, &column.name).into_any()), + ("type", PyString::new(py, &column.data_type).into_any()), + ], + )?); + } + PyTuple::new(py, columns) + } +} + +fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { + match value { + Some(value) => PyString::new(py, value).into_any(), + None => py.None().into_bound(py), + } +} + +#[pymethods] +impl SyncedCopyAdapter { + #[pyo3(signature = (connection_config = None))] + fn connect(&self, connection_config: Option>) -> SyncedCopyConnection { + drop(connection_config); + SyncedCopyConnection + } + + #[pyo3(signature = (connection = None))] + fn close(&self, connection: Option>) { + drop(connection); + } + + #[pyo3(name = "list_relations", signature = (**options))] + fn relation_listing<'py>( + &self, + py: Python<'py>, + options: Option>, + ) -> PyResult> { + let (database, schemas, names) = scope(py, "list_relations", options)?; + let mut relations: Vec> = Vec::new(); + for record in self.listed(database.as_deref(), schemas.as_deref(), names.as_deref()) { + relations.push(self.relation_info(py, record)?); + } + PyTuple::new(py, relations) + } + + #[pyo3(name = "get_all_columns", signature = (**options))] + fn all_columns<'py>( + &self, + py: Python<'py>, + options: Option>, + ) -> PyResult> { + let (database, schemas, names) = scope(py, "get_all_columns", options)?; + let columns: Bound<'py, PyDict> = PyDict::new(py); + for record in self.listed(database.as_deref(), schemas.as_deref(), names.as_deref()) { + columns.set_item(&record.name, self.columns(py, record)?)?; + } + Ok(columns) + } + + #[pyo3(name = "get_columns_for_relations", signature = (*, connection, relations))] + fn columns_by_relation<'py>( + &self, + py: Python<'py>, + connection: Bound<'py, PyAny>, + relations: Bound<'py, PyAny>, + ) -> PyResult> { + drop(connection); + let found: Bound<'py, PyDict> = PyDict::new(py); + for relation in relations.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + let database: Option = native_field(&relation, "database")?.extract()?; + let schema: Option = native_field(&relation, "schema")?.extract()?; + let name: String = native_field(&relation, "name")?.extract()?; + let schemas: Option> = schema.clone().map(|schema| vec![schema]); + let names: [String; 1] = [name.clone()]; + if let Some(record) = self + .listed(database.as_deref(), schemas.as_deref(), Some(&names)) + .first() + { + found.set_item((database, schema, name), self.columns(py, record)?)?; + } + } + Ok(found) + } + + /// The latest-row read over this location's copied fingerprints, through a local view. + #[pyo3(signature = (*, database, schema))] + fn render_read_latest_fingerprints_sql( + &self, + py: Python<'_>, + database: Option, + schema: String, + ) -> PyResult { + let mut views = self.views.lock().unwrap_or_else(PoisonError::into_inner); + let key: (Option, String) = (database.clone(), schema.clone()); + let position: usize = match views.iter().position(|view| *view == key) { + Some(position) => position, + None => { + let mut condition: String = + format!("lower(state_schema) = lower({})", literal(&schema)); + if let Some(database) = &database { + condition.push_str(&format!( + " AND (state_database IS NULL OR lower(state_database) = lower({}))", + literal(database) + )); + } + self.store.execute_native( + py, + &format!( + "CREATE VIEW {FINGERPRINT_STORE}_{} AS SELECT {} FROM {FINGERPRINT_STORE} \ + WHERE {condition}", + views.len(), + FINGERPRINT_COLUMNS.join(", ") + ), + &[], + )?; + views.push(key); + views.len() - 1 + } + }; + Ok(read_latest_sql(&format!("{FINGERPRINT_STORE}_{position}"))) + } + + /// Only reads of the copied fingerprints run; any other statement would reach the warehouse. + #[pyo3(signature = (*, connection, sql))] + fn execute( + &self, + py: Python<'_>, + connection: Bound<'_, PyAny>, + sql: &str, + ) -> PyResult { + drop(connection); + if !sql.contains(&format!("{FINGERPRINT_STORE}_")) { + return Err(refused(sql.trim().lines().next().unwrap_or_default())); + } + self.store.execute_native(py, sql, &[]) + } + + #[pyo3(signature = (*, connection, sql))] + fn query_column_names(&self, connection: Bound<'_, PyAny>, sql: &str) -> PyResult<()> { + drop(connection); + Err(refused(&format!( + "DESCRIBE {}", + sql.trim().lines().next().unwrap_or_default() + ))) + } + + /// Values and dialect methods come from the real adapter; any other method is refused. + fn __getattr__<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let value: Bound<'py, PyAny> = native_field(self.adapter.bind(py), name)?; + if !value.is_callable() + || PASSTHROUGH_NAMES.contains(&name) + || PASSTHROUGH_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) + { + return Ok(value); + } + Err(refused(&format!("adapter.{name}"))) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 9ca83a6dce..8baa0f2968 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -5,6 +5,10 @@ use pyo3::prelude::{ }; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use pyo3::wrap_pyfunction; +use sqlbuild_config::command_connection::errors::CommandConnectionError; +use sqlbuild_core::plan_display::constants::{ + PLAN_SYNCED_COPY_ENV, SYNCED_COPY_DEFER_HELP, SYNCED_COPY_DEFER_REFUSAL, SYNCED_COPY_KEY, +}; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::keywords; @@ -16,11 +20,13 @@ use crate::bindings::_helpers::build_progress::progress_reporters::{ use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspection; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; +use crate::bindings::_helpers::command_runtime::runtime_errors::raised; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::plan_output::plan_json::format_plan_json; use crate::bindings::_helpers::plan_output::plan_text::format_plan; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; @@ -67,6 +73,33 @@ struct PlanInvocation<'py> { connection_progress: Bound<'py, PyAny>, planning_progress: Bound<'py, PyAny>, load_sources: Bound<'py, PyAny>, + /// Set for a location's plan preview: the adapter that answers from the synced copy. + synced_copy: Option>, +} + +/// The synced-copy adapter when a location asked for a preview; a preview never defers. +fn synced_copy<'py>( + request: &Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = request.py(); + let Ok(path) = std::env::var(PLAN_SYNCED_COPY_ENV) else { + return Ok(None); + }; + if native_field(request, "defer_to")?.is_truthy()? { + return Err(raised( + py, + CommandConnectionError::User { + code: None, + message: SYNCED_COPY_DEFER_REFUSAL.to_owned(), + help: Some(SYNCED_COPY_DEFER_HELP.to_owned()), + }, + )); + } + Ok(Some(Bound::new( + py, + SyncedCopyAdapter::load(py, adapter, &path)?, + )?)) } fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult> { @@ -135,6 +168,8 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult> = + synced_copy(request, native_field(&context, "adapter")?)?; Ok(PlanInvocation { project_dir, discovered, @@ -144,6 +179,7 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult( py, &[ ("discovered_inputs", invocation.discovered.clone()), - ("adapter", native_field(&invocation.context, "adapter")?), + ( + "adapter", + match &invocation.synced_copy { + Some(facade) => facade.clone().into_any(), + None => native_field(&invocation.context, "adapter")?, + }, + ), ("options", pipeline_options), ("hooks", connection_hooks(invocation)?), ], @@ -248,13 +290,15 @@ fn write_output( let plan: Bound<'_, PyAny> = native_field(pipeline, "plan_output")?; let python_entries: Bound<'_, PyAny> = native_field(pipeline, "python_plan_entries")?; if native_field(request, "json_output")?.is_truthy()? { - let text: String = format_plan_json( - py, - Some(keywords( - py, - &[("plan", plan), ("python_plan_entries", python_entries)], - )?), - )?; + let mut options: Vec<(&str, Bound<'_, PyAny>)> = + vec![("plan", plan), ("python_plan_entries", python_entries)]; + if let Some(facade) = &invocation.synced_copy { + options.push(( + SYNCED_COPY_KEY, + synced_copy_json(py, facade.get())?.into_any(), + )); + } + let text: String = format_plan_json(py, Some(keywords(py, &options)?))?; imported(py, "builtins", "print")?.call1((text,))?; return Ok(()); } @@ -285,7 +329,51 @@ fn write_output( ], )?), )?; - print_line(py, &format!("\n{text}")) + print_line(py, &format!("\n{text}"))?; + if let Some(facade) = &invocation.synced_copy { + print_line(py, &synced_copy_text(facade.get()))?; + } + Ok(()) +} + +/// The plan JSON's account of the synced copy: when it synced, window starts and unknowns. +fn synced_copy_json<'py>( + py: Python<'py>, + facade: &SyncedCopyAdapter, +) -> PyResult> { + let synced: Bound<'py, PyDict> = PyDict::new(py); + synced.set_item("synced_at", facade.synced_at())?; + synced.set_item("cursor_window_starts", facade.windows())?; + synced.set_item("unknown_until_run", facade.unknowns())?; + Ok(synced) +} + +/// The preview footer: when the copy was synced and what stays unknown until the run. +fn synced_copy_text(facade: &SyncedCopyAdapter) -> String { + let mut lines: Vec = vec![format!( + "Planned from the synced copy ({}); the run re-plans live.", + facade.synced_at().map_or_else( + || "not fully synced yet".to_owned(), + |at| format!("synced {at}") + ) + )]; + let windows = facade.windows(); + if !windows.is_empty() { + lines.push("Cursor windows (up to the latest data when the run starts):".to_owned()); + lines.extend( + windows + .iter() + .map(|(model, start)| format!(" - {model}: from {start}")), + ); + } + lines.push("Unknown until run:".to_owned()); + lines.extend( + facade + .unknowns() + .into_iter() + .map(|item| format!(" - {item}")), + ); + lines.join("\n") } /// Run one `sqb plan` request. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 94b43956a1..551852218b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_outputs::json_report::indented_json; @@ -748,7 +749,7 @@ pub(crate) fn format_plan_json( py, "format_plan_json", options, - (&["plan"], &["python_plan_entries"]), + (&["plan"], &["python_plan_entries", SYNCED_COPY_KEY]), )?; let plan: Bound<'_, PyAny> = arguments.value("plan")?; let python_entries: Vec> = if arguments.passed("python_plan_entries")? { @@ -820,6 +821,10 @@ pub(crate) fn format_plan_json( if metadata.is_truthy()? { result.set_item("metadata", metadata)?; } + let synced_copy: Bound<'_, PyAny> = arguments.value(SYNCED_COPY_KEY)?; + if !synced_copy.is_none() { + result.set_item(SYNCED_COPY_KEY, synced_copy)?; + } indented_json(result.as_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index b1eada0557..fbe46a5102 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -13,6 +13,7 @@ use crate::bindings::_helpers::cursor::scalars::render_value; use crate::bindings::_helpers::cursor::scalars::to_inclusive; use crate::bindings::_helpers::model_config::typed_config::{config_cursor_bound, model_values}; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ by_name, config_str, model_key, native_member, optional_text, planner_error, planner_model, @@ -1150,6 +1151,45 @@ fn assemble<'py>( Ok(snapshots) } +/// Plan previews: target `MAX` values from the synced copy; inputs are read when the run starts. +fn synced_copy_values<'py>( + facade: &SyncedCopyAdapter, + models: &mut [CursorModel<'py>], +) -> PyResult> { + let mut values = Values::new(); + for info in models { + let reads_inputs: bool = !info.upstreams.is_empty(); + info.upstreams.clear(); + let observed: Option<&str> = info + .target_tag + .as_ref() + .and_then(|_| facade.target_max(&info.model_name)); + match (&info.target_tag, observed) { + (Some(tag), Some(observed)) => { + let value = text(info.effective_grain.py(), observed); + values.insert(tag.clone(), parse_warehouse_cursor(&value)?); + if reads_inputs { + facade.mark_window(&info.model_name, observed); + } + if !info.start_cursor_config.is_none() + && !native_field(&info.start_cursor_config, "max_ahead")?.is_none() + { + facade.mark_unknown(format!("future-cursor safety of {}", info.model_name)); + } + } + (Some(_), None) => facade.mark_unknown(format!( + "cursor position of {} (no successful build recorded it)", + info.model_name + )), + (None, _) if reads_inputs => { + facade.mark_unknown(format!("cursor window of {}", info.model_name)); + } + (None, _) => {} + } + } + Ok(values) +} + /// Cursor MIN/MAX values of the selected incremental models, by model name. pub(crate) fn cursor_snapshots<'py>( request: &SnapshotRequest<'py>, @@ -1163,10 +1203,14 @@ pub(crate) fn cursor_snapshots<'py>( seeds: by_name(&request.project, "seeds")?, functions: by_name(&request.project, "functions")?, }; - let models = collect_models(request, existing_relations, inputs, &maps)?; + let mut models = collect_models(request, existing_relations, inputs, &maps)?; if models.is_empty() { return Ok(PyDict::new(py)); } + if let Ok(facade) = request.adapter.downcast::() { + let values = synced_copy_values(facade.get(), &mut models)?; + return assemble(py, &models, (&values, &HashSet::new())); + } let queries = physical_queries(&models); let started = Instant::now(); let concurrency = inspection_concurrency(&request.adapter, &request.connection)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index 5ec08e1c83..fef2043627 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::observations::source_capture::capture_source_columns; use crate::bindings::_helpers::planner_phases::objects::{frozenset, text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ @@ -94,22 +95,41 @@ pub(crate) fn source_inspection<'py>( if !expression.is_none() { if native_field(entry, "type_enforcement")?.is_truthy()? { let sql: String = expression.extract()?; - let names = call_with( - &native_field(adapter, "query_column_names")?, - &[("connection", connection.clone()), ("sql", expression)], - )?; + let source: Bound<'py, PyAny> = native_field(entry, "name")?; + let source_name: String = source.extract()?; + let described: Vec = match adapter.downcast::() { + Ok(facade) => match facade.get().described_columns(&source_name, &sql)? { + Some(described) => described, + None => { + facade.get().mark_unknown(format!( + "query columns of expression source {source_name} (planned as declared)" + )); + native_field(entry, "columns")? + .try_iter()? + .map(|column| native_field(&column?, "name")?.extract()) + .collect::>()? + } + }, + Err(_) => { + let names = call_with( + &native_field(adapter, "query_column_names")?, + &[("connection", connection.clone()), ("sql", expression)], + )?; + let described: Vec = names + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + capture_source_columns(&source_name, &sql, &described); + described + } + }; let mut columns: Vec<_> = Vec::new(); - let mut described: Vec = Vec::new(); - for name in names.try_iter()? { - let name: Bound<'py, PyAny> = name?; - described.push(name.extract()?); + for name in &described { columns.push(call_with( &column_info, - &[("name", name), ("type", text(py, ""))], + &[("name", text(py, name)), ("type", text(py, ""))], )?); } - let source: Bound<'py, PyAny> = native_field(entry, "name")?; - capture_source_columns(&source.extract::()?, &sql, &described); result.set_item(source, tuple(py, columns)?)?; } continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 2a10b7918f..615b0a4fa3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -35,9 +35,6 @@ pub(crate) const DEFAULT_MAX_CELL_BYTES: usize = 4096; pub(crate) const JSON_FORMAT: &str = "json"; /// How text output shows SQL NULL. pub(crate) const NULL_TEXT: &str = "NULL"; -/// The standard base64 alphabet binary JSON cells are encoded with. -pub(crate) const BASE64_ALPHABET: &[u8; 64] = - b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; /// Join keys `sqb recall schema` points out. pub(crate) const JOIN_KEYS: [&str; 5] = [ "invocation_id", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs index f2a5cd4abc..ce75b9d4e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -2,6 +2,7 @@ use pyo3::prelude::{PyResult, Python}; use serde_json::{Map, Value, json}; +use sqlbuild_cache::digest::main::base64_text::base64_text; use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_recall::records::models::RecallValue; use sqlbuild_recall::schema::models::Dialect; @@ -9,31 +10,7 @@ use sqlbuild_recall::store::models::{DescribedColumn, DescribedRelation, QueryRe use crate::bindings::_helpers::query_command::text_output::render_text; use crate::bindings::_helpers::recall::clickhouse_calls::QueryStatistics; -use crate::bindings::_helpers::recall::constants::{ - BASE64_ALPHABET, JOIN_KEYS, JSON_FORMAT, NULL_TEXT, -}; - -fn base64(data: &[u8]) -> String { - let mut encoded = String::with_capacity(data.len().div_ceil(3) * 4); - for chunk in data.chunks(3) { - let bytes = [ - chunk[0], - *chunk.get(1).unwrap_or(&0), - *chunk.get(2).unwrap_or(&0), - ]; - let triple = u32::from(bytes[0]) << 16 | u32::from(bytes[1]) << 8 | u32::from(bytes[2]); - for index in 0..4 { - if index <= chunk.len() { - encoded.push(char::from( - BASE64_ALPHABET[(triple >> (18 - 6 * index) & 63) as usize], - )); - } else { - encoded.push('='); - } - } - } - encoded -} +use crate::bindings::_helpers::recall::constants::{JOIN_KEYS, JSON_FORMAT, NULL_TEXT}; fn truncated(text: &str, max_bytes: Option) -> String { let Some(limit) = max_bytes.filter(|limit| text.len() > *limit) else { @@ -84,7 +61,7 @@ fn json_cell(value: &RecallValue) -> Value { RecallValue::Text(text) => json!(text), RecallValue::Blob(data) => match std::str::from_utf8(data) { Ok(text) => json!(text), - Err(_) => json!({ "base64": base64(data) }), + Err(_) => json!({ "base64": base64_text(data) }), }, } } diff --git a/fensu.toml b/fensu.toml index 964d82b43d..c6d94a1e56 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1633,7 +1633,7 @@ rule_modules = [ "scripts.fensu_policy.rules.observation_boundary", "scripts.fensu_policy.rules.unused_state", ] -select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078"] +select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "XSB079"] [targets.rust.rule_options.XSB076] schema_paths = ["crates/sqlbuild-orchestration/src/store"] @@ -1660,6 +1660,16 @@ allowed_paths = [ "crates/sqlbuild-python/src/bindings/_helpers/observations=Writes observations after a build.", ] +[targets.rust.rule_options.XSB079] +allowed_paths = [ + "crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs=A location's requested plan names the synced copy for its sqb plan child.", + "crates/sqlbuild-core/src/plan_display/constants.rs=Names the synced-copy environment variable.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs=Defines the synced-copy adapter.", + "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only sqb plan activates the synced copy and reports what it could not know.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=Preview branch: target MAX from the copy, watermark inputs read when the run starts.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=Preview branch: expression columns from the copy instead of DESCRIBE.", +] + [targets.rust.rule_options.XSB077] construction_required = [ "Action=A permission is live only when a route requires it; role tables and classifiers do not count.", diff --git a/scripts/fensu_policy/_helpers/unused_state.py b/scripts/fensu_policy/_helpers/unused_state.py index 20303010e8..0dc0cd4761 100644 --- a/scripts/fensu_policy/_helpers/unused_state.py +++ b/scripts/fensu_policy/_helpers/unused_state.py @@ -65,10 +65,12 @@ SQL_VIEW_PREFIX_PATTERN, SQL_WORD_PATTERN, SQL_WRITE_WORDS, + SYNCED_COPY_PATTERN, ) from scripts.fensu_policy.models import EnumVariant, StateDefinition, StateReads _OBSERVATION_STATE: re.Pattern[str] = re.compile(OBSERVATION_STATE_PATTERN) +_SYNCED_COPY: re.Pattern[str] = re.compile(SYNCED_COPY_PATTERN) _LEXEMES: re.Pattern[str] = re.compile(RUST_LITERAL_PATTERN, re.DOTALL) _CONTINUATION: re.Pattern[str] = re.compile(RUST_LINE_CONTINUATION) _TEST_MODULE: re.Pattern[str] = re.compile(RUST_TEST_MODULE_PATTERN) @@ -597,10 +599,20 @@ def _state_key(*, definition: StateDefinition) -> str: def observation_reference_lines(*, source: str) -> list[int]: """One-based lines whose code, not a line comment, names display-only observation state.""" + return _reference_lines(source=source, pattern=_OBSERVATION_STATE) + + +def synced_copy_reference_lines(*, source: str) -> list[int]: + """One-based lines whose code, not a line comment, names the plan preview's synced copy.""" + + return _reference_lines(source=source, pattern=_SYNCED_COPY) + + +def _reference_lines(*, source: str, pattern: re.Pattern[str]) -> list[int]: lines: list[int] = [] number: int line: str for number, line in enumerate(source.splitlines(), start=1): - if _OBSERVATION_STATE.search(line.partition(RUST_LINE_COMMENT)[0]): + if pattern.search(line.partition(RUST_LINE_COMMENT)[0]): lines.append(number) return lines diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 6bcecb4572..8628fb83f9 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -487,4 +487,13 @@ r"\b(?:observations_state|OBSERVATIONS_TABLE_NAME|WarehouseObservationRecord" r"|WarehouseHistoryKind::Observations|_sqlbuild_observations|sqb_warehouse_observations)\b" ) +SYNCED_COPY_ALLOWED_PATHS: RuleOption[tuple[str, ...]] = RuleOption.string_list( + name="allowed_paths", + default=(), + description=( + "`path=reason` entries for the Rust files or directories that may name the plan " + "preview's synced-copy adapter." + ), +) +SYNCED_COPY_PATTERN: str = r"\b(?:SyncedCopyAdapter|SyncedCopyConnection|PLAN_SYNCED_COPY_ENV)\b" RUST_LINE_COMMENT: str = "//" diff --git a/scripts/fensu_policy/rules/observation_boundary.py b/scripts/fensu_policy/rules/observation_boundary.py index 54361ac133..511b159007 100644 --- a/scripts/fensu_policy/rules/observation_boundary.py +++ b/scripts/fensu_policy/rules/observation_boundary.py @@ -1,16 +1,23 @@ -"""Rules keeping build observations out of planning and build decisions.""" +"""Rules keeping build observations and the plan preview's synced copy out of live decisions.""" from __future__ import annotations -from fensu import AnalyzerId, Family, Fault, Project, RuleContext, rule +from collections.abc import Callable + +from fensu import AnalyzerId, Family, Fault, Project, RuleContext, RuleOption, rule from scripts.fensu_policy._helpers.path_checks import allowed_prefix from scripts.fensu_policy._helpers.unused_state import ( is_rust_test_path, observation_reference_lines, reasoned_entries, + synced_copy_reference_lines, +) +from scripts.fensu_policy.constants import ( + OBSERVATION_ALLOWED_PATHS, + POLICY_CONFIG_FILE, + SYNCED_COPY_ALLOWED_PATHS, ) -from scripts.fensu_policy.constants import OBSERVATION_ALLOWED_PATHS, POLICY_CONFIG_FILE @rule( @@ -28,9 +35,49 @@ ) def observation_display_only(*, project: Project, ctx: RuleContext) -> list[Fault]: del project + return _confined( + ctx=ctx, + option=OBSERVATION_ALLOWED_PATHS, + references=observation_reference_lines, + subject="observation state", + ) + + +@rule( + code="XSB079", + family=Family.CUSTOM, + slug="synced-copy-preview-only", + message="only sqb plan previews may plan from the synced copy", + remediation=( + "Plan from the live adapter; only the files in allowed_paths, which define, activate or " + "branch on the plan preview's synced copy, may name it." + ), + cacheable=False, + analyzers=(AnalyzerId.RUST,), + options=(SYNCED_COPY_ALLOWED_PATHS,), +) +def synced_copy_preview_only(*, project: Project, ctx: RuleContext) -> list[Fault]: + del project + return _confined( + ctx=ctx, + option=SYNCED_COPY_ALLOWED_PATHS, + references=synced_copy_reference_lines, + subject="the synced copy", + ) + + +def _confined( + *, + ctx: RuleContext, + option: RuleOption[tuple[str, ...]], + references: Callable[..., list[int]], + subject: str, +) -> list[Fault]: + """Faults for each reference outside the allowed paths, and for stale allowed paths.""" + reasons: dict[str, str] malformed: list[str] - reasons, malformed = reasoned_entries(entries=ctx.option(OBSERVATION_ALLOWED_PATHS)) + reasons, malformed = reasoned_entries(entries=ctx.option(option)) prefixes: list[str] = list(reasons) used: set[str] = set() faults: list[Fault] = [] @@ -38,7 +85,7 @@ def observation_display_only(*, project: Project, ctx: RuleContext) -> list[Faul path: str = facts.file.path.value if facts.test or is_rust_test_path(path=path): continue - lines: list[int] = observation_reference_lines(source=facts.source) + lines: list[int] = references(source=facts.source) if not lines: continue prefix: str | None = allowed_prefix(path=path, prefixes=prefixes) @@ -51,7 +98,7 @@ def observation_display_only(*, project: Project, ctx: RuleContext) -> list[Faul problems: list[str] = [ f"allowed_paths entry {entry!r} needs `path=reason`" for entry in malformed ] + [ - f"allowed path {prefix} no longer names observation state; remove the entry" + f"allowed path {prefix} no longer names {subject}; remove the entry" for prefix in prefixes if prefix not in used ] diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d758ee9ae0..1ffa6ed00d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6366,6 +6366,7 @@ class CostArtifactError(ValueError): ... class CommandTerminated(KeyboardInterrupt): ... class SourceFreshnessInputError(ValueError): ... class SourceFreshnessObservationError(ValueError): ... +class WarehouseAccessRefusedError(RuntimeError): ... class CostTelemetryHealth: def mark_ledger_failure(self, *, run_id: str, error: BaseException) -> None: ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py index 9fe47fd8f2..bab2d7366a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_observations.py @@ -17,7 +17,6 @@ ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( execute_duckdb, - normalize_cli_output, prepare_inline_project, query_duckdb, run_sqb, @@ -247,6 +246,7 @@ def test_given_wrong_cursor_observation_when_planning_and_building_then_live_max ) assert honest.returncode == misled.returncode == 0, misled.stderr - assert normalize_cli_output(misled.stdout) == normalize_cli_output(honest.stdout) + assert misled.stdout.partition("Plan ready")[2] == honest.stdout.partition("Plan ready")[2] + assert "Plan ready" in honest.stdout assert build.returncode == 0, build.stderr assert counts == test_case.expected_row_counts diff --git a/tests/integration/scripts/fensu_policy/_test_types.py b/tests/integration/scripts/fensu_policy/_test_types.py index a4ae42b48c..33d3fb804d 100644 --- a/tests/integration/scripts/fensu_policy/_test_types.py +++ b/tests/integration/scripts/fensu_policy/_test_types.py @@ -13,6 +13,7 @@ class NativeBoundaryTestCase: @dataclass(frozen=True) class ObservationBoundaryTestCase: description: str + rule: str allowed: tuple[str, ...] planner: str expected_returncode: int diff --git a/tests/integration/scripts/fensu_policy/helpers.py b/tests/integration/scripts/fensu_policy/helpers.py index 45e8140bac..e0592a79f3 100644 --- a/tests/integration/scripts/fensu_policy/helpers.py +++ b/tests/integration/scripts/fensu_policy/helpers.py @@ -6,8 +6,10 @@ from pathlib import Path -def write_observation_project(*, root: Path, allowed: tuple[str, ...], planner: str) -> None: - """A crate whose observations module may name observation state, plus the given planner.""" +def write_observation_project( + *, root: Path, rule: str, allowed: tuple[str, ...], planner: str +) -> None: + """A crate whose observations module names both preview-only states, plus the given planner.""" repository: Path = Path(__file__).resolve().parents[4] shutil.copytree( @@ -27,14 +29,15 @@ def write_observation_project(*, root: Path, allowed: tuple[str, ...], planner: (crate / "src/lib.rs").write_text("mod observations;\nmod planner;\n") (crate / "src/observations/mod.rs").write_text( "pub fn table() -> &'static str { OBSERVATIONS_TABLE_NAME }\n" + "pub struct SyncedCopyAdapter;\n" ) (crate / "src/planner.rs").write_text(planner) entries: str = ", ".join(f'"{path}=Writes observations."' for path in allowed) (root / "fensu.toml").write_text( '[targets.rust]\nanalyzer = "rust"\nroots = ["crates"]\n' 'rule_modules = ["scripts.fensu_policy.rules.observation_boundary"]\n' - 'select = ["XSB078"]\n\n' - f"[targets.rust.rule_options.XSB078]\nallowed_paths = [{entries}]\n" + f'select = ["{rule}"]\n\n' + f"[targets.rust.rule_options.{rule}]\nallowed_paths = [{entries}]\n" ) diff --git a/tests/integration/scripts/fensu_policy/test_observation_boundary.py b/tests/integration/scripts/fensu_policy/test_observation_boundary.py index 98b6be983d..70bdb1ad5d 100644 --- a/tests/integration/scripts/fensu_policy/test_observation_boundary.py +++ b/tests/integration/scripts/fensu_policy/test_observation_boundary.py @@ -1,4 +1,4 @@ -"""Exercise the observation display-only policy through Fensu's real Rust analyzer.""" +"""Exercise the observation and synced-copy boundaries through Fensu's real Rust analyzer.""" import subprocess from pathlib import Path @@ -14,6 +14,7 @@ [ ObservationBoundaryTestCase( description="planner mentioning observations only in a comment", + rule="XSB078", allowed=("crates/sqlbuild-python/src/observations",), planner="// OBSERVATIONS_TABLE_NAME is display-only\npub fn plan() {}\n", expected_returncode=0, @@ -21,6 +22,7 @@ ), ObservationBoundaryTestCase( description="planner matching the observations history kind", + rule="XSB078", allowed=("crates/sqlbuild-python/src/observations",), planner=( "pub fn plan() {}\n" @@ -31,6 +33,7 @@ ), ObservationBoundaryTestCase( description="planner reading the observation table by its SQL name", + rule="XSB078", allowed=("crates/sqlbuild-python/src/observations",), planner='pub const READ: &str = "SELECT value FROM _sqlbuild_observations";\n', expected_returncode=1, @@ -38,6 +41,7 @@ ), ObservationBoundaryTestCase( description="allowed path that no longer names observation state", + rule="XSB078", allowed=( "crates/sqlbuild-python/src/observations", "crates/sqlbuild-python/src/planner.rs", @@ -46,17 +50,38 @@ expected_returncode=1, expected_faults=("allowed path crates/sqlbuild-python/src/planner.rs no longer names",), ), + ObservationBoundaryTestCase( + description="live planner naming the synced copy only in a comment", + rule="XSB079", + allowed=("crates/sqlbuild-python/src/observations",), + planner="// SyncedCopyAdapter serves plan previews only\npub fn plan() {}\n", + expected_returncode=0, + expected_faults=(), + ), + ObservationBoundaryTestCase( + description="live planner branching on the synced copy", + rule="XSB079", + allowed=("crates/sqlbuild-python/src/observations",), + planner=( + "pub fn plan() {}\n" + "pub fn preview(adapter: &Any) -> bool { adapter.is::() }\n" + ), + expected_returncode=1, + expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:2:0",), + ), ], ids=lambda case: case.description, ) -def test_given_rust_naming_observation_state_when_checking_then_only_unallowed_code_faults( +def test_given_rust_naming_preview_only_state_when_checking_then_only_unallowed_code_faults( tmp_path: Path, test_case: ObservationBoundaryTestCase ) -> None: - write_observation_project(root=tmp_path, allowed=test_case.allowed, planner=test_case.planner) + write_observation_project( + root=tmp_path, rule=test_case.rule, allowed=test_case.allowed, planner=test_case.planner + ) result: subprocess.CompletedProcess[str] = check_rust(root=tmp_path) assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr - assert result.stdout.count("XSB078") == len(test_case.expected_faults), result.stdout + assert result.stdout.count(test_case.rule) == len(test_case.expected_faults), result.stdout assert all(fault in result.stdout for fault in test_case.expected_faults), result.stdout assert "src/observations/mod.rs" not in result.stdout diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index e542ca01a9..75d15866d9 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -374,3 +374,13 @@ class AdhocRunTestCase: job_settings: str = "" expected_scheduled_status: str = "success" expected_worker_log: str = "" + + +@dataclass(frozen=True) +class PlanPreviewTestCase: + description: str + edited_files: dict[str, str] + injected_sql: tuple[str, ...] + window_models: tuple[str, ...] + unplanned_cursor_models: tuple[str, ...] + added_unknowns: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py index 65f57034c2..417c30573d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -87,3 +87,28 @@ def wait_for_statistics( records = [json.loads(row[0]) for row in rows] time.sleep(0.05) return records + + +def request_plan(control: Path, selection: list[str]) -> str: + """Queue the plan work a person's Plan creates, frozen to exact node keys.""" + request_id: str = str(uuid4()) + now: datetime = datetime.now(UTC) + with closing(sqlite3.connect(control)) as database: + _ = database.execute( + "INSERT INTO sqb_location_work (request_id,location_id,target,snapshot_id,generation," + "request_json,requested_by,requested_at,expires_at) SELECT ?,c.location_id,c.target," + "c.snapshot_id,s.generation,json_object('request_id',?,'location_id',c.location_id," + "'target',c.target,'reason','Plan','work',json_object('kind','plan','command'," + "json('[\"build\"]'),'selection',json(?),'settings',json_object()))," + "'test-admin',?,? FROM sqb_current_snapshots c JOIN sqb_snapshots s " + "ON s.snapshot_id=c.snapshot_id", + ( + request_id, + request_id, + json.dumps(selection), + now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + (now + timedelta(minutes=5)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + ), + ) + database.commit() + return request_id diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py new file mode 100644 index 0000000000..38bc6dbac4 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -0,0 +1,246 @@ +"""A location plans from the server's synced copy and never opens the warehouse to do it.""" + +import json +import os +import subprocess +import sys +import time +from datetime import UTC, datetime +from pathlib import Path +from textwrap import dedent +from typing import Any + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + PlanPreviewTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + request_plan, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + snapshot_api, + wait_for_generation, + wait_for_location_work, + wait_for_warehouse_sync_health, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + +_EXPRESSION: str = "SELECT * FROM (VALUES (1, 100)) AS refunds(refund_id, amount_cents)" +_SOURCES: str = dedent( + """ + sources: + - name: raw_refunds + description: Refunds described from an inline query. + expression: | + {expression} + type_enforcement: true + columns: + - name: refund_id + type: INTEGER + - name: amount_cents + type: INTEGER + """ +).strip() +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n[connections.local]\ndatabase = "warehouse.db"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": "", + "seeds/raw_orders.csv": "id,ordered_at\n1,2026-01-01 00:00:00\n2,2026-01-01 01:00:00\n", + "seeds/raw_orders.yml": dedent( + """ + seeds: + - name: raw_orders + description: Raw orders. + columns: + - name: id + type: INTEGER + - name: ordered_at + type: TIMESTAMP + """ + ).strip() + + "\n", + "models/orders.sql": dedent( + """ + MODEL (description "Orders by timestamp cursor.", + materialized incremental, + incremental_strategy append, + cursor ordered_at, + cursor_type timestamp, + cursor_grain second, + ); + + SELECT id, ordered_at FROM __seed("raw_orders") + """ + ).strip() + + "\n", + "models/order_count.sql": dedent( + """ + MODEL (description "Order count without a cursor.", materialized table); + + SELECT COUNT(*) AS order_count FROM __seed("raw_orders") + """ + ).strip() + + "\n", + "models/refunds.sql": dedent( + """ + MODEL (description "Refunds.", materialized table); + + SELECT refund_id, amount_cents FROM __source("raw_refunds") + """ + ).strip() + + "\n", + "sources/raw.yml": _SOURCES.format(expression=_EXPRESSION) + "\n", +} +_SELECTION: list[str] = ["model:order_count", "model:orders", "model:refunds"] +_UNKNOWN: list[str] = [ + "source freshness", + "retention", + "column rename support", + "sources outside mirrored schemas", +] + + +def _sqb(project: Path, *arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(Path(sys.executable).with_name("sqb")), "--project-dir", str(project), *arguments], + capture_output=True, + text=True, + check=False, + ) + + +def _live_plan(project: Path) -> subprocess.CompletedProcess[str]: + selected: list[str] = [part for key in _SELECTION for part in ("--select", key.split(":")[1])] + return _sqb(project, "plan", "--target", "development", "--json", "--no-color", *selected) + + +def _locked(database: Path) -> duckdb.DuckDBPyConnection: + """Hold the warehouse's write lock, so any process that opens it fails.""" + deadline: float = time.monotonic() + 15 + while True: + try: + return duckdb.connect(database) + except duckdb.IOException: + assert time.monotonic() < deadline + time.sleep(0.05) + + +@pytest.mark.parametrize( + "test_case", + [ + PlanPreviewTestCase( + description="matching observations plan like the live warehouse", + edited_files={}, + injected_sql=(), + window_models=("orders",), + unplanned_cursor_models=(), + added_unknowns=(), + ), + PlanPreviewTestCase( + description="a cursor_max from no successful build is not a cursor position", + edited_files={}, + injected_sql=( + "INSERT INTO main._sqlbuild_observations VALUES ('injected-orders', 'cursor_max', " + "'orders', 'timestamp', '2099-01-01 00:00:00', 'injected-run', " + "'2099-01-01 00:00:00')", + ), + window_models=(), + unplanned_cursor_models=("orders",), + added_unknowns=("cursor position of orders (no successful build recorded it)",), + ), + PlanPreviewTestCase( + description="an edited expression plans its declared columns and says so", + edited_files={ + "sources/raw.yml": _SOURCES.format(expression=f"{_EXPRESSION} WHERE true") + "\n" + }, + injected_sql=(), + window_models=("orders",), + unplanned_cursor_models=(), + added_unknowns=( + "query columns of expression source raw_refunds (planned as declared)", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_synced_location_when_a_plan_is_requested_then_it_plans_from_the_copy_without_the_warehouse( + test_case: PlanPreviewTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files(project_dir=project, files=_PROJECT_FILES) + built: subprocess.CompletedProcess[str] = _sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + with duckdb.connect(project / "warehouse.db") as warehouse: + for statement in test_case.injected_sql: + _ = warehouse.execute(statement) + write_files(project_dir=project, files=test_case.edited_files) + live_run: subprocess.CompletedProcess[str] = _live_plan(project) + assert live_run.returncode == 0, live_run.stderr + live: dict[str, Any] = json.loads(live_run.stdout) + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + started: datetime = datetime.now(UTC) + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + _ = wait_for_warehouse_sync_health( + api, + {"failure": None, "tables": ["catalog", "consumption", "fingerprints", "observations"]}, + location, + log, + after=started, + ) + lock: duckdb.DuckDBPyConnection = _locked(project / "warehouse.db") + try: + refused: subprocess.CompletedProcess[str] = _live_plan(project) + request_id: str = request_plan(control, _SELECTION) + result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) + finally: + lock.close() + + assert refused.returncode != 0 + outcome: dict[str, Any] = result["result_json"]["outcome"] + assert outcome["kind"] == "plan", outcome + preview: dict[str, Any] = outcome["document"] + assert "synced_copy" in preview, log.read_text() + synced: dict[str, Any] = preview.pop("synced_copy") + assert synced["synced_at"] is not None + assert synced["unknown_until_run"] == [*_UNKNOWN, *test_case.added_unknowns] + live_models: dict[str, dict[str, Any]] = {model["name"]: model for model in live.pop("models")} + preview_models: dict[str, dict[str, Any]] = { + model["name"]: model for model in preview.pop("models") + } + assert synced["cursor_window_starts"] == { + name: live_models[name]["cursor_bounds"]["start"] for name in test_case.window_models + } + for name in (*test_case.window_models, *test_case.unplanned_cursor_models): + for model in (live_models[name], preview_models.pop(name)): + assert model["action"] == "incremental_append" + assert model["reason"] == "normal_incremental" + _ = live_models.pop(name) + assert preview_models == live_models + assert preview == live From 4d1198a03deed35718fa591929751deec6f1daf8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:09:19 +0000 Subject: [PATCH 09/17] refactor(state): share the fixed-column state table DDL between consumption and observations --- .../_helpers/native_host/consumption_state.rs | 36 +++++++++++------- .../native_host/observations_state.rs | 37 +++---------------- .../_helpers/native_host/synced_copy.rs | 8 +--- 3 files changed, 30 insertions(+), 51 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index d385970b39..3677e129f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -106,28 +106,38 @@ fn build_consumption_create_table_sql( render_qualified_name: &Bound<'_, PyAny>, render_framework_type: &Bound<'_, PyAny>, ) -> PyResult { - let py: Python<'_> = database.py(); - let Some(table) = state_table_name( - render_qualified_name, + fixed_state_table_sql( (database, schema), - CONSUMPTION_TABLE_NAME, - )? + (render_qualified_name, render_framework_type), + ( + CONSUMPTION_TABLE_NAME, + &CONSUMPTION_COLUMN_TYPES, + &NULLABLE_COLUMNS, + ), + ) +} + +/// Create-if-missing DDL for a state table with fixed columns, typed by the adapter. +pub(crate) fn fixed_state_table_sql( + (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (render_qualified_name, render_framework_type): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (table_name, column_types, nullable): (&str, &[(&str, &str)], &[&str]), +) -> PyResult { + let py: Python<'_> = database.py(); + let Some(table) = state_table_name(render_qualified_name, (database, schema), table_name)? else { return Ok(String::new()); }; - let columns: Bound<'_, PyAny> = PyTuple::new( - py, - CONSUMPTION_COLUMN_TYPES.iter().map(|(column, _)| *column), - )? - .into_any(); + let columns: Bound<'_, PyAny> = + PyTuple::new(py, column_types.iter().map(|(column, _)| *column))?.into_any(); let types: Bound<'_, PyDict> = PyDict::new(py); - for (column, declared) in CONSUMPTION_COLUMN_TYPES { + for (column, declared) in column_types { types.set_item(column, declared)?; } - let required: Vec<&str> = CONSUMPTION_COLUMN_TYPES + let required: Vec<&str> = column_types .iter() .map(|(column, _)| *column) - .filter(|column| !NULLABLE_COLUMNS.contains(column)) + .filter(|column| !nullable.contains(column)) .collect(); state_table_create_sql( &table, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs index d2f68fa321..b049b9f4c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs @@ -1,7 +1,6 @@ //! DDL and pruning of the display-only observations table through the adapter. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::OBSERVATIONS_TABLE_NAME; use sqlbuild_state::observations::constants::{ @@ -13,10 +12,9 @@ use uuid::Uuid; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::native_host::consumption_state::{ - Location, ROWS_PER_STATEMENT, execute, literal, state_table_name, + Location, ROWS_PER_STATEMENT, execute, fixed_state_table_sql, literal, state_table_name, }; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; /// Create-if-missing DDL for the observations table, with adapter-rendered column types. #[pyfunction] @@ -27,37 +25,14 @@ fn build_observations_create_table_sql( render_qualified_name: &Bound<'_, PyAny>, render_framework_type: &Bound<'_, PyAny>, ) -> PyResult { - let py: Python<'_> = database.py(); - let Some(table) = state_table_name( - render_qualified_name, + fixed_state_table_sql( (database, schema), - OBSERVATIONS_TABLE_NAME, - )? - else { - return Ok(String::new()); - }; - let columns: Bound<'_, PyAny> = PyTuple::new( - py, - OBSERVATION_COLUMN_TYPES.iter().map(|(column, _)| *column), - )? - .into_any(); - let types: Bound<'_, PyDict> = PyDict::new(py); - for (column, declared) in OBSERVATION_COLUMN_TYPES { - types.set_item(column, declared)?; - } - let required: Vec<&str> = OBSERVATION_COLUMN_TYPES - .iter() - .map(|(column, _)| *column) - .filter(|column| !NULLABLE_OBSERVATION_COLUMNS.contains(column)) - .collect(); - state_table_create_sql( - &table, + (render_qualified_name, render_framework_type), ( - &columns, - &types.into_any(), - &PyFrozenSet::new(py, &required)?.into_any(), + OBSERVATIONS_TABLE_NAME, + &OBSERVATION_COLUMN_TYPES, + &NULLABLE_OBSERVATION_COLUMNS, ), - render_framework_type, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs index b089d388af..1d2f1ebd81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs @@ -22,6 +22,7 @@ use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KI use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, SqliteCursor}; +use crate::bindings::_helpers::planner_phases::objects::optional_text; use crate::bindings::_helpers::records::fields::field as native_field; create_exception!( @@ -359,13 +360,6 @@ impl SyncedCopyAdapter { } } -fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { - match value { - Some(value) => PyString::new(py, value).into_any(), - None => py.None().into_bound(py), - } -} - #[pymethods] impl SyncedCopyAdapter { #[pyo3(signature = (connection_config = None))] From 4690a1b4992627f1036d2eed2c0c160b29d2d3a6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:14:27 +0000 Subject: [PATCH 10/17] refactor(plan): render the copy's optional database without a higher-layer helper --- .../src/bindings/_helpers/native_host/synced_copy.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs index 1d2f1ebd81..601c4b91f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Mutex, PoisonError}; use chrono::SecondsFormat; +use pyo3::IntoPyObject; use pyo3::create_exception; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; @@ -22,7 +23,6 @@ use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KI use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, SqliteCursor}; -use crate::bindings::_helpers::planner_phases::objects::optional_text; use crate::bindings::_helpers::records::fields::field as native_field; create_exception!( @@ -328,7 +328,11 @@ impl SyncedCopyAdapter { &[ ( "database", - optional_text(py, record.state_database.as_deref()), + record + .state_database + .as_deref() + .into_pyobject(py)? + .into_any(), ), ("schema", PyString::new(py, &record.state_schema).into_any()), ("name", PyString::new(py, &record.name).into_any()), From c090def7c700370efd8cd3d6a901449e8307ae83 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:22:15 +0000 Subject: [PATCH 11/17] feat(ui): show what a plan preview planned from and each incremental window start A location plans a build from the server's synced copy of warehouse metadata. The plan page now says when that copy synced, lists what stays unknown until the run, and shows each incremental model's window start running to the latest data when the run starts. --- ui/src/lib/api/http-runs.ts | 3 +- ui/src/lib/api/mock/operate-store.ts | 30 +++++++++++------ ui/src/lib/api/types/operate.ts | 12 +++++++ ui/src/lib/run/plan-report.svelte | 48 ++++++++++++++++++++++++++++ ui/src/lib/run/plan-text.test.ts | 13 ++++++++ ui/src/lib/run/plan-text.ts | 15 +++++---- ui/src/routes/plan/+page.svelte | 5 +-- 7 files changed, 108 insertions(+), 18 deletions(-) diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index 5930b21bc8..78965a49c0 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -34,7 +34,8 @@ export function planResultOf(outcome: Wire['LocationWorkOutcome']): PlanResult { migrations: list(document.migrations), column_migrations: list(document.column_migrations), table_type_conversions: list(document.table_type_conversions), - retention: list(document.retention) + retention: list(document.retention), + synced_copy: document.synced_copy ?? null }, error: null }; diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index 57d5ff7ead..c03592d390 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -538,16 +538,28 @@ export class MockOperate { if (check.error) return this.#ctx.reply({ plan: null, error: check.error }); if (check.verb !== 'build') return this.#ctx.reply({ plan: null, error: `sqb plan previews builds; sqb ${check.verb} cannot be planned` }); const reason = command.includes('--full-refresh') ? 'full_refresh' : 'no_change'; - const models = check.selection.filter((key) => key.startsWith('model:')).map((key) => ({ - name: key.slice('model:'.length), - materialization_type: 'table', - action: 'create_table', - reason, - backfill: { action: reason === 'full_refresh' ? 'full' : 'forward', duration: null }, - qualified_name: `main.${key.slice('model:'.length)}` - })); + const modelKinds = new Set(['view', 'table', 'incremental']); + const models = check.selection.filter((key) => modelKinds.has(snapshotNode(key)?.kind ?? '')).map((key) => { + const incremental = snapshotNode(key)?.kind === 'incremental'; + return { + name: key, + materialization_type: incremental ? 'incremental' : 'table', + action: incremental ? 'incremental_append' : 'create_table', + reason: incremental && reason === 'no_change' ? 'normal_incremental' : reason, + backfill: { action: reason === 'full_refresh' ? 'full' : 'forward', duration: null }, + qualified_name: `main.${key}`, + ...(incremental ? { incremental_strategy: 'append', cursor_column: 'updated_at', cursor_type: 'timestamp' } : {}) + }; + }); + // Locations plan from the server's synced copy: incremental windows start at the synced cursor. + const windowStart = new Date(Date.now() - 3_600_000).toISOString().slice(0, 19).replace('T', ' '); + const synced_copy = { + synced_at: new Date(Date.now() - 300_000).toISOString(), + cursor_window_starts: Object.fromEntries(models.filter((m) => m.materialization_type === 'incremental' && reason !== 'full_refresh').map((m) => [m.name, windowStart])), + unknown_until_run: ['source freshness', 'retention', 'column rename support', 'sources outside mirrored schemas'] + }; return this.#ctx.reply({ - plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [] }, + plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [], synced_copy }, error: null }); } diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 72bc25ccd7..1b41832ec0 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -529,6 +529,18 @@ export type PlanDocument = { column_migrations: { model: string; origin_column: string; destination_column: string; decision: string }[]; table_type_conversions: { model: string; actual_type?: string; desired_type?: string }[]; retention: { model: string; action?: string }[]; + /** What a location's preview planned from instead of the warehouse; `null` for a live plan. */ + synced_copy: PlanSyncedCopy | null; +}; + +/** The server's synced copy of warehouse metadata a location's plan preview planned from. */ +export type PlanSyncedCopy = { + /** When every table the preview reads last synced; `null` until all of them have. */ + synced_at: string | null; + /** Each incremental model's window start; its end is the latest data when the run starts. */ + cursor_window_starts: Record; + /** What the preview could not know, which the run plans live. */ + unknown_until_run: string[]; }; /** The location's plan of a build command, or why it could not plan it. */ diff --git a/ui/src/lib/run/plan-report.svelte b/ui/src/lib/run/plan-report.svelte index 044b2a33fe..23d93fa2bf 100644 --- a/ui/src/lib/run/plan-report.svelte +++ b/ui/src/lib/run/plan-report.svelte @@ -3,6 +3,8 @@ // then the routine models. import { to } from '$lib/paths'; import type { PlanDocument } from '$lib/api'; + import { formatAgo } from '$lib/format/time'; + import { app } from '$lib/state/app.svelte'; import { planSections, type PlanEntry } from './plan-text'; let { plan }: { plan: PlanDocument } = $props(); @@ -17,6 +19,21 @@
+ {#if plan.synced_copy} + {@const copy = plan.synced_copy} +
+

+ Planned from warehouse metadata {#if copy.synced_at}synced {:else}that has not fully synced yet{/if}. + The run plans again against the warehouse. +

+ {#if copy.unknown_until_run.length} +
+ Unknown until the run +
    {#each copy.unknown_until_run as item (item)}
  • {item}
  • {/each}
+
+ {/if} +
+ {/if} {#each sections as s (s.title)} {@const shown = expanded[s.title] ? s.entries : s.entries.slice(0, limit)}
@@ -172,6 +189,37 @@ .warning.err { border-left-color: var(--sb-error); } + .preview { + display: flex; + flex-direction: column; + gap: 6px; + padding: 8px 10px; + border: 1px solid var(--border); + border-left: 2px solid var(--primary); + border-radius: 2px; + background: var(--sb-inset); + color: var(--muted-foreground); + } + .preview p { + margin: 0; + } + .preview time { + color: var(--foreground); + } + .unknown span { + color: var(--sb-text-faint); + font-size: 11.5px; + } + .unknown ul { + display: flex; + flex-wrap: wrap; + gap: 4px 14px; + margin: 2px 0 0; + padding: 0; + list-style: none; + font-family: var(--font-mono); + font-size: 11px; + } code { color: var(--sb-text-faint); font-family: var(--font-mono); diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index 1ec8b747ba..dd73c21e65 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -22,6 +22,7 @@ const plan = (models: PlanModel[], over: Partial = {}): PlanDocume column_migrations: [], table_type_conversions: [], retention: [], + synced_copy: null, ...over }); @@ -57,6 +58,18 @@ describe('plan text, as sqb plan prints it', () => { ]); }); + it('shows a preview window from the synced cursor to the run, and keeps a range the plan resolved', () => { + const incremental = { materialization_type: 'incremental', incremental_strategy: 'append', cursor_type: 'timestamp', cursor_column: 'placed_at', action: 'incremental_append', reason: 'normal_incremental' } as const; + const resolved = { start: '2026-01-02 00:00:00', end: '2026-01-03 00:00:00' }; + const preview = plan([model('lines', incremental), model('events', { ...incremental, cursor_bounds: resolved })], { + synced_copy: { synced_at: '2026-01-03T00:05:00Z', cursor_window_starts: { lines: '2026-01-01 01:00:00', events: '2025-12-31 00:00:00' }, unknown_until_run: ['retention'] } + }); + expect(planSections(preview)[0].entries.map((e) => e.detail)).toEqual([ + ['cursor: placed_at (timestamp)', 'range: 2026-01-01 01:00:00 → latest data when the run starts'], + ['cursor: placed_at (timestamp)', 'range: 2026-01-02 00:00:00 → 2026-01-03 00:00:00'] + ]); + }); + it('collapses a plan that is all full refresh into counts by materialization', () => { const full = plan([model('a', { reason: 'full_refresh' }), model('b', { reason: 'full_refresh' }), model('v', { reason: 'full_refresh', materialization_type: 'view' })]); expect(planSections(full)).toEqual([{ title: 'Full refresh', changed: true, entries: [{ name: 'table', value: '2 models', detail: [] }, { name: 'view', value: '1 model', detail: [] }] }]); diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index f5c87c8cfd..5ebd9bd18c 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -45,7 +45,8 @@ export function actionText(m: PlanModel): string { return 'continue forward'; } -function cursorLines(m: PlanModel, showRange: boolean): string[] { +/** The cursor lines; a preview's `windowStart` stands in for a range it could not bound. */ +function cursorLines(m: PlanModel, showRange: boolean, windowStart?: string): string[] { const column = m.cursor_column; if (!column) return []; const lines = [`cursor: ${column}${m.cursor_type ? ` (${m.cursor_type})` : ''}`]; @@ -53,6 +54,7 @@ function cursorLines(m: PlanModel, showRange: boolean): string[] { if (requested && (requested.start || requested.end)) lines.push(`requested: ${requested.start ?? 'earliest available'} → ${requested.end ?? 'latest available'}`); const range = m.cursor?.resolved_bounds ?? m.cursor_bounds; if (showRange && range && (range.start || range.end)) lines.push(`range: ${range.start ?? '…'} → ${range.end ?? '…'}`); + else if (showRange && windowStart) lines.push(`range: ${windowStart} → latest data when the run starts`); return lines; } @@ -68,13 +70,13 @@ const SCHEMA_SIGNS: Record = { column_type_changed: { sign: '~', label: 'type changed' } }; -function changedEntry(m: PlanModel, reason: string): PlanEntry { - if (reason === 'first_run') return { name: m.name, value: materializationLabel(m), detail: cursorLines(m, true) }; +function changedEntry(m: PlanModel, reason: string, windowStart?: string): PlanEntry { + if (reason === 'first_run') return { name: m.name, value: materializationLabel(m), detail: cursorLines(m, true, windowStart) }; return { name: m.name, value: actionText(m), detail: [ - ...cursorLines(m, reason !== 'full_refresh'), + ...cursorLines(m, reason !== 'full_refresh', windowStart), ...policyLine(m), ...(m.changed_functions?.length ? [`cause: function ${m.changed_functions.join(', ')} changed`] : []) ], @@ -119,8 +121,9 @@ export function planSections(plan: PlanDocument): PlanSection[] { for (const m of active) counts.set(m.materialization_type, (counts.get(m.materialization_type) ?? 0) + 1); add('Full refresh', [...counts].map(([kind, n]) => ({ name: kind, value: `${n} ${n === 1 ? 'model' : 'models'}`, detail: [] })), true); } else { - for (const [reason, title] of REASON_GROUPS) add(title, active.filter((m) => m.reason === reason).map((m) => changedEntry(m, reason)), true); - add('Models', active.filter((m) => ROUTINE.has(m.reason)).map((m) => ({ name: m.name, value: materializationLabel(m), detail: cursorLines(m, true) }))); + const starts = plan.synced_copy?.cursor_window_starts ?? {}; + for (const [reason, title] of REASON_GROUPS) add(title, active.filter((m) => m.reason === reason).map((m) => changedEntry(m, reason, starts[m.name])), true); + add('Models', active.filter((m) => ROUTINE.has(m.reason)).map((m) => ({ name: m.name, value: materializationLabel(m), detail: cursorLines(m, true, starts[m.name]) }))); } add('Functions', plan.functions.filter((f) => !f.reason || f.reason === 'no_change').map(functionEntry)); const seedLabel: Record = { first_run: 'first_run', config_changed: 'seed_changed', no_change: 'current' }; diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index 2d68318502..92bf838f3b 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -1,7 +1,8 @@