From 8ab5bdf6e205848e89adeeb54662353331eee4f9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:51:38 +0000 Subject: [PATCH 1/9] feat(janitor): prune audit result history per audit binding --- .../src/bindings/_helpers/cli/registration.rs | 1 + .../_helpers/janitor/janitor_facts.rs | 10 ++- .../native_host/audit_results_state.rs | 30 +++++++++ .../src/bindings/_helpers/native_host/mod.rs | 1 + .../src/audit_results/main/mod.rs | 1 + .../main/prune_audit_results_sql.rs | 11 +++ .../sqlbuild-state/src/audit_results/mod.rs | 1 + crates/sqlbuild-state/src/lib.rs | 1 + fensu.toml | 3 +- src/sqlbuild/_native.pyi | 7 ++ .../adapter/contract/classes/audit_results.py | 24 +++++++ .../adapter/contract/classes/base_adapter.py | 12 ++++ .../contract/classes/strict_adapter.py | 11 +++ .../bigquery/classes/bigquery_adapter.py | 8 ++- .../clickhouse/classes/clickhouse_adapter.py | 8 ++- .../databricks/classes/databricks_adapter.py | 8 ++- .../adapters/duckdb/classes/duckdb_adapter.py | 5 +- .../motherduck/classes/motherduck_adapter.py | 5 +- .../postgres/classes/postgres_adapter.py | 8 ++- .../snowflake/classes/snowflake_adapter.py | 8 ++- .../adapters/sqlite/classes/sqlite_adapter.py | 8 ++- .../sqlserver/classes/sqlserver_adapter.py | 8 ++- .../cli/commands/main/janitor/_test_types.py | 1 + .../cli/commands/main/janitor/helpers.py | 67 +++++++++++++++++++ .../cli/commands/main/janitor/test_janitor.py | 19 +++++- 25 files changed, 252 insertions(+), 14 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs create mode 100644 crates/sqlbuild-state/src/audit_results/main/mod.rs create mode 100644 crates/sqlbuild-state/src/audit_results/main/prune_audit_results_sql.rs create mode 100644 crates/sqlbuild-state/src/audit_results/mod.rs create mode 100644 src/sqlbuild/adapter/contract/classes/audit_results.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 9367712a19..6f4297132c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -30,6 +30,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { freshness::registration::register(module)?; crate::bindings::_helpers::native_host::consumption_state::register(module)?; crate::bindings::_helpers::native_host::observations_state::register(module)?; + crate::bindings::_helpers::native_host::audit_results_state::register(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; execution_history::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 9378dea579..36c6485896 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -26,7 +26,8 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::{ - CONSUMPTION_TABLE_NAME, OBSERVATIONS_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, + AUDIT_RESULTS_TABLE_NAME, CONSUMPTION_TABLE_NAME, OBSERVATIONS_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, }; /// The adapter and its open connection. @@ -477,7 +478,7 @@ pub(crate) fn gather_warehouse_facts<'py>( }) } -/// Fingerprint, source freshness, consumption and observation history prune candidates for target schemas. +/// Fingerprint, source freshness, consumption, observation and audit result history prune candidates for target schemas. pub(crate) fn direct_state_prune_candidates<'py>( warehouse: &Warehouse<'py>, target_schemas: &[SchemaKey], @@ -498,6 +499,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( )); locations.push((database.clone(), schema.clone(), CONSUMPTION_TABLE_NAME)); locations.push((database.clone(), schema.clone(), OBSERVATIONS_TABLE_NAME)); + locations.push((database.clone(), schema.clone(), AUDIT_RESULTS_TABLE_NAME)); } let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut candidates: Vec> = Vec::new(); @@ -519,6 +521,10 @@ pub(crate) fn direct_state_prune_candidates<'py>( OBSERVATIONS_TABLE_NAME, "render_prune_observations_history_sql", ), + ( + AUDIT_RESULTS_TABLE_NAME, + "render_prune_audit_results_history_sql", + ), ] { if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs new file mode 100644 index 0000000000..dbb1234c40 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs @@ -0,0 +1,30 @@ +//! Pruning of the audit results state table through the adapter. + +use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::constants::AUDIT_RESULTS_TABLE_NAME; +use sqlbuild_state::audit_results::main::prune_audit_results_sql::prune_audit_results_sql; + +use crate::bindings::_helpers::native_host::consumption_state::state_table_name; + +/// SQL deleting audit results beyond the newest `retain_versions` per audit binding. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] +fn build_audit_results_prune_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + retain_versions: i64, +) -> PyResult { + Ok(state_table_name( + render_qualified_name, + (database, schema), + AUDIT_RESULTS_TABLE_NAME, + )? + .map(|table| prune_audit_results_sql(&table, retain_versions)) + .unwrap_or_default()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_audit_results_prune_sql, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 59eca3d642..1794e8b508 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,5 +1,6 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. +pub(crate) mod audit_results_state; pub(crate) mod connections; pub(crate) mod constants; pub(crate) mod consumption_state; diff --git a/crates/sqlbuild-state/src/audit_results/main/mod.rs b/crates/sqlbuild-state/src/audit_results/main/mod.rs new file mode 100644 index 0000000000..83d8c33dff --- /dev/null +++ b/crates/sqlbuild-state/src/audit_results/main/mod.rs @@ -0,0 +1 @@ +pub mod prune_audit_results_sql; diff --git a/crates/sqlbuild-state/src/audit_results/main/prune_audit_results_sql.rs b/crates/sqlbuild-state/src/audit_results/main/prune_audit_results_sql.rs new file mode 100644 index 0000000000..d4c871b63f --- /dev/null +++ b/crates/sqlbuild-state/src/audit_results/main/prune_audit_results_sql.rs @@ -0,0 +1,11 @@ +//! Prune old audit results from one audit results table. + +/// Delete every audit result ranked beyond the newest `retain` per audit binding. +pub fn prune_audit_results_sql(qualified_name: &str, retain: i64) -> String { + format!( + "DELETE FROM {qualified_name} WHERE result_id IN (SELECT result_id FROM \ +(SELECT result_id, ROW_NUMBER() OVER (PARTITION BY binding_key ORDER BY \ +occurred_at DESC, result_id DESC) AS result_rank FROM {qualified_name}) \ +AS ranked WHERE result_rank > {retain})" + ) +} diff --git a/crates/sqlbuild-state/src/audit_results/mod.rs b/crates/sqlbuild-state/src/audit_results/mod.rs new file mode 100644 index 0000000000..2a043412b4 --- /dev/null +++ b/crates/sqlbuild-state/src/audit_results/mod.rs @@ -0,0 +1 @@ +pub mod main; diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index 3fc9db2953..72ebf3aca6 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] +pub mod audit_results; pub mod consumption; pub mod events; pub mod execution_history; diff --git a/fensu.toml b/fensu.toml index 415e477619..66dcc9a651 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1852,7 +1852,7 @@ paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/cla reason = "tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py requires every first-class adapter to define each StrictAdapter method itself instead of inheriting it from BaseAdapter." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/native_host/audit_results_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/observations_state.rs"] reason = "Each state table has its own Python-facing create and prune bindings. The DDL and table naming live in the shared fixed_state_table_sql and state_table_name; each prune keeps its own retention rule." [[dupes.allowlist]] @@ -2878,6 +2878,7 @@ module_layers = [ "microbatches=plan", "model_config=front", "native_host=infra", + "native_host/audit_results_state=front", "native_host/consumption_state=front", "native_host/model_lookup=project", "native_host/observations_state=front", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1ffa6ed00d..7fa3ba1a37 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5510,6 +5510,13 @@ def build_observations_create_table_sql( render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], ) -> str: ... +def build_audit_results_prune_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + retain_versions: int, +) -> str: ... def build_observations_prune_sql( *, database: str | None, diff --git a/src/sqlbuild/adapter/contract/classes/audit_results.py b/src/sqlbuild/adapter/contract/classes/audit_results.py new file mode 100644 index 0000000000..592bcf8521 --- /dev/null +++ b/src/sqlbuild/adapter/contract/classes/audit_results.py @@ -0,0 +1,24 @@ +"""Default adapter pruning for the audit results state table.""" + +from __future__ import annotations + +from typing import Any, cast + + +class AuditResultsMixin: + """Provide portable audit results pruning for first-class warehouse adapters.""" + + def render_prune_audit_results_history_sql( + self, *, database: str | None, schema: str, retain_versions: int + ) -> str: + """Render SQL deleting audit results beyond the newest `retain_versions` per binding.""" + + from sqlbuild._native import build_audit_results_prune_sql + + adapter: Any = cast(Any, self) + return build_audit_results_prune_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + retain_versions=retain_versions, + ) diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 7366fe01c9..847c7a2c65 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2500,6 +2500,18 @@ def render_prune_observations_history_sql( del database, schema, retain_versions raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def render_prune_audit_results_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old audit results.""" + + del database, schema, retain_versions + raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def sql_analysis_dialect(self) -> str | None: """Return the configured SQL analysis dialect name, if any.""" diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 1e40f0a464..4f8ec2cb57 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -1163,6 +1163,17 @@ def render_prune_observations_history_sql( """Render SQL that prunes old observations.""" ... + @abstractmethod + def render_prune_audit_results_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old audit results.""" + ... + @abstractmethod def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 4658bb818c..3aa5666d9a 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -18,6 +18,7 @@ SqlValueKind, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( _COLLECTION_SQL_VALUE_KINDS, BaseAdapter, @@ -153,7 +154,12 @@ class BigQueryAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """BigQuery adapter backed by google-cloud-bigquery.""" diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index cc8d367733..e4ef093412 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -22,6 +22,7 @@ SqlValueKind, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -156,7 +157,12 @@ class ClickHouseAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """ClickHouse adapter that builds tables aside and rewrites whole partitions.""" diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index b5a1c03c8a..8914544982 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -21,6 +21,7 @@ SqlValueKind, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -146,7 +147,12 @@ class DatabricksAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """Databricks adapter backed by databricks-sql-connector.""" diff --git a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py index 820c5d6447..2a9eb06c16 100644 --- a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py +++ b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py @@ -6,6 +6,7 @@ from typing import Any, ClassVar from uuid import uuid4 +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import _quote_sql_string from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter @@ -15,7 +16,9 @@ from sqlbuild.microbatches.constants import MICROBATCH_GENERATION_COMMENT_PREFIX -class DuckDbAdapter(MicrobatchMixin, ConsumptionMixin, ObservationsMixin, DuckDbBackedAdapter): +class DuckDbAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, AuditResultsMixin, DuckDbBackedAdapter +): """DuckDB adapter allowing implicit main for targetless and named local targets.""" adapter_name: ClassVar[str] = BuiltinAdapter.DUCKDB.value diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index 6419cae9f0..d14283f6b6 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -8,6 +8,7 @@ from typing import Any, ClassVar from urllib.parse import urlencode +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin @@ -23,7 +24,9 @@ ) -class MotherDuckAdapter(MicrobatchMixin, ConsumptionMixin, ObservationsMixin, DuckDbBackedAdapter): +class MotherDuckAdapter( + MicrobatchMixin, ConsumptionMixin, ObservationsMixin, AuditResultsMixin, DuckDbBackedAdapter +): """MotherDuck adapter backed by DuckDB's MotherDuck connection support.""" adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 7907c13f32..581f912dbd 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -17,6 +17,7 @@ SqlValue, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _build_names_filter, @@ -124,7 +125,12 @@ class PostgresAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """PostgreSQL adapter backed by psycopg.""" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index a0469a709a..f5eae198f6 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -23,6 +23,7 @@ collect_snowflake_cost, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -199,7 +200,12 @@ class SnowflakeAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """Snowflake adapter backed by snowflake-connector-python.""" diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 3ae20ff28b..b933c09664 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -25,6 +25,7 @@ SqlValue, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -168,7 +169,12 @@ class SqliteAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """SQLite adapter: one database file, SQLBuild schemas stored as `schema__name` tables.""" diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index f6ecbc9cf5..fe336d2821 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -19,6 +19,7 @@ SqlValueKind, render_insert_source_freshness_records_sql, ) +from sqlbuild.adapter.contract.classes.audit_results import AuditResultsMixin from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _build_names_filter, @@ -135,7 +136,12 @@ def _render_tsql_distinct_condition(*, left: str, right: str) -> str: class SqlServerAdapter( - MicrobatchMixin, ConsumptionMixin, ObservationsMixin, UnkeyedDiffMixin, BaseAdapter + MicrobatchMixin, + ConsumptionMixin, + ObservationsMixin, + AuditResultsMixin, + UnkeyedDiffMixin, + BaseAdapter, ): """Microsoft SQL Server adapter backed by pymssql.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index 5ba61a7da0..ed9c45ac8c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -65,6 +65,7 @@ class JanitorDirectStatePruningE2ETestCase: expected_source_freshness_count_after: int expected_fingerprint_run_ids_after: tuple[str, ...] expected_source_freshness_run_ids_after: tuple[str, ...] + expected_audit_results_after: tuple[tuple[str, str], ...] @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index 6c2047da66..018c5da813 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -20,6 +20,8 @@ write_source_freshness_records, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.audit_results.main._write import write_audit_result_records +from sqlbuild.executor.audit_results.models import AuditResultRecord from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( execute_duckdb, prepare_inline_project, @@ -143,6 +145,55 @@ def create_janitor_scenario_relations(*, db_path: Path) -> None: connection.close() +JANITOR_AUDIT_BINDING_KEYS: tuple[str, ...] = ("orders.valid_orders", "customers.unique_ids") + + +def audit_record(*, binding_key: str, run_id: str, occurred_at: datetime) -> AuditResultRecord: + """One passing final audit result for `binding_key` in `run_id`.""" + + audit_name: str = binding_key.split(".")[-1] + return AuditResultRecord( + result_id=f"{binding_key}:{run_id}", + schema_version=1, + occurred_at=occurred_at, + invocation_id=f"invocation_{run_id}", + run_id=run_id, + audit_name=audit_name, + audit_definition_name=audit_name, + binding_key=binding_key, + definition_fingerprint="definition-fingerprint", + execution_fingerprint=f"execution_{run_id}", + evaluation_mode="violations", + run_scope_phase="final", + attachment_kind="model", + attached_target_kind="model", + attached_target_name="orders", + attached_column_name=None, + target_database=None, + target_schema="main", + target_name="orders", + severity="error", + outcome="pass", + execution_error=None, + violation_count=0, + measured_value=None, + sample_count=None, + sample_unit=None, + minimum_samples=None, + thresholds_json=None, + evidence_json=None, + evidence_count=None, + evidence_truncated=None, + evidence_error=None, + measurement_sql=None, + evidence_sql=None, + executed_sql="SELECT 1 WHERE FALSE", + sql_digest="digest", + metadata_json=None, + reused=False, + ) + + def create_direct_state_history(*, db_path: Path, schema: str = "main") -> None: import duckdb @@ -178,6 +229,22 @@ def create_direct_state_history(*, db_path: Path, schema: str = "main") -> None: render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, ) + write_audit_result_records( + connection=connection, + execute=lambda *, connection, sql: connection.execute(sql), + database=None, + schema=schema, + records=tuple( + audit_record( + binding_key=binding_key, + run_id=run_id, + occurred_at=datetime(2026, 1, 15, observed_hour, 10, 0, tzinfo=UTC), + ) + for binding_key in JANITOR_AUDIT_BINDING_KEYS + ), + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) write_source_freshness_records( connection=connection, execute=lambda *, connection, sql: connection.execute(sql), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index a7c21d7385..79694159b2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -393,12 +393,13 @@ def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_t plan_command=("--no-color", "plan"), expected_exit_code=0, expected_stdout_fragments=( - "direct state pruned 3", + "direct state pruned 4", "Eligible direct state pruning", "main._sqlbuild_fingerprints keep latest 2", "main._sqlbuild_source_freshness keep latest 2", "main._sqlbuild_consumption keep latest 2", - "Pruned 3 direct state tables.", + "main._sqlbuild_audit_results keep latest 2", + "Pruned 4 direct state tables.", ), expected_fingerprint_count_before=5, expected_fingerprint_count_after=3, @@ -406,6 +407,13 @@ def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_t expected_source_freshness_count_after=2, expected_fingerprint_run_ids_after=("run_003", "run_002"), expected_source_freshness_run_ids_after=("run_003", "run_002"), + # Newest two per audit binding. + expected_audit_results_after=( + ("customers.unique_ids", "run_003"), + ("customers.unique_ids", "run_002"), + ("orders.valid_orders", "run_003"), + ("orders.valid_orders", "run_002"), + ), ) ], ids=lambda case: case.description, @@ -477,6 +485,13 @@ def test_given_direct_state_history_when_running_janitor_then_it_prunes_history( "ORDER BY observed_at DESC, run_id DESC" ), ) == [(run_id,) for run_id in test_case.expected_source_freshness_run_ids_after] + assert query_duckdb( + db_path=db_path, + sql=( + "SELECT binding_key, run_id FROM main._sqlbuild_audit_results " + "WHERE run_id LIKE 'run_%' ORDER BY binding_key, occurred_at DESC, result_id DESC" + ), + ) == list(test_case.expected_audit_results_after) plan_result: subprocess.CompletedProcess[str] = run_sqb( command=test_case.plan_command, From c44579f790b702a327dd6fcf58385ce4be333ff3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:31:20 +0000 Subject: [PATCH 2/9] feat(location): sync after a signed-in build and at least every 30 minutes A [control] server and location in sqlbuild_project.toml, or SQLBUILD_CONTROL_* variables, let a signed-in sqb build ask that location to sync the target it built. The request is best-effort, bounded to five seconds and never changes the exit code. A sync request while one is pending for the target joins it instead of queueing another. sqb location serve also syncs an idle location at least every sync_at_least_every (default 30m, 0 or off disables it), as one more trigger into the same outstanding sync. --- .../sqlbuild-cli/src/argv/command_table.json | 7 + .../src/native_commands/_helpers/execution.rs | 55 ++++- .../src/native_commands/constants.rs | 4 + .../src/native_commands/main/_run_location.rs | 5 +- .../src/native_commands/main/mod.rs | 1 + .../native_commands/main/sync_after_build.rs | 9 + .../src/native_commands/models.rs | 24 +++ .../src/native_commands/tests/helpers.rs | 10 +- .../tests/test_location_metadata.rs | 189 +++++++++++++++++- .../src/native_commands/tests/test_types.rs | 20 ++ .../src/person_login/_helpers/transport.rs | 23 ++- .../src/person_login/constants.rs | 6 + .../sqlbuild-cli/src/person_login/main/mod.rs | 1 + .../main/request_warehouse_sync.rs | 8 +- .../src/person_login/main/sync_after_build.rs | 99 +++++++++ .../sqlbuild-cli/src/person_login/models.rs | 18 ++ .../src/project/_helpers/documents.rs | 17 +- .../src/project/_helpers/sections.rs | 56 +++++- .../sqlbuild-config/src/project/constants.rs | 6 + .../src/project/main/load_control_config.rs | 9 + .../sqlbuild-config/src/project/main/mod.rs | 2 + .../src/project/main/sync_interval_seconds.rs | 6 + crates/sqlbuild-config/src/project/models.rs | 11 + .../src/project/tests/test_load_config.rs | 117 ++++++++++- .../src/project/tests/test_types.rs | 8 +- .../platform/main/request_location_work.rs | 18 +- .../tests/api/test_person_location_work.rs | 26 +++ .../_helpers/build_command/command.rs | 23 ++- .../publication/person/helpers.py | 59 ++++++ .../publication/test_person_sql.py | 21 +- website/src/content/docs/docs/cli/server.mdx | 45 ++++- 31 files changed, 858 insertions(+), 45 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/sync_after_build.rs create mode 100644 crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs create mode 100644 crates/sqlbuild-config/src/project/main/load_control_config.rs create mode 100644 crates/sqlbuild-config/src/project/main/sync_interval_seconds.rs diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index bc15cbc49d..97cddf35e6 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -211,6 +211,13 @@ "help": "immutable deployment image identity to pin for workers", "group": 1 }, + { + "kind": "store", + "options": ["--sync-at-least-every"], + "dest": "location_sync_at_least_every", + "help": "for serve, sync the warehouse at least this often, such as 30m; 0 or off turns it off", + "group": 1 + }, { "kind": "store", "options": ["--run-id"], diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index d6903fafd9..9c10d453ec 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -14,6 +14,10 @@ use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; use crate::adhoc::main::adhoc_planner::adhoc_planner; +use sqlbuild_config::project::main::load_control_config::load_control_config; +use sqlbuild_config::project::main::project_config_paths::project_config_paths; +use sqlbuild_config::project::main::sync_interval_seconds::sync_interval_seconds; +use sqlbuild_config::project::models::ControlConfig; use sqlbuild_orchestration::api::models::{ ApiAuthentication, DevelopmentScope, HumanLogin, ServerFacts, }; @@ -73,8 +77,8 @@ use crate::native_commands::_helpers::terminal::{ use crate::native_commands::constants::{ ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_OVERLAP_SECONDS, METADATA_POLL_SECONDS, METADATA_READER_CLOSE_SECONDS, METADATA_RECONCILE_SECONDS, - METADATA_REPLY_BYTES, METADATA_SYNC_CHECK_SECONDS, SCHEDULER_COMMAND, - UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, + METADATA_REPLY_BYTES, METADATA_SAFETY_SYNC_SECONDS, METADATA_SYNC_CHECK_SECONDS, + SAFETY_SYNC_ENV, SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; @@ -335,7 +339,7 @@ pub(crate) fn location_result( pub(crate) fn metadata_sync( (locations, generation): (Vec, i64), - reconcile_seconds: u32, + (reconcile_seconds, safety_seconds): (u32, u64), ) -> LocationMetadataSync { LocationMetadataSync { locations, @@ -355,6 +359,8 @@ pub(crate) fn metadata_sync( next_at: Instant::now(), idle: true, cycle_started_at: None, + safety_seconds, + safety_at: Instant::now() + Duration::from_secs(safety_seconds), change_tokens: HashMap::new(), scan_token: None, scan_delivered: false, @@ -483,15 +489,9 @@ pub(crate) async fn begin_after_run( Ok(due) => due, Err(error) => return (state, Err(error)), }; - let ran = match (state.cycle_started_at, due.runs_ended_at) { - (None, _) => true, - (Some(started_at), Some(ended_at)) => ended_at >= started_at, - (Some(_), None) => false, - }; + let ran = state.sync_due(due.runs_ended_at); if ran { - state.idle = false; - state.cycle_started_at = Some(due.now); - state.next_at = Instant::now(); + state.begin_sync(due.now); } (state, Ok(ran)) } @@ -589,6 +589,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync } state.idle = true; state.next_at = Instant::now(); + state.safety_at = Instant::now() + Duration::from_secs(state.safety_seconds); drop(write_output(&[( Stream::Stderr, "Warehouse history synchronization completed.\n", @@ -699,6 +700,19 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result Result, + environment: Option<&str>, + project: impl FnOnce() -> Result, +) -> Result { + if let Some(text) = flag { + return sync_interval_seconds(text).map_err(|help| format!("--sync-at-least-every {help}")); + } + if let Some(text) = environment.filter(|text| !text.trim().is_empty()) { + return sync_interval_seconds(text).map_err(|help| format!("{SAFETY_SYNC_ENV} {help}")); + } + Ok(project()? + .sync_at_least_every_seconds + .unwrap_or(METADATA_SAFETY_SYNC_SECONDS)) +} + /// Reconcile from the server after restarts or lost acknowledgements; transient outages retry. pub(crate) async fn location_generation( options: &LocationServe, diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 4b7f7d30a8..a4ed2b6af9 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -98,6 +98,10 @@ pub(crate) const METADATA_READER_CLOSE_SECONDS: u64 = 5; pub(crate) const METADATA_SYNC_CHECK_SECONDS: u64 = 5; /// Full key-set reconciliation is independent of the incremental poll and overlap window. pub(crate) const METADATA_RECONCILE_SECONDS: u32 = 300; +/// An idle location syncs at least this often, so a lost run or request signal cannot leave drift. +pub(crate) const METADATA_SAFETY_SYNC_SECONDS: u64 = 1800; +/// Overrides `[control] sync_at_least_every`; `--sync-at-least-every` overrides both. +pub(crate) const SAFETY_SYNC_ENV: &str = "SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY"; pub(crate) const METADATA_OVERLAP_SECONDS: i64 = 300; /// A source-key cursor can repeat the last row's base64-encoded definition beside the records. pub(crate) const METADATA_REPLY_BYTES: u64 = diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index b97562018e..30e7528589 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -121,7 +121,10 @@ async fn watch_location( published_generation = Some(generation); metadata = Some(metadata_sync( (locations, generation), - options.metadata_reconcile_seconds, + ( + options.metadata_reconcile_seconds, + options.safety_sync_seconds, + ), )); let Some(files) = check_published_location( options, diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index c5ee3267d2..8da5144bc0 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -18,3 +18,4 @@ pub(crate) mod run_location_invocation; pub(crate) mod run_login_invocation; pub mod run_native_invocation; pub(crate) mod run_person_invocation; +pub mod sync_after_build; diff --git a/crates/sqlbuild-cli/src/native_commands/main/sync_after_build.rs b/crates/sqlbuild-cli/src/native_commands/main/sync_after_build.rs new file mode 100644 index 0000000000..b9dfd37511 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/sync_after_build.rs @@ -0,0 +1,9 @@ +//! After a direct build, ask the configured location to sync the target the build used. + +use std::path::Path; + +/// One stderr line about the sync, or `None` when no location is configured or nobody is signed in. +/// Never fails: every problem becomes a skipped line, and the request is bounded to a few seconds. +pub fn sync_after_build(project_dir: &Path, target: &str) -> Option { + crate::person_login::main::sync_after_build::sync_after_build(project_dir, target) +} diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 4123e2b9e1..9a1e6ce218 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -149,6 +149,8 @@ pub(crate) struct LocationServe { pub(crate) target: String, pub(crate) image_id: Option, pub(crate) metadata_reconcile_seconds: u32, + /// Seconds between safety syncs of an idle location; `0` turns them off. + pub(crate) safety_sync_seconds: u64, } /// Retained source observations survive compilation; warehouse exclusions never replace them. @@ -209,6 +211,10 @@ pub(crate) struct LocationMetadataSync { pub(crate) idle: bool, /// Server time when the current or last synchronization began; `None` before the first. pub(crate) cycle_started_at: Option>, + /// Seconds between safety syncs; `0` turns them off. + pub(crate) safety_seconds: u64, + /// When an idle location syncs even though no run ended and nobody asked. + pub(crate) safety_at: tokio::time::Instant, /// Per scan index: the table's change token after its last complete read. pub(crate) change_tokens: HashMap, /// The token read before the current scan's first page, recorded once the scan completes. @@ -240,6 +246,24 @@ impl LocationMetadataSync { } } + /// Whether an idle location starts a sync: never synced, asked to, a run ended since the last + /// one began, or the safety interval passed. Each is one outstanding sync, never a second one. + pub(crate) fn sync_due(&self, runs_ended_at: Option>) -> bool { + let ran = match (self.cycle_started_at, runs_ended_at) { + (None, _) => true, + (Some(started_at), Some(ended_at)) => ended_at >= started_at, + (Some(_), None) => false, + }; + ran || (self.safety_seconds > 0 && tokio::time::Instant::now() >= self.safety_at) + } + + /// Start the sync `sync_due` asked for, at server time `now`. + pub(crate) fn begin_sync(&mut self, now: DateTime) { + self.idle = false; + self.cycle_started_at = Some(now); + self.next_at = tokio::time::Instant::now(); + } + /// Failed samples revisit catalog before retrying; no frozen delivery is discarded. pub(crate) fn retry_read(&mut self) { self.statistics = None; diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs index ed08157d2e..e8afdadad9 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -1,6 +1,6 @@ use std::path::Path; -use sqlbuild_orchestration::warehouse_metadata::models::MetadataBatch; +use sqlbuild_orchestration::warehouse_metadata::models::{MetadataBatch, WarehouseStateLocation}; use uuid::Uuid; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; @@ -60,3 +60,11 @@ pub(super) fn acknowledge_complete_pages( acknowledge_metadata_page(state) }) } + +/// One state location, for metadata sync tests. +pub(super) fn one_location() -> Vec { + vec![WarehouseStateLocation { + database: Some("analytics".into()), + schema: "state".into(), + }] +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index cb3a03c279..e6a3d35aff 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -3,9 +3,15 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryKind, WarehouseStateLocation, }; -use crate::native_commands::_helpers::execution::metadata_sync; -use crate::native_commands::tests::helpers::acknowledge_complete_pages; -use crate::native_commands::tests::test_types::{MetadataRetryTestCase, UnchangedScanTestCase}; +use chrono::Utc; +use sqlbuild_config::project::models::ControlConfig; +use tokio::time::Instant; + +use crate::native_commands::_helpers::execution::{metadata_sync, safety_sync_seconds}; +use crate::native_commands::tests::helpers::{acknowledge_complete_pages, one_location}; +use crate::native_commands::tests::test_types::{ + MetadataRetryTestCase, SafetySyncSettingTestCase, SyncCoalescingTestCase, UnchangedScanTestCase, +}; #[test] fn given_failed_metadata_read_when_back_at_statistics_then_page_shrinks_until_scan_completes() { @@ -34,7 +40,7 @@ fn given_failed_metadata_read_when_back_at_statistics_then_page_shrinks_until_sc database: Some("analytics".into()), schema: "state".into(), }; - let mut state = metadata_sync((vec![location], 1), 3600); + let mut state = metadata_sync((vec![location], 1), (3600, 0)); state.index = WarehouseHistoryKind::ALL .iter() .position(|kind| *kind == test_case.failed_kind) @@ -133,7 +139,7 @@ fn given_recorded_change_token_when_scanning_again_then_only_proven_unchanged_re database: Some("analytics".into()), schema: "state".into(), }; - let mut state = metadata_sync((vec![location], 1), 3600); + let mut state = metadata_sync((vec![location], 1), (3600, 0)); state.reconcile = test_case.read_reconciled; state.scan_token = Some("t1".into()); state.scan_delivered = test_case.read_delivered; @@ -161,3 +167,176 @@ fn given_recorded_change_token_when_scanning_again_then_only_proven_unchanged_re ); } } + +#[test] +fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync_runs() { + let test_cases = [ + SyncCoalescingTestCase { + description: "four build requests and a run ending share one follow-up", + requests: 4, + run_ends_during_sync: true, + safety_seconds: 1800, + safety_passes_during_sync: false, + expected_follow_ups: 1, + }, + SyncCoalescingTestCase { + description: "a passed safety interval joins pending requests instead of adding a sync", + requests: 2, + run_ends_during_sync: false, + safety_seconds: 1800, + safety_passes_during_sync: true, + expected_follow_ups: 1, + }, + SyncCoalescingTestCase { + description: "a safety interval that passes during a sync adds no extra one", + requests: 0, + run_ends_during_sync: false, + safety_seconds: 1800, + safety_passes_during_sync: true, + expected_follow_ups: 0, + }, + SyncCoalescingTestCase { + description: "no signal leaves the location idle", + requests: 0, + run_ends_during_sync: false, + safety_seconds: 1800, + safety_passes_during_sync: false, + expected_follow_ups: 0, + }, + ]; + for test_case in test_cases { + let started = Utc::now(); + let mut state = metadata_sync((one_location(), 1), (3600, test_case.safety_seconds)); + assert!( + state.sync_due(None), + "{}: the first sync is due", + test_case.description + ); + state.begin_sync(started); + for _ in 0..test_case.requests { + state.request_sync(); + } + let runs_ended_at = test_case + .run_ends_during_sync + .then(|| started + chrono::Duration::seconds(1)); + if test_case.safety_passes_during_sync { + state.safety_at = Instant::now(); + } + state = acknowledge_complete_pages(state, WarehouseHistoryKind::ALL.len()); + assert!( + state.idle, + "{}: the first sync completed", + test_case.description + ); + let mut follow_ups = 0; + for poll in 1..=5 { + if state.sync_due(runs_ended_at) { + follow_ups += 1; + state.begin_sync(started + chrono::Duration::seconds(10 * poll)); + state = acknowledge_complete_pages(state, WarehouseHistoryKind::ALL.len()); + } + } + assert_eq!( + follow_ups, test_case.expected_follow_ups, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_an_idle_location_when_the_safety_interval_passes_then_it_syncs_unless_turned_off() { + let started = Utc::now(); + for (description, safety_seconds, passed, expected) in [ + ("the interval passed while idle", 1800, true, true), + ("the interval has not passed", 1800, false, false), + ("0 turns safety syncs off", 0, true, false), + ] { + let mut state = metadata_sync((one_location(), 1), (3600, safety_seconds)); + state.begin_sync(started); + state = acknowledge_complete_pages(state, WarehouseHistoryKind::ALL.len()); + if passed { + state.safety_at = Instant::now(); + } + assert_eq!(state.sync_due(None), expected, "{description}"); + } +} + +#[test] +fn given_safety_sync_settings_when_resolving_then_flag_beats_environment_beats_project() { + let test_cases = [ + SafetySyncSettingTestCase { + description: "nothing set uses thirty minutes", + flag: None, + environment: None, + project: None, + expected: Ok(1800), + }, + SafetySyncSettingTestCase { + description: "the project file sets it", + flag: None, + environment: None, + project: Some(3600), + expected: Ok(3600), + }, + SafetySyncSettingTestCase { + description: "the environment overrides the project file", + flag: None, + environment: Some("2h"), + project: Some(3600), + expected: Ok(7200), + }, + SafetySyncSettingTestCase { + description: "a blank environment value falls through to the project file", + flag: None, + environment: Some(" "), + project: Some(3600), + expected: Ok(3600), + }, + SafetySyncSettingTestCase { + description: "the flag overrides the environment", + flag: Some("off"), + environment: Some("2h"), + project: Some(3600), + expected: Ok(0), + }, + SafetySyncSettingTestCase { + description: "a flag under a minute is refused", + flag: Some("30s"), + environment: None, + project: None, + expected: Err("--sync-at-least-every must be a duration of at least '1m'"), + }, + SafetySyncSettingTestCase { + description: "an unparseable environment value is refused", + flag: None, + environment: Some("-5m"), + project: None, + expected: Err( + "SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY must be a duration of at least '1m'", + ), + }, + ]; + for test_case in test_cases { + let resolved = safety_sync_seconds(test_case.flag, test_case.environment, || { + Ok(ControlConfig { + sync_at_least_every_seconds: test_case.project, + ..ControlConfig::default() + }) + }); + match (resolved, test_case.expected) { + (Ok(seconds), Ok(expected)) => { + assert_eq!(seconds, expected, "{}", test_case.description); + } + (Err(message), Err(fragment)) => assert!( + message.starts_with(fragment), + "{}: {message}", + test_case.description + ), + (resolved, expected) => panic!( + "{}: resolved {resolved:?}, expected {expected:?}", + test_case.description + ), + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 46d3cc8389..2c7a10743c 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -111,3 +111,23 @@ pub(super) struct UnchangedScanTestCase { pub(super) mirror_holds_rows: bool, pub(super) expected_token: Option<&'static str>, } + +pub(super) struct SyncCoalescingTestCase { + pub(super) description: &'static str, + /// Sync requests that arrive while one sync is in progress, such as one per CLI build. + pub(super) requests: usize, + pub(super) run_ends_during_sync: bool, + /// Seconds between safety syncs; `0` turns them off. + pub(super) safety_seconds: u64, + /// The safety interval passes while the sync is in progress. + pub(super) safety_passes_during_sync: bool, + pub(super) expected_follow_ups: usize, +} + +pub(super) struct SafetySyncSettingTestCase { + pub(super) description: &'static str, + pub(super) flag: Option<&'static str>, + pub(super) environment: Option<&'static str>, + pub(super) project: Option, + pub(super) expected: Result, +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs index 7703fe34ff..e6bdf19a7f 100644 --- a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs +++ b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs @@ -13,7 +13,7 @@ use uuid::Uuid; use crate::person_login::_helpers::credentials; use crate::person_login::constants::REQUEST_SECONDS; -use crate::person_login::models::{PersonClient, PersonCredential}; +use crate::person_login::models::{Bootstrap, LocationTarget, PersonClient, PersonCredential}; pub(crate) fn connect(origin: &str, ca: Option<&Path>) -> Result { let mut builder = reqwest::Client::builder() @@ -139,11 +139,12 @@ pub(crate) async fn warehouse_sync( client: &PersonClient, credential: &PersonCredential, (location, target): (Uuid, &str), + reason: &str, ) -> Result { let path = target_path(client, (location, target), &["warehouse-syncs"])?; let request = PersonWorkRequest { request_id: Uuid::now_v7(), - reason: "Requested from the CLI".into(), + reason: reason.into(), }; send_json::( authorized(client, reqwest::Method::POST, &path, credential)?.json(&request), @@ -157,6 +158,24 @@ pub(crate) async fn warehouse_sync( .map(|response| response.applied) } +/// The location targets the signed-in person can view. +pub(crate) async fn location_targets( + client: &PersonClient, + credential: &PersonCredential, +) -> Result, std::io::Error> { + send_json::( + authorized( + client, + reqwest::Method::GET, + "/api/control/bootstrap", + credential, + )?, + ("read locations from", "Location lookup", "location list"), + ) + .await + .map(|bootstrap| bootstrap.access.locations) +} + /// Each failure names the action, e.g. "could not read published SQL from the server". async fn send_json( request: reqwest::RequestBuilder, diff --git a/crates/sqlbuild-cli/src/person_login/constants.rs b/crates/sqlbuild-cli/src/person_login/constants.rs index d525dc760e..5b55c33035 100644 --- a/crates/sqlbuild-cli/src/person_login/constants.rs +++ b/crates/sqlbuild-cli/src/person_login/constants.rs @@ -5,3 +5,9 @@ pub(super) const POLL_SECONDS: u64 = 1; pub(super) const LOGIN_SECONDS: u64 = 300; pub(super) const REQUEST_SECONDS: u64 = 15; pub(super) const KEYRING_SERVICE: &str = "sqlbuild-cli"; +/// Overrides `[control] server` and `[control] location`; an extra trusted CA for the server. +pub(super) const CONTROL_SERVER_ENV: &str = "SQLBUILD_CONTROL_SERVER"; +pub(super) const CONTROL_LOCATION_ENV: &str = "SQLBUILD_CONTROL_LOCATION"; +pub(super) const CONTROL_CA_CERT_ENV: &str = "SQLBUILD_CONTROL_CA_CERT"; +/// A post-build sync request gives up after this long so the build's exit is never held. +pub(super) const BUILD_SYNC_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-cli/src/person_login/main/mod.rs b/crates/sqlbuild-cli/src/person_login/main/mod.rs index db72101ec9..89c8db7e03 100644 --- a/crates/sqlbuild-cli/src/person_login/main/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/main/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod read_node_sql; pub(crate) mod request_warehouse_sync; pub(crate) mod run_login; +pub(crate) mod sync_after_build; diff --git a/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs b/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs index 28f04804a5..3f51a28e27 100644 --- a/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs +++ b/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs @@ -11,6 +11,12 @@ pub(crate) fn request_warehouse_sync( (location, target): (Uuid, &str), ) -> Result { transport::with_person_session(server, async |client, credential| { - transport::warehouse_sync(client, credential, (location, target)).await + transport::warehouse_sync( + client, + credential, + (location, target), + "Requested from the CLI", + ) + .await }) } diff --git a/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs b/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs new file mode 100644 index 0000000000..52f08d76c7 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs @@ -0,0 +1,99 @@ +//! A signed-in `sqb build` asks its configured location to refresh the warehouse copy, best-effort. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use sqlbuild_config::project::main::load_control_config::load_control_config; +use sqlbuild_config::project::main::project_config_paths::project_config_paths; +use sqlbuild_orchestration::api::main::parse_origin::parse_origin; + +use crate::person_login::_helpers::{credentials, transport}; +use crate::person_login::constants::{ + BUILD_SYNC_SECONDS, CONTROL_CA_CERT_ENV, CONTROL_LOCATION_ENV, CONTROL_SERVER_ENV, +}; + +fn setting(name: &str, configured: Option) -> Option { + std::env::var(name) + .ok() + .filter(|value| !value.trim().is_empty()) + .or(configured) +} + +/// One stderr line about the sync, or `None` when no location is configured or nobody is signed in. +pub(crate) fn sync_after_build(project_dir: &Path, target: &str) -> Option { + let configured = load_control_config(&project_config_paths(project_dir)).ok()?; + let server = setting(CONTROL_SERVER_ENV, configured.server); + let location = setting(CONTROL_LOCATION_ENV, configured.location); + let (server, location) = match (server, location) { + (Some(server), Some(location)) => (server, location), + (None, None) => return None, + _ => { + return Some( + "Warehouse sync skipped: set both [control] server and location, or neither.\n" + .into(), + ); + } + }; + let skipped = |error: std::io::Error| Some(format!("Warehouse sync skipped: {error}\n")); + let origin = match parse_origin(&server, "CLI") { + Ok(origin) => origin, + Err(error) => return skipped(std::io::Error::other(error)), + }; + let credential = match credentials::load(&origin) { + Ok(Some(credential)) => credential, + Ok(None) => return None, + Err(error) => return skipped(error), + }; + let ca = std::env::var_os(CONTROL_CA_CERT_ENV).map(PathBuf::from); + let client = match transport::connect(&origin, ca.as_deref()) { + Ok(client) => client, + Err(error) => return skipped(error), + }; + let request = async { + let targets = transport::location_targets(&client, &credential).await?; + let Some(found) = targets + .iter() + .find(|candidate| candidate.name == location && candidate.target == target) + else { + let named = targets.iter().any(|candidate| candidate.name == location); + return Ok(if named { + format!( + "Warehouse sync skipped: location '{location}' has no target '{target}' you can view on {origin}.\n" + ) + } else { + format!( + "Warehouse sync skipped: no location named '{location}' that you can view on {origin}.\n" + ) + }); + }; + transport::warehouse_sync( + &client, + &credential, + (found.location_id, target), + "Requested after sqb build", + ) + .await?; + Ok::<_, std::io::Error>(format!( + "Warehouse sync requested for location '{location}' target '{target}'.\n" + )) + }; + let outcome = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(std::io::Error::other) + .and_then(|runtime| { + runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(BUILD_SYNC_SECONDS), request) + .await + .unwrap_or_else(|_| { + Err(std::io::Error::other(format!( + "the server did not answer within {BUILD_SYNC_SECONDS} seconds" + ))) + }) + }) + }); + match outcome { + Ok(line) => Some(line), + Err(error) => skipped(error), + } +} diff --git a/crates/sqlbuild-cli/src/person_login/models.rs b/crates/sqlbuild-cli/src/person_login/models.rs index ddcc46e8da..0d9dc926b0 100644 --- a/crates/sqlbuild-cli/src/person_login/models.rs +++ b/crates/sqlbuild-cli/src/person_login/models.rs @@ -55,3 +55,21 @@ pub(super) struct PersonClient { pub(super) client: reqwest::Client, pub(super) origin: String, } + +/// The parts of `GET /api/control/bootstrap` that name the location targets a person can view. +#[derive(Deserialize)] +pub(super) struct Bootstrap { + pub(super) access: BootstrapAccess, +} + +#[derive(Deserialize)] +pub(super) struct BootstrapAccess { + pub(super) locations: Vec, +} + +#[derive(Deserialize)] +pub(super) struct LocationTarget { + pub(super) location_id: uuid::Uuid, + pub(super) name: String, + pub(super) target: String, +} diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 3ddb50b887..7acfc90165 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -14,9 +14,9 @@ use crate::project::_helpers::help::{ }; use crate::project::_helpers::keys::Place; use crate::project::_helpers::sections::{ - load_constants, load_cost, load_cursors, load_dbt, load_diff, load_git, load_janitor, - load_local_dbt, load_microbatches, load_migrations, load_policy_flag, load_review, - load_scenario, load_settings, load_snapshots, setting_overrides, + load_constants, load_control, load_cost, load_cursors, load_dbt, load_diff, load_git, + load_janitor, load_local_dbt, load_microbatches, load_migrations, load_policy_flag, + load_review, load_scenario, load_settings, load_snapshots, setting_overrides, }; use crate::project::_helpers::sinks::load_sinks; use crate::project::_helpers::targets::{ @@ -28,7 +28,7 @@ use crate::project::constants::{ LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; -use crate::project::models::{Built, ConfigFile, ConfigPaths, ReviewConfig}; +use crate::project::models::{Built, ConfigFile, ConfigPaths, ControlConfig, ReviewConfig}; use crate::project::types::{Entries, Fields, Loaded}; use crate::recall::constants::RECALL_SECTION; use crate::recall::main::recall_section::recall_section; @@ -278,6 +278,7 @@ fn project_fields(payload: &Entries) -> Loaded { } load_git(get(payload, "git"), PROJECT_FILE)?; load_review(get(payload, "review"), PROJECT_FILE)?; + load_control(get(payload, "control"), PROJECT_FILE)?; let mut fields = identity(payload)?; fields.extend(sections(payload)?); fields.push(("targets", targets(payload)?)); @@ -434,6 +435,14 @@ pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result, LoadFa Ok(load_git(get(payload, "git"), PROJECT_FILE)?) } +/// `[control]` of `sqlbuild_project.toml`, validated as project loading validates it. +pub(crate) fn control_config(paths: &ConfigPaths) -> Result { + let file = resolve(paths, false)?; + let table = load_toml_mapping(paths, file)?; + let payload = table.as_map().unwrap_or_default(); + Ok(load_control(get(payload, "control"), PROJECT_FILE)?) +} + /// `[review]` of `sqlbuild_project.toml`, validated as project loading validates it. pub(crate) fn review_config(paths: &ConfigPaths) -> Result { let file = resolve(paths, false)?; diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 31016fb3df..4e4f3ae0bb 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -22,12 +22,15 @@ use crate::project::constants::{ SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; +use crate::project::constants::{ + CONTROL_KEYS, MIN_SYNC_INTERVAL_SECONDS, SYNC_INTERVAL_HELP, SYNC_INTERVAL_OFF, +}; use crate::project::constants::{ DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, }; use crate::project::errors::ConfigFailure; -use crate::project::models::{Built, ConfigFile, ReviewConfig}; +use crate::project::models::{Built, ConfigFile, ControlConfig, ReviewConfig}; use crate::project::types::{Entries, Fields, Loaded}; use sqlbuild_core::constants::ZERO_DAY_DURATION; @@ -571,6 +574,57 @@ pub(crate) fn load_git(payload: Option<&ConfigValue>, file: ConfigFile) -> Loade }) } +/// A safety sync interval in seconds, `0` when off; the error completes " ...". +pub(crate) fn sync_interval(text: &str) -> Result { + if SYNC_INTERVAL_OFF.contains(&text) { + return Ok(0); + } + let seconds: u64 = fixed_duration_seconds(text, false) + .ok() + .flatten() + .and_then(|seconds| seconds.parse().ok()) + .ok_or(SYNC_INTERVAL_HELP)?; + if seconds < MIN_SYNC_INTERVAL_SECONDS { + return Err(SYNC_INTERVAL_HELP); + } + Ok(seconds) +} + +/// `[control]`: where a signed-in build asks for a sync, and how often a location syncs anyway. +pub(crate) fn load_control( + payload: Option<&ConfigValue>, + file: ConfigFile, +) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "control"))?; + validate_allowed_keys(mapping, &CONTROL_KEYS, place(file, "control"))?; + let server = optional_str(mapping, "server").map_err(|_| { + ConfigFailure::at( + file, + "control.server must be an HTTPS server origin such as 'https://sqlbuild.example.com'", + ) + })?; + let location = optional_str(mapping, "location").map_err(|_| { + ConfigFailure::at(file, "control.location must be a non-empty location name") + })?; + let sync_at_least_every_seconds = match get(mapping, "sync_at_least_every") { + None => None, + Some(value) => Some( + match value { + ConfigValue::String(text) => sync_interval(text), + _ => Err(SYNC_INTERVAL_HELP), + } + .map_err(|help| { + ConfigFailure::at(file, format!("control.sync_at_least_every {help}")) + })?, + ), + }; + Ok(ControlConfig { + server, + location, + sync_at_least_every_seconds, + }) +} + /// `[review]`: the defaults of `sqb review`, validated so a typo fails like any other section. pub(crate) fn load_review(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { let mapping = coerce_mapping(payload, place(file, "review"))?; diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 753734ebd8..987a3563ea 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -200,6 +200,12 @@ pub(crate) const TARGET_KEYS: [&str; 21] = [ "recall", "concurrency", ]; +pub(crate) const CONTROL_KEYS: [&str; 3] = ["location", "server", "sync_at_least_every"]; +/// Shortest safety sync interval, so a typo cannot bring back aggressive warehouse polling. +pub(crate) const MIN_SYNC_INTERVAL_SECONDS: u64 = 60; +pub(crate) const SYNC_INTERVAL_OFF: [&str; 3] = ["0", "0s", "off"]; +pub(crate) const SYNC_INTERVAL_HELP: &str = + "must be a duration of at least '1m' such as '30m', or '0' or 'off' to turn safety syncs off"; pub(crate) const REVIEW_KEYS: [&str; 4] = ["against", "critical_tags", "downstream", "schema_prefix"]; pub(crate) const REVIEW_DOWNSTREAM_POLICIES: [&str; 3] = ["critical", "all", "none"]; diff --git a/crates/sqlbuild-config/src/project/main/load_control_config.rs b/crates/sqlbuild-config/src/project/main/load_control_config.rs new file mode 100644 index 0000000000..0bb195b57a --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/load_control_config.rs @@ -0,0 +1,9 @@ +//! Read `[control]` from `sqlbuild_project.toml`. + +use crate::project::errors::LoadFailure; +use crate::project::models::{ConfigPaths, ControlConfig}; + +/// The configured server, location and safety sync interval; every field is `None` when unset. +pub fn load_control_config(paths: &ConfigPaths) -> Result { + crate::project::_helpers::documents::control_config(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 8cd4f858ed..7d1a244037 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -1,5 +1,6 @@ pub mod deprecated_setting_warnings; pub mod explicit_references_help; +pub mod load_control_config; pub mod load_git_base_ref; pub mod load_local_config; pub mod load_project_config; @@ -11,4 +12,5 @@ pub mod resolve_config_file; pub mod setting_help_text; pub mod snippet_help_text; pub mod sql_analysis_off_guidance; +pub mod sync_interval_seconds; pub mod unsupported_target_warehouse; diff --git a/crates/sqlbuild-config/src/project/main/sync_interval_seconds.rs b/crates/sqlbuild-config/src/project/main/sync_interval_seconds.rs new file mode 100644 index 0000000000..e06a803f1a --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/sync_interval_seconds.rs @@ -0,0 +1,6 @@ +//! Parse a safety sync interval the way `[control] sync_at_least_every` accepts it. + +/// Whole seconds, `0` for `"0"`, `"0s"` or `"off"`; the error completes " ...". +pub fn sync_interval_seconds(text: &str) -> Result { + crate::project::_helpers::sections::sync_interval(text) +} diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index a36260035b..3c9b9b461e 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -56,6 +56,17 @@ pub struct SettingSite<'a> { pub key: &'a str, } +/// `[control]` of `sqlbuild_project.toml`: the server and location this project syncs with. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ControlConfig { + /// HTTPS origin of the SQLBuild server, as `sqb login --server` takes it. + pub server: Option, + /// Name of the location whose warehouse copy a signed-in `sqb build` refreshes. + pub location: Option, + /// Seconds between safety syncs of `sqb location serve`; `Some(0)` turns them off. + pub sync_at_least_every_seconds: Option, +} + /// `[review]` of `sqlbuild_project.toml`: the defaults `sqb review` uses when flags are omitted. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ReviewConfig { diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index e556567d29..7b57c16140 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,11 +1,13 @@ use crate::project::errors::ConfigFailure; +use crate::project::main::load_control_config::load_control_config; use crate::project::main::load_git_base_ref::load_git_base_ref; use crate::project::main::load_review_config::load_review_config; use crate::project::models::ConfigFile; -use crate::project::models::ReviewConfig; +use crate::project::models::{ControlConfig, ReviewConfig}; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; use crate::project::tests::test_types::{ - GitBaseRefTestCase, ProjectFailureTestCase, ReviewConfigErrorTestCase, ReviewConfigTestCase, + ControlConfigTestCase, GitBaseRefTestCase, ProjectFailureTestCase, ReviewConfigErrorTestCase, + ReviewConfigTestCase, }; const PROJECT: &str = "name = 'orders'\nadapter = 'duckdb'\n"; @@ -261,3 +263,114 @@ fn given_invalid_review_section_when_reading_then_it_is_rejected() { ); } } + +#[test] +fn given_control_section_when_reading_then_returns_its_values_and_interval_seconds() { + let control = |sync: &str| ControlConfig { + server: Some("https://sqlbuild.example.com".to_owned()), + location: Some("orders-warehouse".to_owned()), + sync_at_least_every_seconds: sync.parse().ok(), + }; + let test_cases = [ + ControlConfigTestCase { + description: "every control key with a compound interval", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\n\ + server = 'https://sqlbuild.example.com'\nlocation = 'orders-warehouse'\n\ + sync_at_least_every = '1h30m'\n", + expected_config: control("5400"), + }, + ControlConfigTestCase { + description: "exactly the one-minute minimum", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\n\ + server = 'https://sqlbuild.example.com'\nlocation = 'orders-warehouse'\n\ + sync_at_least_every = '60s'\n", + expected_config: control("60"), + }, + ControlConfigTestCase { + description: "off turns safety syncs off", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\n\ + server = 'https://sqlbuild.example.com'\nlocation = 'orders-warehouse'\n\ + sync_at_least_every = 'off'\n", + expected_config: control("0"), + }, + ControlConfigTestCase { + description: "a bare zero turns safety syncs off", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\n\ + server = 'https://sqlbuild.example.com'\nlocation = 'orders-warehouse'\n\ + sync_at_least_every = '0'\n", + expected_config: control("0"), + }, + ControlConfigTestCase { + description: "no control section", + project: PROJECT, + expected_config: ControlConfig::default(), + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let config: ControlConfig = load_control_config(&paths).expect(test_case.description); + + assert_eq!( + config, test_case.expected_config, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_control_section_when_loading_then_project_loading_rejects_it() { + let interval = "control.sync_at_least_every must be a duration of at least '1m'"; + let test_cases = [ + ReviewConfigErrorTestCase { + description: "an interval one second under the minimum", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = '59s'\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "a negative interval", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = '-30m'\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "an interval that is not a duration", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = 'hourly'\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "a calendar interval", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = '1mo'\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "an integer interval", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nsync_at_least_every = 1800\n", + expected_fragment: interval, + }, + ReviewConfigErrorTestCase { + description: "a misspelt control key", + project: "name = 'orders'\nadapter = 'duckdb'\n[control]\nlocaton = 'orders-warehouse'\n", + expected_fragment: "control contains unknown key(s): locaton", + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let failures: [String; 2] = [ + format!( + "{:?}", + load_control_config(&paths).expect_err(test_case.description) + ), + format!("{:?}", first_failure(&paths).expect(test_case.description)), + ]; + + for failure in failures { + assert!( + failure.contains(test_case.expected_fragment), + "{}: {failure}", + test_case.description + ); + } + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index e4dda5fb4e..66f541daa4 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::project::errors::ConfigFailure; -use crate::project::models::ReviewConfig; +use crate::project::models::{ControlConfig, ReviewConfig}; pub(super) struct ProjectFailureTestCase { pub(super) description: &'static str, @@ -21,6 +21,12 @@ pub(crate) struct GitBaseRefTestCase { pub(crate) expected_base_ref: Option<&'static str>, } +pub(crate) struct ControlConfigTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) expected_config: ControlConfig, +} + pub(crate) struct ReviewConfigTestCase { pub(crate) description: &'static str, pub(crate) project: &'static str, diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index fa1e4427f7..315985ac78 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -1,4 +1,5 @@ //! Named work requires a trusted owner or live person authority; retries preserve the original pin. +//! A sync request while one is still pending for the target is already applied: one sync covers both. use chrono::Duration; use uuid::Uuid; @@ -11,7 +12,8 @@ use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub async fn request_location_work( store: &ControlStore, @@ -56,6 +58,20 @@ pub async fn request_location_work( commit(transaction).await?; return Ok(false); } + let transaction = if matches!(request.work, LocationWork::SyncWarehouse {}) { + let now = transaction.now; + let (transaction, pending) = query(transaction, + "SELECT request_id FROM sqb_location_work WHERE location_id=$1 AND target=$2 \ + AND completed_at IS NULL AND expires_at>$3 AND request_json->'work'->>'kind'='sync_warehouse' LIMIT 1", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Time(now)]).await?; + if !pending.is_empty() { + commit(transaction).await?; + return Ok(false); + } + transaction + } else { + transaction + }; commit(enqueue_location_work(transaction, &context, request, Duration::minutes(5)).await?) .await?; Ok(true) diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs index 0e5058430d..5e6e61e5a1 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs @@ -15,6 +15,7 @@ use crate::credentials::models::{ }; use crate::credentials::tests::location_helpers::create_location; use crate::platform::main::{ + read_location_work::read_location_work, read_person_connection_test::read_person_connection_test, request_location_work::request_location_work, }; @@ -22,7 +23,10 @@ use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::runs::tests::helpers::snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; +use crate::store::models::TransactionMode; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; @@ -141,6 +145,28 @@ fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_preced .await .expect("connection-test authority requests a sync") ); + let repeated = LocationWorkRequest { + request_id: Uuid::now_v7(), + reason: "Requested after sqb build".into(), + ..sync.clone() + }; + assert!( + !request_location_work(store, (&context, Some(credentials)), &repeated) + .await + .expect("a second sync while one is pending"), + "a sync already pending for the target answers as applied" + ); + let (transaction, stored) = read_location_work( + begin(store, TransactionMode::Read).await.expect("read"), + repeated.request_id, + ) + .await + .expect("read work"); + commit(transaction).await.expect("commit"); + assert!( + stored.is_none(), + "the coalesced sync request leaves no second pending work row" + ); assert!( read_person_connection_test( store, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 15c8a81788..38646757e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -5,6 +5,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cli::native_commands::main::sync_after_build::sync_after_build; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::boundary::python_calls::{call_with, within}; @@ -24,6 +25,7 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::observations::source_capture::{ arm_source_capture, take_source_capture, }; +use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::records::fields::field as native_field; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { @@ -116,7 +118,26 @@ fn run_within_limits( ("remediation", native_field(&limits, "remediation")?), ], )?; - within(&deadline, |_| run_with_providers(&context)) + let exit_code: i32 = within(&deadline, |_| run_with_providers(&context))?; + if context.worker.is_none() { + request_location_sync(&context)?; + } + Ok(exit_code) +} + +/// A signed-in person's build refreshes its configured location's warehouse copy; never fails it. +fn request_location_sync(context: &BuildContext<'_>) -> PyResult<()> { + let py: Python<'_> = context.py(); + let target: Bound<'_, PyAny> = context.get("effective_target_name")?; + if target.is_none() { + return Ok(()); + } + let target: String = target.extract()?; + let project_dir: std::path::PathBuf = context.get("effective_project_dir")?.extract()?; + if let Some(line) = py.detach(|| sync_after_build(&project_dir, &target)) { + write_stderr(py, &line)?; + } + Ok(()) } /// Run one `sqb build` request. diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index ea206b7b33..512b7a78dd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -2,9 +2,12 @@ import json import os +import sqlite3 import subprocess import sys import time +from contextlib import closing +from functools import partial from http.cookies import SimpleCookie from pathlib import Path from subprocess import Popen @@ -300,3 +303,59 @@ def verify_connection_test_transport( with pytest.raises(HTTPError) as denied: person_request(api, path, body=body | extra, session=administrator) assert denied.value.code == 422 + + +def build_stderr( + invoke: partial[subprocess.CompletedProcess[Any]], + project: tuple[str, Path], + environment: dict[str, str], + configuration: str, + extra: dict[str, str] | None = None, +) -> str: + """Build the project with `configuration` as its project file; the build always succeeds.""" + binary, project_file = project + _ = project_file.write_text(configuration) + built: subprocess.CompletedProcess[str] = invoke( + [binary, "--no-color", "build", "--project-dir", str(project_file.parent)] + + ["--target", "development"], + env=environment | (extra or {}), + ) + assert built.returncode == 0, built.stderr + return built.stderr + + +def quiet_builds( + invoke: partial[subprocess.CompletedProcess[Any]], + project: tuple[str, Path], + environment: dict[str, str], + origin: str, +) -> tuple[partial[str], str]: + """Builds print no sync line without a [control] location or before anyone signs in.""" + unconfigured: str = project[1].read_text() + build: partial[str] = partial(build_stderr, invoke, project, environment) + for control in ("", f"[control]\nserver = '{origin}'\nlocation = 'orders'\n"): + assert "Warehouse sync" not in build(unconfigured + control), control + return build, unconfigured + + +def assert_build_syncs_coalesce( + build: partial[str], project: tuple[str, str, str], control_store: Path +) -> None: + """An unknown location is named; the environment overrides the file; repeats queue nothing.""" + unconfigured, origin, requester = project + returns: str = unconfigured + f"[control]\nserver = '{origin}'\nlocation = 'returns'\n" + assert ( + f"Warehouse sync skipped: no location named 'returns' that you can view on {origin}." + in build(returns) + ) + for _ in range(4): + assert "Warehouse sync requested for location 'orders' target 'development'." in build( + returns, {"SQLBUILD_CONTROL_LOCATION": "orders"} + ) + with closing(sqlite3.connect(control_store)) as store: + queued: list[tuple[str]] = store.execute( + "SELECT requested_by FROM sqb_location_work " + "WHERE json_extract(request_json,'$.work.kind')='sync_warehouse'" + ).fetchall() + assert queued == [(requester,)], "build syncs join the person's pending request" + _ = build(unconfigured) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py index e6e33f0a39..d926447448 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py @@ -1,10 +1,8 @@ """An installed CLI inspects actual published SQL through a bounded live person session.""" import json -import sqlite3 import subprocess import sys -from contextlib import closing from functools import partial from pathlib import Path from typing import Any @@ -28,7 +26,9 @@ sign_in, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + assert_build_syncs_coalesce, provision_person, + quiet_builds, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( administer, @@ -124,6 +124,12 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori ) absent: subprocess.CompletedProcess[str] = invoke(command) assert absent.returncode != 0 and not absent.stdout and "not signed in" in absent.stderr + build, unconfigured = quiet_builds( + invoke, + (binary, worker.project / "sqlbuild_project.toml"), + client_environment | {"SQLBUILD_CONTROL_CA_CERT": str(certificate)}, + origin, + ) with pending_login( [binary, "login", *common, "--no-browser"], client_environment, @@ -168,12 +174,11 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori requested: subprocess.CompletedProcess[str] = invoke(sync) assert requested.returncode == 0, requested.stderr assert json.loads(requested.stdout) == {"applied": True} - with closing(sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3")) as store: - queued: list[tuple[str]] = store.execute( - "SELECT requested_by FROM sqb_location_work " - "WHERE json_extract(request_json,'$.work.kind')='sync_warehouse'" - ).fetchall() - assert queued == [("principal:" + person["principal_id"],)] + assert_build_syncs_coalesce( + build, + (unconfigured, origin, "principal:" + person["principal_id"]), + tmp_path / "control/.sqlbuild/control.sqlite3", + ) inspected: subprocess.CompletedProcess[str] = invoke(command) assert inspected.returncode == 0 and "Published SQL read successfully" in inspected.stderr sql: dict[str, Any] = json.loads(inspected.stdout) diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 9ebd2dad66..4f797e35bd 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -222,8 +222,18 @@ next request, including sessions created by [`sqb login`](/docs/cli/login/). ## Sync from the warehouse now A code location reads what has been built, checks and freshness back from the warehouse when it -starts and after each run the server starts has ended. It does not see a `sqb build` you run on -your own machine until then. To sync now, select **Sync now** on the location's page, or run: +starts, after each run the server starts has ended, and at least every 30 minutes. To pick up a +`sqb build` you run on your own machine straight away, name the server and location in +`sqlbuild_project.toml` and sign in with [`sqb login`](/docs/cli/login/): + +```toml +[control] +server = "https://control.example.com" +location = "orders-warehouse" +``` + +Each `sqb build` then asks that location to sync the target the build used, and prints one line +saying whether it did. You can also select **Sync now** on the location's page, or run: ```bash sqb control sync \ @@ -232,14 +242,37 @@ sqb control sync \ --target development ``` -The command prints `{"applied": true}` when the server has queued the request. The location starts -the sync within a few seconds, and the top bar shows when it last synced. A sync reads the -warehouse and never writes to it. On Snowflake, it skips state tables that haven't changed since -the last sync, so a sync with nothing new doesn't resume the warehouse. +The command prints `{"applied": true}` when the server has queued the request, and +`{"applied": false}` when a sync for that target is already waiting, which covers this request too. +The location starts the sync within a few seconds, and the top bar shows when it last synced. + +| Setting | Environment variable | Default | Meaning | +| --- | --- | --- | --- | +| `[control] server` | `SQLBUILD_CONTROL_SERVER` | none | HTTPS origin of the server, as `sqb login --server` takes it | +| `[control] location` | `SQLBUILD_CONTROL_LOCATION` | none | Location name a signed-in `sqb build` asks to sync | +| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"` | Longest a location waits between syncs; `"0"` or `"off"` turns it off | +| none | `SQLBUILD_CONTROL_CA_CERT` | none | Extra trusted PEM CA certificate for the server | + +An environment variable overrides the project file. `sqb location serve --sync-at-least-every` +overrides both. The interval uses the project's duration syntax, such as `"45m"` or `"2h"`, and +must be at least `"1m"`. Requesting a sync needs the same access as a connection test: Developer or Admin on the location and target. Viewer and Operator cannot request one. Every request is recorded in the audit log. +Details: + +- The sync after a build never changes the build's exit code and gives up after 5 seconds. It + prints nothing when `[control]` sets neither key or nobody is signed in to that server. When the + location is not one you can view, it prints `Warehouse sync skipped:` and the reason. +- Requests that arrive while a sync is running, from builds, **Sync now**, ended runs or the + interval, add one more sync after it, not one each. +- The interval restarts after every completed sync, so a location that syncs often for other + reasons never syncs extra. +- A sync reads the warehouse and never writes to it. On Snowflake, it skips state tables that + haven't changed since the last sync, so a sync with nothing new doesn't resume the warehouse. +- `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a From ebe6a3b506565d09fa6e5bfaee1049cb1b98c75d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:54:30 +0000 Subject: [PATCH 3/9] fix(location): wait for a build that holds the DuckDB warehouse file A metadata read that finds the DuckDB file locked by another process now reports a busy sync instead of a failed read. The location backs off one poll interval without shrinking its page, logs one line when the wait starts, and the console shows the table as waiting rather than failing. --- .../incremental_warehouse_history.json | 4 +- clients/typescript/src/schema.d.ts | 2 +- .../src/native_commands/_helpers/execution.rs | 26 +++++++++---- .../main/_location_metadata.rs | 11 +++++- .../src/native_commands/models.rs | 14 ++++++- .../src/platform/models.rs | 2 + .../_helpers/location_command/constants.rs | 3 ++ .../_helpers/location_command/runtime.rs | 39 ++++++++++++++----- .../publication/test_warehouse_metadata.py | 18 ++++++++- ui/src/lib/api/http-locations.test.ts | 9 ++++- ui/src/lib/api/http-locations.ts | 9 +++-- 11 files changed, 110 insertions(+), 27 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json index 5be86e4798..e7438917fa 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json +++ b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json @@ -13,7 +13,9 @@ "health": {"failure": null, "tables": ["fingerprints", "source_freshness"]}, "availability": [ {"available": false, "source": "warehouse.db", "destination": "../retained-warehouse.db", "failure": "read", "tables": ["fingerprints"]}, - {"available": true, "source": "../retained-warehouse.db", "destination": "warehouse.db", "failure": null, "tables": ["fingerprints", "source_freshness"]} + {"available": true, "source": "../retained-warehouse.db", "destination": "warehouse.db", "failure": null, "tables": ["fingerprints", "source_freshness"]}, + {"available": true, "hold_write_lock": true, "adapters": ["duckdb"], "failure": "busy", "tables": ["source_freshness"]}, + {"available": true, "adapters": ["duckdb"], "failure": null, "tables": ["fingerprints", "source_freshness"]} ], "insert_sql": [ "INSERT INTO _sqlbuild_fingerprints SELECT node_type,node_name,target_database,target_schema,target_name,'{run_id}',definition_hash,version_hash,schema_fingerprint,definition_b64,metadata_json_b64,'{observed_at}' FROM _sqlbuild_fingerprints WHERE run_id='legacy-run'", diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 9e7d3974da..2ff9f117e8 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -3277,7 +3277,7 @@ export interface components { * @description Closed failure stages keep warehouse errors and credentials out of health observations. * @enum {string} */ - WarehouseSyncFailure: "watermark" | "read" | "delivery"; + WarehouseSyncFailure: "watermark" | "read" | "delivery" | "busy"; WarehouseSyncResponse: { applied: boolean; }; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 9c10d453ec..58d8cc46c7 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -86,8 +86,8 @@ use crate::native_commands::main::_run_control::run_control; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, - LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, NativeCommand, - NativeInvocation, ProjectLocation, ScanToken, + LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, MetadataReply, + NativeCommand, NativeInvocation, ProjectLocation, ScanToken, }; use crate::native_commands::types::LocationPublication; use crate::workers::main::docker_image_sources::docker_image_sources; @@ -361,6 +361,7 @@ pub(crate) fn metadata_sync( cycle_started_at: None, safety_seconds, safety_at: Instant::now() + Duration::from_secs(safety_seconds), + busy: false, change_tokens: HashMap::new(), scan_token: None, scan_delivered: false, @@ -1214,10 +1215,16 @@ pub(crate) async fn read_metadata_page( Duration::from_secs(METADATA_POLL_SECONDS), exchange_metadata_read(reader, request), ) => match result { - Ok((reader, Ok(page))) => { + Ok((reader, Ok(MetadataReply::Page(page)))) => { + state.busy = false; state.reader = Some(reader); Ok(Some((state, page))) } + Ok((reader, Ok(MetadataReply::Busy))) => { + state.busy = true; + state.reader = Some(reader); + Ok(Some((state, None))) + } Ok((reader, Err(_))) => { stop_location_runtime(Some(reader)).await?; Ok(Some((state, None))) @@ -1263,10 +1270,7 @@ pub(crate) async fn close_metadata_reader( async fn exchange_metadata_read( mut reader: LocationCompiler, request: Vec, -) -> ( - LocationCompiler, - Result, ControlError>, -) { +) -> (LocationCompiler, Result) { let result = async { reader.stdin.write_all(&request).await?; reader.stdin.write_all(b"\n").await?; @@ -1286,7 +1290,13 @@ async fn exchange_metadata_read( "location reply exceeds its byte limit".into(), )); } - serde_json::from_str(&line) + let reply: serde_json::Value = serde_json::from_str(&line) + .map_err(|_| ControlError::Invalid("invalid location reply".into()))?; + if reply.get("busy") == Some(&serde_json::Value::Bool(true)) { + return Ok(MetadataReply::Busy); + } + serde_json::from_value(reply) + .map(MetadataReply::Page) .map_err(|_| ControlError::Invalid("invalid location reply".into())) } .await; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index ea5ddaf6f2..73039d9b37 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -78,9 +78,18 @@ pub(super) async fn sync_location_metadata( }; state = read; let Some(page) = page else { + let failure = if state.busy { + WarehouseSyncFailure::Busy + } else { + WarehouseSyncFailure::Read + }; tokio::select! { result = &mut stopped => { result?; return Ok(None); }, - () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Read))) => {}, + () = metadata_health(&state, (client, poll), (started, Some(failure))) => {}, + } + if state.busy { + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + return Ok(Some(state)); } state.retry_read(); drop(write_output(&[( diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 9a1e6ce218..1d57558f67 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -10,7 +10,8 @@ use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::LocationWorkRequest; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use sqlbuild_orchestration::warehouse_metadata::models::{ - MetadataBatch, StatisticsRead, WarehouseHistoryKind, WarehouseStateLocation, + MetadataBatch, StatisticsRead, WarehouseHistoryKind, WarehouseHistoryPage, + WarehouseStateLocation, }; use uuid::Uuid; @@ -215,6 +216,8 @@ pub(crate) struct LocationMetadataSync { pub(crate) safety_seconds: u64, /// When an idle location syncs even though no run ended and nobody asked. pub(crate) safety_at: tokio::time::Instant, + /// The last read found the warehouse file held by another process, such as a DuckDB build. + pub(crate) busy: bool, /// Per scan index: the table's change token after its last complete read. pub(crate) change_tokens: HashMap, /// The token read before the current scan's first page, recorded once the scan completes. @@ -227,6 +230,15 @@ pub(crate) struct LocationMetadataSync { pub(crate) reader: Option, } +/// The metadata reader's answer to one read. +#[derive(Debug)] +pub(crate) enum MetadataReply { + /// A page, or `None` when the read failed. + Page(Option), + /// Another process holds the warehouse file; nothing was read. + Busy, +} + /// A table's change token after a complete read, and what that read proves about the mirror. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct ScanToken { diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 541216c440..77f1d5ad40 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -47,6 +47,8 @@ pub enum WarehouseSyncFailure { Watermark, Read, Delivery, + /// Another process, such as a DuckDB build, holds the warehouse file; the sync waits for it. + Busy, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index c372c93cc4..ef6bb7fe9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -14,3 +14,6 @@ pub(crate) const COMMAND_IMPLEMENTATIONS: [(RunCommand, &str); 5] = [ (RunCommand::Audit, "run_audit_command"), (RunCommand::Freshness, "run_freshness_command"), ]; + +/// DuckDB's message when another process, such as a running build, holds the database file. +pub(crate) const DUCKDB_LOCK_CONFLICT: &str = "Conflicting lock is held"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 5df26a40db..6c4af1c5ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -12,6 +12,7 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; +use crate::bindings::_helpers::location_command::constants::DUCKDB_LOCK_CONFLICT; use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; use crate::bindings::_helpers::native_host::warehouse_metadata::{ @@ -25,6 +26,7 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul within(&redirect, |_| { let mut session = None; let mut stream = None; + let mut reported_busy = false; let mut serve = || -> PyResult { loop { let line = py.detach(|| -> io::Result> { @@ -61,15 +63,24 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul serde_json::from_value(request).map_err(|_| { PyValueError::new_err("runtime received an invalid metadata read") })?; + let mut busy = false; if session.is_none() { let arguments = location_arguments(py, Some(options.clone()))?; session = match open_history_session(py, &arguments) { Ok(opened) => Some(opened), - Err(_) => { - write_stderr( - py, - "Warehouse connection for metadata reads failed.\n", - )?; + Err(error) => { + busy = error.to_string().contains(DUCKDB_LOCK_CONFLICT); + if !busy { + write_stderr( + py, + "Warehouse connection for metadata reads failed.\n", + )?; + } else if !reported_busy { + write_stderr( + py, + "Warehouse file is in use by another process, such as a DuckDB build; the sync waits for it.\n", + )?; + } None } }; @@ -91,13 +102,21 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul { drop(close_history_session(opened, stream.take())); } - if page.is_none() { - if let Some(opened) = session.take() { - drop(close_history_session(opened, None)); + reported_busy = busy; + if busy { + serde_json::to_string(&serde_json::json!({ "busy": true })) + } else { + if page.is_none() { + if let Some(opened) = session.take() { + drop(close_history_session(opened, None)); + } + write_stderr( + py, + "Warehouse metadata read failed. No page delivered.\n", + )?; } - write_stderr(py, "Warehouse metadata read failed. No page delivered.\n")?; + serde_json::to_string(&page) } - serde_json::to_string(&page) } else { let evaluation: SensorEvaluationRequest = serde_json::from_value(request) .map_err(|_| { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 488c18d903..63cfe832be 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -60,6 +60,7 @@ append_observation_history, append_old_name_history, append_retried_fingerprints, + hold_write_lock, location_process, open_legacy_warehouse, snapshot_api, @@ -1140,8 +1141,21 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre assert location.poll() is None, log.read_text() assert time.monotonic() < deadline, log.read_text() time.sleep(0.05) + lock: Any = None for availability in fixture["availability"]: - _ = (project / availability["source"]).rename(project / availability["destination"]) + if test_case.adapter_name not in availability.get( + "adapters", [test_case.adapter_name] + ): + continue + if lock is not None: + lock.close() + lock = None + if "source" in availability: + _ = (project / availability["source"]).rename( + project / availability["destination"] + ) + if availability.get("hold_write_lock"): + lock = hold_write_lock(project / "warehouse.db") record_finished_run(control) reports = wait_for_warehouse_sync_health( api, availability, location, log, after=observed_health_at @@ -1150,6 +1164,8 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre datetime.fromisoformat(report["received_at"]) for report in reports ) assert (project / "warehouse.db").exists() == availability["available"] + if test_case.adapter_name == "duckdb": + assert log.read_text().count("Warehouse file is in use by another process") == 1 @pytest.mark.parametrize( diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 50e6d53950..120d631a1b 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -57,7 +57,7 @@ describe('toLocationDetail', () => { }); describe('warehouseSyncOf', () => { - const sync = (at: string, table: 'catalog' | 'statistics', failure: 'read' | 'delivery' | null) => ({ + const sync = (at: string, table: 'catalog' | 'statistics', failure: 'read' | 'delivery' | 'busy' | null) => ({ received_at: at, report: { event_id: at, session_id: 's1', target: 'development', health: { kind: 'warehouse_sync' as const, generation: 1, location: { schema: 'main' }, table, failure, records_sent: 0, elapsed_millis: 5 } } }); @@ -91,6 +91,13 @@ describe('warehouseSyncOf', () => { ]); }); + it('shows a build holding the warehouse file as a wait, not a failure', () => { + const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; + const observations = [sync('2026-11-08T12:03:00Z', 'catalog', 'busy'), sync('2026-11-08T12:01:00Z', 'catalog', null)]; + expect(toWarehouseConnection(location, observations).tables.map((t) => [t.failure, t.lastSyncedAt])).toEqual([['Waiting for a build that holds the warehouse file', '2026-11-08T12:01:00Z']]); + expect(warehouseSyncOf(observations)).toEqual({ lastSyncedAt: '2026-11-08T12:01:00Z', lastError: null, lastReportedAt: '2026-11-08T12:03:00Z' }); + }); + it('clears the error once the failing table syncs again', () => { expect(warehouseSyncOf([sync('2026-11-08T12:05:00Z', 'catalog', null), sync('2026-11-08T12:03:00Z', 'catalog', 'read')])?.lastError).toBeNull(); }); diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index 436cb4d6ad..b329488727 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -120,12 +120,15 @@ export function toLocationDetail(location: Location, observations: Wire['Locatio }; } -const SYNC_STAGE: Record = { +const SYNC_STAGE: Record, string> = { watermark: 'reading where the last sync stopped', read: 'reading the warehouse', delivery: 'sending results to the server' }; +/** What held up a table's newest sync; a build holding the warehouse file is a wait, not a failure. */ +const syncProblemOf = (failure: Wire['WarehouseSyncFailure']) => (failure === 'busy' ? 'Waiting for a build that holds the warehouse file' : `Failed while ${SYNC_STAGE[failure]}`); + type Sync = Wire['LocationHealthObservation'] & { report: { health: Extract } }; /** Sync reports newest first, and the newest of each table. */ @@ -145,7 +148,7 @@ export function toWarehouseConnection(location: Location, observations: Wire['Lo at: received_at, recordsSent: health.records_sent, elapsedMs: health.elapsed_millis, - failure: health.failure ? `Failed while ${SYNC_STAGE[health.failure]}` : null, + failure: health.failure ? syncProblemOf(health.failure) : null, lastSyncedAt: syncs.find((s) => s.report.health.table === health.table && !s.report.health.failure)?.received_at ?? null })) .sort((a, b) => a.table.localeCompare(b.table)); @@ -164,7 +167,7 @@ export function toWarehouseConnection(location: Location, observations: Wire['Lo export function warehouseSyncOf(observations: Wire['LocationHealthObservation'][]): WarehouseSync | null { const { syncs, newest } = syncsOf(observations); if (!syncs.length) return null; - const failing = [...newest.values()].flatMap((s) => (s.report.health.failure ? [`${s.report.health.table}: failed while ${SYNC_STAGE[s.report.health.failure]}`] : [])); + const failing = [...newest.values()].flatMap(({ report: { health } }) => (health.failure && health.failure !== 'busy' ? [`${health.table}: failed while ${SYNC_STAGE[health.failure]}`] : [])); return { lastSyncedAt: syncs.find((s) => !s.report.health.failure)?.received_at ?? null, lastError: failing.length ? failing.sort().join('\n') : null, From 11842a933df6f0259c34365350de31f3d29f4464 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:56:32 +0000 Subject: [PATCH 4/9] test(janitor): keep the direct state pruning timeline in DuckShop --- .../orchestration/direct_state_pruning.json | 5 +++++ .../sqlbuild/cli/commands/main/janitor/helpers.py | 14 ++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/direct_state_pruning.json diff --git a/benchmarks/duckshop/edge_cases/orchestration/direct_state_pruning.json b/benchmarks/duckshop/edge_cases/orchestration/direct_state_pruning.json new file mode 100644 index 0000000000..eda7dc2280 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/direct_state_pruning.json @@ -0,0 +1,5 @@ +{ + "description": "Direct state history for janitor pruning that keeps the newest two rows per key. run_001 and run_002 share an hour at the keep boundary, so the tie breaks on the newer run or result id.", + "runs": [["run_000", 10], ["run_001", 12], ["run_002", 12], ["run_003", 13]], + "audit_binding_keys": ["orders.valid_orders", "customers.unique_ids"] +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index 018c5da813..b6ca3a8f9d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -11,6 +11,7 @@ from pathlib import Path from textwrap import dedent from types import MappingProxyType +from typing import Any from sqlbuild._native import ( Fingerprint, @@ -145,7 +146,9 @@ def create_janitor_scenario_relations(*, db_path: Path) -> None: connection.close() -JANITOR_AUDIT_BINDING_KEYS: tuple[str, ...] = ("orders.valid_orders", "customers.unique_ids") +DIRECT_STATE_PRUNING: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/direct_state_pruning.json").read_text() +) def audit_record(*, binding_key: str, run_id: str, occurred_at: datetime) -> AuditResultRecord: @@ -202,12 +205,7 @@ def create_direct_state_history(*, db_path: Path, schema: str = "main") -> None: try: run_id: str observed_hour: int - for run_id, observed_hour in ( - ("run_000", 10), - ("run_001", 12), - ("run_002", 12), - ("run_003", 13), - ): + for run_id, observed_hour in DIRECT_STATE_PRUNING["runs"]: _ = write_fingerprint( connection=connection, execute=lambda *, connection, sql: connection.execute(sql), @@ -240,7 +238,7 @@ def create_direct_state_history(*, db_path: Path, schema: str = "main") -> None: run_id=run_id, occurred_at=datetime(2026, 1, 15, observed_hour, 10, 0, tzinfo=UTC), ) - for binding_key in JANITOR_AUDIT_BINDING_KEYS + for binding_key in DIRECT_STATE_PRUNING["audit_binding_keys"] ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, From 6f2cbd3028ec2b50809580ab96ef89486589e34d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:58:43 +0000 Subject: [PATCH 5/9] refactor(config): load single project sections through one helper --- .../src/project/_helpers/documents.rs | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 7acfc90165..0521263158 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -427,28 +427,31 @@ pub(crate) fn local_sql_analysis_failure( ) } -/// `[git] base_ref` of `sqlbuild_project.toml`, validated as project loading validates it. -pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { +/// One section of `sqlbuild_project.toml`, validated as project loading validates it. +fn project_section( + paths: &ConfigPaths, + section: &str, + load: fn(Option<&ConfigValue>, ConfigFile) -> Loaded, +) -> Result { let file = resolve(paths, false)?; let table = load_toml_mapping(paths, file)?; let payload = table.as_map().unwrap_or_default(); - Ok(load_git(get(payload, "git"), PROJECT_FILE)?) + Ok(load(get(payload, section), PROJECT_FILE)?) +} + +/// `[git] base_ref` of `sqlbuild_project.toml`. +pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { + project_section(paths, "git", load_git) } -/// `[control]` of `sqlbuild_project.toml`, validated as project loading validates it. +/// `[control]` of `sqlbuild_project.toml`. pub(crate) fn control_config(paths: &ConfigPaths) -> Result { - let file = resolve(paths, false)?; - let table = load_toml_mapping(paths, file)?; - let payload = table.as_map().unwrap_or_default(); - Ok(load_control(get(payload, "control"), PROJECT_FILE)?) + project_section(paths, "control", load_control) } -/// `[review]` of `sqlbuild_project.toml`, validated as project loading validates it. +/// `[review]` of `sqlbuild_project.toml`. pub(crate) fn review_config(paths: &ConfigPaths) -> Result { - let file = resolve(paths, false)?; - let table = load_toml_mapping(paths, file)?; - let payload = table.as_map().unwrap_or_default(); - Ok(load_review(get(payload, "review"), PROJECT_FILE)?) + project_section(paths, "review", load_review) } /// One warning line per configuration file whose `[settings]` still sets `column_contract_mode`. From f14e354bd7a8f3f9b9869743896ed25821dd3f9b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:00:55 +0000 Subject: [PATCH 6/9] docs(skills): regenerate the server reference --- .../sqlbuild/references/docs/cli/server.md | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 7ca1ede23e..06b138a6f6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -236,8 +236,18 @@ next request, including sessions created by [`sqb login`](login.md). ## Sync from the warehouse now A code location reads what has been built, checks and freshness back from the warehouse when it -starts and after each run the server starts has ended. It does not see a `sqb build` you run on -your own machine until then. To sync now, select **Sync now** on the location's page, or run: +starts, after each run the server starts has ended, and at least every 30 minutes. To pick up a +`sqb build` you run on your own machine straight away, name the server and location in +`sqlbuild_project.toml` and sign in with [`sqb login`](login.md): + +```toml +[control] +server = "https://control.example.com" +location = "orders-warehouse" +``` + +Each `sqb build` then asks that location to sync the target the build used, and prints one line +saying whether it did. You can also select **Sync now** on the location's page, or run: ```bash sqb control sync \ @@ -246,14 +256,37 @@ sqb control sync \ --target development ``` -The command prints `{"applied": true}` when the server has queued the request. The location starts -the sync within a few seconds, and the top bar shows when it last synced. A sync reads the -warehouse and never writes to it. On Snowflake, it skips state tables that haven't changed since -the last sync, so a sync with nothing new doesn't resume the warehouse. +The command prints `{"applied": true}` when the server has queued the request, and +`{"applied": false}` when a sync for that target is already waiting, which covers this request too. +The location starts the sync within a few seconds, and the top bar shows when it last synced. + +| Setting | Environment variable | Default | Meaning | +| --- | --- | --- | --- | +| `[control] server` | `SQLBUILD_CONTROL_SERVER` | none | HTTPS origin of the server, as `sqb login --server` takes it | +| `[control] location` | `SQLBUILD_CONTROL_LOCATION` | none | Location name a signed-in `sqb build` asks to sync | +| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"` | Longest a location waits between syncs; `"0"` or `"off"` turns it off | +| none | `SQLBUILD_CONTROL_CA_CERT` | none | Extra trusted PEM CA certificate for the server | + +An environment variable overrides the project file. `sqb location serve --sync-at-least-every` +overrides both. The interval uses the project's duration syntax, such as `"45m"` or `"2h"`, and +must be at least `"1m"`. Requesting a sync needs the same access as a connection test: Developer or Admin on the location and target. Viewer and Operator cannot request one. Every request is recorded in the audit log. +Details: + +- The sync after a build never changes the build's exit code and gives up after 5 seconds. It + prints nothing when `[control]` sets neither key or nobody is signed in to that server. When the + location is not one you can view, it prints `Warehouse sync skipped:` and the reason. +- Requests that arrive while a sync is running, from builds, **Sync now**, ended runs or the + interval, add one more sync after it, not one each. +- The interval restarts after every completed sync, so a location that syncs often for other + reasons never syncs extra. +- A sync reads the warehouse and never writes to it. On Snowflake, it skips state tables that + haven't changed since the last sync, so a sync with nothing new doesn't resume the warehouse. +- `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a From fae7457d49b41c412349df4025a5731984af073f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:18:59 +0000 Subject: [PATCH 7/9] fix(location): clear busy before each read and bound the post-build sync on its own thread A failed read after a busy reply no longer stays on the busy path, the post-build request runs entirely within its bound (keyring, DNS and server), its errors never change the build's result, and the console shows a waiting table in the warning tone. --- .../src/native_commands/_helpers/execution.rs | 2 +- .../src/person_login/main/sync_after_build.rs | 31 ++++++++++++------- .../_helpers/build_command/command.rs | 2 +- ui/src/lib/api/http-locations.test.ts | 2 +- ui/src/lib/api/http-locations.ts | 1 + ui/src/lib/api/mock/platform.ts | 2 +- ui/src/lib/api/types/platform.ts | 4 ++- ui/src/routes/connections/+page.svelte | 2 +- 8 files changed, 29 insertions(+), 17 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 58d8cc46c7..8fd1d0cae7 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -1186,6 +1186,7 @@ pub(crate) async fn read_metadata_page( mut state: LocationMetadataSync, stopped: impl Future>, ) -> Result)>, ControlError> { + state.busy = false; if let Some(statistics) = &state.statistics && statistics.names.is_empty() { @@ -1216,7 +1217,6 @@ pub(crate) async fn read_metadata_page( exchange_metadata_read(reader, request), ) => match result { Ok((reader, Ok(MetadataReply::Page(page)))) => { - state.busy = false; state.reader = Some(reader); Ok(Some((state, page))) } diff --git a/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs b/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs index 52f08d76c7..d88756b8fe 100644 --- a/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs +++ b/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs @@ -1,6 +1,7 @@ //! A signed-in `sqb build` asks its configured location to refresh the warehouse copy, best-effort. use std::path::{Path, PathBuf}; +use std::sync::mpsc::RecvTimeoutError; use std::time::Duration; use sqlbuild_config::project::main::load_control_config::load_control_config; @@ -20,7 +21,25 @@ fn setting(name: &str, configured: Option) -> Option { } /// One stderr line about the sync, or `None` when no location is configured or nobody is signed in. +/// +/// The request runs on its own thread, so a slow keyring, resolver or server cannot hold the build's +/// exit past the bound; an abandoned request ends with the process. pub(crate) fn sync_after_build(project_dir: &Path, target: &str) -> Option { + let (sender, receiver) = std::sync::mpsc::channel(); + let (project_dir, target) = (project_dir.to_path_buf(), target.to_owned()); + drop(std::thread::spawn(move || { + drop(sender.send(request_sync(&project_dir, &target))); + })); + match receiver.recv_timeout(Duration::from_secs(BUILD_SYNC_SECONDS)) { + Ok(line) => line, + Err(RecvTimeoutError::Timeout) => Some(format!( + "Warehouse sync skipped: the request did not finish within {BUILD_SYNC_SECONDS} seconds.\n" + )), + Err(RecvTimeoutError::Disconnected) => None, + } +} + +fn request_sync(project_dir: &Path, target: &str) -> Option { let configured = load_control_config(&project_config_paths(project_dir)).ok()?; let server = setting(CONTROL_SERVER_ENV, configured.server); let location = setting(CONTROL_LOCATION_ENV, configured.location); @@ -81,17 +100,7 @@ pub(crate) fn sync_after_build(project_dir: &Path, target: &str) -> Option Some(line), Err(error) => skipped(error), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 38646757e2..6d70b6b2e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -120,7 +120,7 @@ fn run_within_limits( )?; let exit_code: i32 = within(&deadline, |_| run_with_providers(&context))?; if context.worker.is_none() { - request_location_sync(&context)?; + let _ = request_location_sync(&context); } Ok(exit_code) } diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 120d631a1b..0a4342f570 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -94,7 +94,7 @@ describe('warehouseSyncOf', () => { it('shows a build holding the warehouse file as a wait, not a failure', () => { const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; const observations = [sync('2026-11-08T12:03:00Z', 'catalog', 'busy'), sync('2026-11-08T12:01:00Z', 'catalog', null)]; - expect(toWarehouseConnection(location, observations).tables.map((t) => [t.failure, t.lastSyncedAt])).toEqual([['Waiting for a build that holds the warehouse file', '2026-11-08T12:01:00Z']]); + expect(toWarehouseConnection(location, observations).tables.map((t) => [t.failure, t.waiting, t.lastSyncedAt])).toEqual([['Waiting for a build that holds the warehouse file', true, '2026-11-08T12:01:00Z']]); expect(warehouseSyncOf(observations)).toEqual({ lastSyncedAt: '2026-11-08T12:01:00Z', lastError: null, lastReportedAt: '2026-11-08T12:03:00Z' }); }); diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index b329488727..357b298f52 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -149,6 +149,7 @@ export function toWarehouseConnection(location: Location, observations: Wire['Lo recordsSent: health.records_sent, elapsedMs: health.elapsed_millis, failure: health.failure ? syncProblemOf(health.failure) : null, + waiting: health.failure === 'busy', lastSyncedAt: syncs.find((s) => s.report.health.table === health.table && !s.report.health.failure)?.received_at ?? null })) .sort((a, b) => a.table.localeCompare(b.table)); diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index 944f0e0cbb..223fae92da 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -702,7 +702,7 @@ export class MockPlatformApi implements PlatformApi { target: c.target, stateLocation: { database: typeof c.values.database === 'string' ? c.values.database : null, schema: 'sqlbuild' }, lastTest: this.#connectionTests.get(`${l.id}/${c.name}`) ?? c.lastTest, - tables: ['audit_results', 'catalog', 'fingerprints', 'statistics'].map((table, i) => ({ table, at: synced, recordsSent: 12 * (i + 1), elapsedMs: 180 + 40 * i, failure: null, lastSyncedAt: synced })) + tables: ['audit_results', 'catalog', 'fingerprints', 'statistics'].map((table, i) => ({ table, at: synced, recordsSent: 12 * (i + 1), elapsedMs: 180 + 40 * i, failure: null, waiting: false, lastSyncedAt: synced })) })) ); return this.#ctx.reply(structuredClone(connections)); diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index b374e8b8b0..087d374610 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -233,8 +233,10 @@ export type WarehouseTableSync = { at: Iso; recordsSent: number; elapsedMs: number; - /** What the newest sync was doing when it failed; null when it succeeded. */ + /** What the newest sync was doing when it failed, or what it is waiting for; null when it succeeded. */ failure: string | null; + /** The newest sync is waiting for a build that holds the warehouse file, which is not a failure. */ + waiting: boolean; /** The newest sync of the table that succeeded. */ lastSyncedAt: Iso | null; }; diff --git a/ui/src/routes/connections/+page.svelte b/ui/src/routes/connections/+page.svelte index 00c18a612b..85b11f1237 100644 --- a/ui/src/routes/connections/+page.svelte +++ b/ui/src/routes/connections/+page.svelte @@ -44,7 +44,7 @@ {t.table} - {#if t.failure}{:else}{/if} + {#if t.failure}{:else}{/if} {formatNumber(t.recordsSent)} From 4e98acae3c0d96bfe93c683134be34bba948ec39 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:48:01 +0000 Subject: [PATCH 8/9] refactor(location): split post-build sync phases and keep tests branch-free Satisfies the repository's structure and test rules: the build sync's phases live in a helper module, the binding imports it directly, environment reads match their errors, and the safety and lock tests use case data instead of branches. --- .../incremental_warehouse_history.json | 13 +- crates/sqlbuild-cli/src/lib.rs | 2 +- .../src/native_commands/_helpers/execution.rs | 14 ++- .../main/_location_metadata.rs | 48 +++++--- .../src/native_commands/main/mod.rs | 1 - .../native_commands/main/sync_after_build.rs | 9 -- .../src/native_commands/models.rs | 3 +- .../tests/test_location_metadata.rs | 114 ++++++++++-------- .../src/native_commands/tests/test_types.rs | 15 ++- .../src/person_login/_helpers/build_sync.rs | 96 +++++++++++++++ .../src/person_login/_helpers/mod.rs | 1 + .../sqlbuild-cli/src/person_login/main/mod.rs | 2 +- .../src/person_login/main/sync_after_build.rs | 90 ++------------ crates/sqlbuild-cli/src/person_login/mod.rs | 2 +- .../sqlbuild-cli/src/person_login/models.rs | 6 +- .../src/project/_helpers/sections.rs | 9 +- .../platform/main/request_location_work.rs | 3 +- .../_helpers/build_command/command.rs | 2 +- .../publication/test_warehouse_metadata.py | 39 +++--- 19 files changed, 270 insertions(+), 199 deletions(-) delete mode 100644 crates/sqlbuild-cli/src/native_commands/main/sync_after_build.rs create mode 100644 crates/sqlbuild-cli/src/person_login/_helpers/build_sync.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json index e7438917fa..ae0af98857 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json +++ b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json @@ -13,10 +13,17 @@ "health": {"failure": null, "tables": ["fingerprints", "source_freshness"]}, "availability": [ {"available": false, "source": "warehouse.db", "destination": "../retained-warehouse.db", "failure": "read", "tables": ["fingerprints"]}, - {"available": true, "source": "../retained-warehouse.db", "destination": "warehouse.db", "failure": null, "tables": ["fingerprints", "source_freshness"]}, - {"available": true, "hold_write_lock": true, "adapters": ["duckdb"], "failure": "busy", "tables": ["source_freshness"]}, - {"available": true, "adapters": ["duckdb"], "failure": null, "tables": ["fingerprints", "source_freshness"]} + {"available": true, "source": "../retained-warehouse.db", "destination": "warehouse.db", "failure": null, "tables": ["fingerprints", "source_freshness"]} ], + "held_write_lock": { + "duckdb": [ + { + "held": {"failure": "busy", "tables": ["source_freshness"]}, + "released": {"failure": null, "tables": ["fingerprints", "source_freshness"]} + } + ], + "sqlite": [] + }, "insert_sql": [ "INSERT INTO _sqlbuild_fingerprints SELECT node_type,node_name,target_database,target_schema,target_name,'{run_id}',definition_hash,version_hash,schema_fingerprint,definition_b64,metadata_json_b64,'{observed_at}' FROM _sqlbuild_fingerprints WHERE run_id='legacy-run'", "INSERT INTO _sqlbuild_source_freshness SELECT source_name,target_database,target_schema,target_name,'{run_id}',strategy,value_kind,data_version,data_version_hash,'{observed_at}' FROM _sqlbuild_source_freshness WHERE run_id='legacy-run'" diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 6fd859c0bb..6e85c29d77 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -10,6 +10,6 @@ pub mod diff_report; pub mod dispatch; pub mod launcher; pub mod native_commands; -pub(crate) mod person_login; +pub mod person_login; pub mod replay; pub(crate) mod workers; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 8fd1d0cae7..efec73492f 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -701,9 +701,18 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result Some(value), + Err(std::env::VarError::NotPresent) => None, + Err(std::env::VarError::NotUnicode(_)) => { + return Err(ControlError::Invalid(format!( + "{SAFETY_SYNC_ENV} must be text" + ))); + } + }; let safety_sync_seconds = safety_sync_seconds( text(namespace, "location_sync_at_least_every").as_deref(), - std::env::var(SAFETY_SYNC_ENV).ok().as_deref(), + environment.as_deref(), || { load_control_config(&project_config_paths(&project_dir)).map_err(|failure| { failure.into_config_failure().map_or_else( @@ -725,8 +734,7 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result, environment: Option<&str>, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index 73039d9b37..d5ee1a5c47 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -78,25 +78,7 @@ pub(super) async fn sync_location_metadata( }; state = read; let Some(page) = page else { - let failure = if state.busy { - WarehouseSyncFailure::Busy - } else { - WarehouseSyncFailure::Read - }; - tokio::select! { - result = &mut stopped => { result?; return Ok(None); }, - () = metadata_health(&state, (client, poll), (started, Some(failure))) => {}, - } - if state.busy { - state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); - return Ok(Some(state)); - } - state.retry_read(); - drop(write_output(&[( - Stream::Stderr, - "Warehouse history read failed; retaining mirror and retrying a bounded page.\n", - )])); - return Ok(Some(state)); + return failed_read(state, (client, poll), started, stopped.as_mut()).await; }; if location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed { @@ -132,6 +114,34 @@ pub(super) async fn sync_location_metadata( finish_metadata_page(acknowledge_metadata_page(state)).await } +/// A busy warehouse file waits a poll interval; any other failed read retries a smaller page. +async fn failed_read( + mut state: LocationMetadataSync, + (client, poll): (&LocationWorkClient, &LocationWorkPoll), + started: Instant, + stopped: impl Future>, +) -> Result, ControlError> { + let failure = if state.busy { + WarehouseSyncFailure::Busy + } else { + WarehouseSyncFailure::Read + }; + tokio::select! { + result = stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, Some(failure))) => {}, + } + if state.busy { + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + return Ok(Some(state)); + } + state.retry_read(); + drop(write_output(&[( + Stream::Stderr, + "Warehouse history read failed; retaining mirror and retrying a bounded page.\n", + )])); + Ok(Some(state)) +} + fn retry_metadata( mut state: LocationMetadataSync, error: ClientError, diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 8da5144bc0..c5ee3267d2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -18,4 +18,3 @@ pub(crate) mod run_location_invocation; pub(crate) mod run_login_invocation; pub mod run_native_invocation; pub(crate) mod run_person_invocation; -pub mod sync_after_build; diff --git a/crates/sqlbuild-cli/src/native_commands/main/sync_after_build.rs b/crates/sqlbuild-cli/src/native_commands/main/sync_after_build.rs deleted file mode 100644 index b9dfd37511..0000000000 --- a/crates/sqlbuild-cli/src/native_commands/main/sync_after_build.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! After a direct build, ask the configured location to sync the target the build used. - -use std::path::Path; - -/// One stderr line about the sync, or `None` when no location is configured or nobody is signed in. -/// Never fails: every problem becomes a skipped line, and the request is bounded to a few seconds. -pub fn sync_after_build(project_dir: &Path, target: &str) -> Option { - crate::person_login::main::sync_after_build::sync_after_build(project_dir, target) -} diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 1d57558f67..f242f0d0f6 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -258,8 +258,7 @@ impl LocationMetadataSync { } } - /// Whether an idle location starts a sync: never synced, asked to, a run ended since the last - /// one began, or the safety interval passed. Each is one outstanding sync, never a second one. + /// Whether an idle location starts its one outstanding sync: first, requested, after a run, or safety. pub(crate) fn sync_due(&self, runs_ended_at: Option>) -> bool { let ran = match (self.cycle_started_at, runs_ended_at) { (None, _) => true, diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index e6a3d35aff..83282fde2c 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -5,12 +5,14 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ use chrono::Utc; use sqlbuild_config::project::models::ControlConfig; +use std::time::Duration; use tokio::time::Instant; use crate::native_commands::_helpers::execution::{metadata_sync, safety_sync_seconds}; use crate::native_commands::tests::helpers::{acknowledge_complete_pages, one_location}; use crate::native_commands::tests::test_types::{ - MetadataRetryTestCase, SafetySyncSettingTestCase, SyncCoalescingTestCase, UnchangedScanTestCase, + MetadataRetryTestCase, SafetySyncIdleTestCase, SafetySyncSettingTestCase, + SyncCoalescingTestCase, UnchangedScanTestCase, }; #[test] @@ -176,7 +178,7 @@ fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync requests: 4, run_ends_during_sync: true, safety_seconds: 1800, - safety_passes_during_sync: false, + safety_due_after_seconds: 3600, expected_follow_ups: 1, }, SyncCoalescingTestCase { @@ -184,7 +186,7 @@ fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync requests: 2, run_ends_during_sync: false, safety_seconds: 1800, - safety_passes_during_sync: true, + safety_due_after_seconds: 0, expected_follow_ups: 1, }, SyncCoalescingTestCase { @@ -192,7 +194,7 @@ fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync requests: 0, run_ends_during_sync: false, safety_seconds: 1800, - safety_passes_during_sync: true, + safety_due_after_seconds: 0, expected_follow_ups: 0, }, SyncCoalescingTestCase { @@ -200,7 +202,7 @@ fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync requests: 0, run_ends_during_sync: false, safety_seconds: 1800, - safety_passes_during_sync: false, + safety_due_after_seconds: 3600, expected_follow_ups: 0, }, ]; @@ -219,23 +221,26 @@ fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync let runs_ended_at = test_case .run_ends_during_sync .then(|| started + chrono::Duration::seconds(1)); - if test_case.safety_passes_during_sync { - state.safety_at = Instant::now(); - } + state.safety_at = Instant::now() + Duration::from_secs(test_case.safety_due_after_seconds); state = acknowledge_complete_pages(state, WarehouseHistoryKind::ALL.len()); assert!( state.idle, "{}: the first sync completed", test_case.description ); - let mut follow_ups = 0; - for poll in 1..=5 { - if state.sync_due(runs_ended_at) { - follow_ups += 1; - state.begin_sync(started + chrono::Duration::seconds(10 * poll)); - state = acknowledge_complete_pages(state, WarehouseHistoryKind::ALL.len()); - } - } + let mut polled = Some(state); + let follow_ups = (1..=5) + .take_while(|poll| { + let mut current = polled.take().expect("one state between polls"); + let due = current.sync_due(runs_ended_at); + current.begin_sync(started + chrono::Duration::seconds(10 * poll)); + polled = Some(acknowledge_complete_pages( + current, + WarehouseHistoryKind::ALL.len(), + )); + due + }) + .count(); assert_eq!( follow_ups, test_case.expected_follow_ups, "{}", @@ -246,19 +251,38 @@ fn given_signals_during_a_sync_when_it_completes_then_exactly_one_follow_up_sync #[test] fn given_an_idle_location_when_the_safety_interval_passes_then_it_syncs_unless_turned_off() { + let test_cases = [ + SafetySyncIdleTestCase { + description: "the interval passed while idle", + safety_seconds: 1800, + safety_due_after_seconds: 0, + expected_sync_due: true, + }, + SafetySyncIdleTestCase { + description: "the interval has not passed", + safety_seconds: 1800, + safety_due_after_seconds: 3600, + expected_sync_due: false, + }, + SafetySyncIdleTestCase { + description: "0 turns safety syncs off", + safety_seconds: 0, + safety_due_after_seconds: 0, + expected_sync_due: false, + }, + ]; let started = Utc::now(); - for (description, safety_seconds, passed, expected) in [ - ("the interval passed while idle", 1800, true, true), - ("the interval has not passed", 1800, false, false), - ("0 turns safety syncs off", 0, true, false), - ] { - let mut state = metadata_sync((one_location(), 1), (3600, safety_seconds)); + for test_case in test_cases { + let mut state = metadata_sync((one_location(), 1), (3600, test_case.safety_seconds)); state.begin_sync(started); state = acknowledge_complete_pages(state, WarehouseHistoryKind::ALL.len()); - if passed { - state.safety_at = Instant::now(); - } - assert_eq!(state.sync_due(None), expected, "{description}"); + state.safety_at = Instant::now() + Duration::from_secs(test_case.safety_due_after_seconds); + assert_eq!( + state.sync_due(None), + test_case.expected_sync_due, + "{}", + test_case.description + ); } } @@ -270,50 +294,52 @@ fn given_safety_sync_settings_when_resolving_then_flag_beats_environment_beats_p flag: None, environment: None, project: None, - expected: Ok(1800), + expected_seconds: Ok(1800), }, SafetySyncSettingTestCase { description: "the project file sets it", flag: None, environment: None, project: Some(3600), - expected: Ok(3600), + expected_seconds: Ok(3600), }, SafetySyncSettingTestCase { description: "the environment overrides the project file", flag: None, environment: Some("2h"), project: Some(3600), - expected: Ok(7200), + expected_seconds: Ok(7200), }, SafetySyncSettingTestCase { description: "a blank environment value falls through to the project file", flag: None, environment: Some(" "), project: Some(3600), - expected: Ok(3600), + expected_seconds: Ok(3600), }, SafetySyncSettingTestCase { description: "the flag overrides the environment", flag: Some("off"), environment: Some("2h"), project: Some(3600), - expected: Ok(0), + expected_seconds: Ok(0), }, SafetySyncSettingTestCase { description: "a flag under a minute is refused", flag: Some("30s"), environment: None, project: None, - expected: Err("--sync-at-least-every must be a duration of at least '1m'"), + expected_seconds: Err( + "--sync-at-least-every must be a duration of at least '1m' such as '30m', or '0' or 'off' to turn safety syncs off", + ), }, SafetySyncSettingTestCase { description: "an unparseable environment value is refused", flag: None, environment: Some("-5m"), project: None, - expected: Err( - "SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY must be a duration of at least '1m'", + expected_seconds: Err( + "SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY must be a duration of at least '1m' such as '30m', or '0' or 'off' to turn safety syncs off", ), }, ]; @@ -324,19 +350,11 @@ fn given_safety_sync_settings_when_resolving_then_flag_beats_environment_beats_p ..ControlConfig::default() }) }); - match (resolved, test_case.expected) { - (Ok(seconds), Ok(expected)) => { - assert_eq!(seconds, expected, "{}", test_case.description); - } - (Err(message), Err(fragment)) => assert!( - message.starts_with(fragment), - "{}: {message}", - test_case.description - ), - (resolved, expected) => panic!( - "{}: resolved {resolved:?}, expected {expected:?}", - test_case.description - ), - } + assert_eq!( + resolved, + test_case.expected_seconds.map_err(str::to_owned), + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 2c7a10743c..8282a11eaf 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -119,8 +119,8 @@ pub(super) struct SyncCoalescingTestCase { pub(super) run_ends_during_sync: bool, /// Seconds between safety syncs; `0` turns them off. pub(super) safety_seconds: u64, - /// The safety interval passes while the sync is in progress. - pub(super) safety_passes_during_sync: bool, + /// Seconds until the safety interval passes, set while the sync runs; `0` passes during it. + pub(super) safety_due_after_seconds: u64, pub(super) expected_follow_ups: usize, } @@ -129,5 +129,14 @@ pub(super) struct SafetySyncSettingTestCase { pub(super) flag: Option<&'static str>, pub(super) environment: Option<&'static str>, pub(super) project: Option, - pub(super) expected: Result, + pub(super) expected_seconds: Result, +} + +/// An idle location after one complete sync, and whether its safety interval makes a sync due. +pub(super) struct SafetySyncIdleTestCase { + pub(super) description: &'static str, + /// Seconds between safety syncs; `0` turns them off. + pub(super) safety_seconds: u64, + pub(super) safety_due_after_seconds: u64, + pub(super) expected_sync_due: bool, } diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/build_sync.rs b/crates/sqlbuild-cli/src/person_login/_helpers/build_sync.rs new file mode 100644 index 0000000000..300fe441e4 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/build_sync.rs @@ -0,0 +1,96 @@ +//! The post-build sync's phases: read the configured location, sign in, and ask for the sync. + +use std::env::VarError; +use std::path::{Path, PathBuf}; + +use sqlbuild_config::project::main::load_control_config::load_control_config; +use sqlbuild_config::project::main::project_config_paths::project_config_paths; +use sqlbuild_orchestration::api::main::parse_origin::parse_origin; + +use crate::person_login::_helpers::{credentials, transport}; +use crate::person_login::constants::{ + CONTROL_CA_CERT_ENV, CONTROL_LOCATION_ENV, CONTROL_SERVER_ENV, +}; +use crate::person_login::models::{PersonClient, PersonCredential}; + +/// The configured server and location name, from the environment or `[control]`; `None` when neither is set. +pub(crate) fn configured_location( + project_dir: &Path, +) -> Result, std::io::Error> { + let configured = load_control_config(&project_config_paths(project_dir)) + .map_err(|_| std::io::Error::other("sqlbuild_project.toml [control] could not be read."))?; + match ( + setting(CONTROL_SERVER_ENV, configured.server)?, + setting(CONTROL_LOCATION_ENV, configured.location)?, + ) { + (Some(server), Some(location)) => Ok(Some((server, location))), + (None, None) => Ok(None), + _ => Err(std::io::Error::other( + "set both [control] server and location, or neither.", + )), + } +} + +fn setting(name: &str, configured: Option) -> Result, std::io::Error> { + match std::env::var(name) { + Ok(value) if !value.trim().is_empty() => Ok(Some(value)), + Ok(_) | Err(VarError::NotPresent) => Ok(configured), + Err(VarError::NotUnicode(_)) => Err(std::io::Error::other(format!("{name} must be text."))), + } +} + +/// Signs in with the stored session and asks for the sync; `None` when nobody is signed in to the server. +pub(crate) fn sign_in_and_sync( + (server, location): (String, String), + target: &str, +) -> Result, std::io::Error> { + let origin = parse_origin(&server, "CLI").map_err(std::io::Error::other)?; + let Some(credential) = credentials::load(&origin)? else { + return Ok(None); + }; + let ca = std::env::var_os(CONTROL_CA_CERT_ENV).map(PathBuf::from); + let client = transport::connect(&origin, ca.as_deref())?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime + .block_on(sync_location_target( + (&client, &credential), + (&location, target), + &origin, + )) + .map(Some) +} + +async fn sync_location_target( + (client, credential): (&PersonClient, &PersonCredential), + (location, target): (&str, &str), + origin: &str, +) -> Result { + let targets = transport::location_targets(client, credential).await?; + let Some(found) = targets + .iter() + .find(|candidate| candidate.name == location && candidate.target == target) + else { + let named = targets.iter().any(|candidate| candidate.name == location); + return Ok(if named { + format!( + "Warehouse sync skipped: location '{location}' has no target '{target}' you can view on {origin}.\n" + ) + } else { + format!( + "Warehouse sync skipped: no location named '{location}' that you can view on {origin}.\n" + ) + }); + }; + transport::warehouse_sync( + client, + credential, + (found.location_id, target), + "Requested after sqb build", + ) + .await?; + Ok(format!( + "Warehouse sync requested for location '{location}' target '{target}'.\n" + )) +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs b/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs index 11f145944a..3cbc2b7b34 100644 --- a/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(super) mod build_sync; pub(super) mod credentials; pub(super) mod flow; pub(super) mod transport; diff --git a/crates/sqlbuild-cli/src/person_login/main/mod.rs b/crates/sqlbuild-cli/src/person_login/main/mod.rs index 89c8db7e03..ce45b29160 100644 --- a/crates/sqlbuild-cli/src/person_login/main/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/main/mod.rs @@ -1,4 +1,4 @@ pub(crate) mod read_node_sql; pub(crate) mod request_warehouse_sync; pub(crate) mod run_login; -pub(crate) mod sync_after_build; +pub mod sync_after_build; diff --git a/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs b/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs index d88756b8fe..676fa0721f 100644 --- a/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs +++ b/crates/sqlbuild-cli/src/person_login/main/sync_after_build.rs @@ -1,30 +1,14 @@ //! A signed-in `sqb build` asks its configured location to refresh the warehouse copy, best-effort. -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::mpsc::RecvTimeoutError; use std::time::Duration; -use sqlbuild_config::project::main::load_control_config::load_control_config; -use sqlbuild_config::project::main::project_config_paths::project_config_paths; -use sqlbuild_orchestration::api::main::parse_origin::parse_origin; - -use crate::person_login::_helpers::{credentials, transport}; -use crate::person_login::constants::{ - BUILD_SYNC_SECONDS, CONTROL_CA_CERT_ENV, CONTROL_LOCATION_ENV, CONTROL_SERVER_ENV, -}; - -fn setting(name: &str, configured: Option) -> Option { - std::env::var(name) - .ok() - .filter(|value| !value.trim().is_empty()) - .or(configured) -} +use crate::person_login::_helpers::build_sync::{configured_location, sign_in_and_sync}; +use crate::person_login::constants::BUILD_SYNC_SECONDS; /// One stderr line about the sync, or `None` when no location is configured or nobody is signed in. -/// -/// The request runs on its own thread, so a slow keyring, resolver or server cannot hold the build's -/// exit past the bound; an abandoned request ends with the process. -pub(crate) fn sync_after_build(project_dir: &Path, target: &str) -> Option { +pub fn sync_after_build(project_dir: &Path, target: &str) -> Option { let (sender, receiver) = std::sync::mpsc::channel(); let (project_dir, target) = (project_dir.to_path_buf(), target.to_owned()); drop(std::thread::spawn(move || { @@ -40,69 +24,13 @@ pub(crate) fn sync_after_build(project_dir: &Path, target: &str) -> Option Option { - let configured = load_control_config(&project_config_paths(project_dir)).ok()?; - let server = setting(CONTROL_SERVER_ENV, configured.server); - let location = setting(CONTROL_LOCATION_ENV, configured.location); - let (server, location) = match (server, location) { - (Some(server), Some(location)) => (server, location), - (None, None) => return None, - _ => { - return Some( - "Warehouse sync skipped: set both [control] server and location, or neither.\n" - .into(), - ); - } - }; - let skipped = |error: std::io::Error| Some(format!("Warehouse sync skipped: {error}\n")); - let origin = match parse_origin(&server, "CLI") { - Ok(origin) => origin, - Err(error) => return skipped(std::io::Error::other(error)), - }; - let credential = match credentials::load(&origin) { - Ok(Some(credential)) => credential, + let outcome = match configured_location(project_dir) { + Ok(Some(configured)) => sign_in_and_sync(configured, target), Ok(None) => return None, - Err(error) => return skipped(error), - }; - let ca = std::env::var_os(CONTROL_CA_CERT_ENV).map(PathBuf::from); - let client = match transport::connect(&origin, ca.as_deref()) { - Ok(client) => client, - Err(error) => return skipped(error), + Err(error) => Err(error), }; - let request = async { - let targets = transport::location_targets(&client, &credential).await?; - let Some(found) = targets - .iter() - .find(|candidate| candidate.name == location && candidate.target == target) - else { - let named = targets.iter().any(|candidate| candidate.name == location); - return Ok(if named { - format!( - "Warehouse sync skipped: location '{location}' has no target '{target}' you can view on {origin}.\n" - ) - } else { - format!( - "Warehouse sync skipped: no location named '{location}' that you can view on {origin}.\n" - ) - }); - }; - transport::warehouse_sync( - &client, - &credential, - (found.location_id, target), - "Requested after sqb build", - ) - .await?; - Ok::<_, std::io::Error>(format!( - "Warehouse sync requested for location '{location}' target '{target}'.\n" - )) - }; - let outcome = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(std::io::Error::other) - .and_then(|runtime| runtime.block_on(request)); match outcome { - Ok(line) => Some(line), - Err(error) => skipped(error), + Ok(line) => line, + Err(error) => Some(format!("Warehouse sync skipped: {error}\n")), } } diff --git a/crates/sqlbuild-cli/src/person_login/mod.rs b/crates/sqlbuild-cli/src/person_login/mod.rs index 1c79f915ed..8ab8e70044 100644 --- a/crates/sqlbuild-cli/src/person_login/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/mod.rs @@ -2,5 +2,5 @@ pub(crate) mod _helpers; pub(crate) mod constants; -pub(crate) mod main; +pub mod main; pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/person_login/models.rs b/crates/sqlbuild-cli/src/person_login/models.rs index 0d9dc926b0..6b973f418b 100644 --- a/crates/sqlbuild-cli/src/person_login/models.rs +++ b/crates/sqlbuild-cli/src/person_login/models.rs @@ -57,17 +57,17 @@ pub(super) struct PersonClient { } /// The parts of `GET /api/control/bootstrap` that name the location targets a person can view. -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] pub(super) struct Bootstrap { pub(super) access: BootstrapAccess, } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] pub(super) struct BootstrapAccess { pub(super) locations: Vec, } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] pub(super) struct LocationTarget { pub(super) location_id: uuid::Uuid, pub(super) name: String, diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 4e4f3ae0bb..2bceff46dc 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -579,11 +579,10 @@ pub(crate) fn sync_interval(text: &str) -> Result { if SYNC_INTERVAL_OFF.contains(&text) { return Ok(0); } - let seconds: u64 = fixed_duration_seconds(text, false) - .ok() - .flatten() - .and_then(|seconds| seconds.parse().ok()) - .ok_or(SYNC_INTERVAL_HELP)?; + let seconds: u64 = match fixed_duration_seconds(text, false) { + Ok(Some(seconds)) => seconds.parse().map_err(|_| SYNC_INTERVAL_HELP)?, + Ok(None) | Err(_) => return Err(SYNC_INTERVAL_HELP), + }; if seconds < MIN_SYNC_INTERVAL_SECONDS { return Err(SYNC_INTERVAL_HELP); } diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index 315985ac78..cfa9eb6e03 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -1,5 +1,4 @@ -//! Named work requires a trusted owner or live person authority; retries preserve the original pin. -//! A sync request while one is still pending for the target is already applied: one sync covers both. +//! Named work requires a trusted owner or live person authority; a pending sync covers a repeated one. use chrono::Duration; use uuid::Uuid; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 6d70b6b2e9..98cff67d73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -5,7 +5,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_cli::native_commands::main::sync_after_build::sync_after_build; +use sqlbuild_cli::person_login::main::sync_after_build::sync_after_build; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::boundary::python_calls::{call_with, within}; diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 63cfe832be..c07d61f44e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -15,6 +15,7 @@ from urllib.request import Request, urlopen from uuid import uuid4 +import duckdb import pytest from sqlbuild._native import ( @@ -1141,21 +1142,8 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre assert location.poll() is None, log.read_text() assert time.monotonic() < deadline, log.read_text() time.sleep(0.05) - lock: Any = None for availability in fixture["availability"]: - if test_case.adapter_name not in availability.get( - "adapters", [test_case.adapter_name] - ): - continue - if lock is not None: - lock.close() - lock = None - if "source" in availability: - _ = (project / availability["source"]).rename( - project / availability["destination"] - ) - if availability.get("hold_write_lock"): - lock = hold_write_lock(project / "warehouse.db") + _ = (project / availability["source"]).rename(project / availability["destination"]) record_finished_run(control) reports = wait_for_warehouse_sync_health( api, availability, location, log, after=observed_health_at @@ -1164,8 +1152,27 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre datetime.fromisoformat(report["received_at"]) for report in reports ) assert (project / "warehouse.db").exists() == availability["available"] - if test_case.adapter_name == "duckdb": - assert log.read_text().count("Warehouse file is in use by another process") == 1 + held_locks: list[dict[str, Any]] = fixture["held_write_lock"][test_case.adapter_name] + for held_lock in held_locks: + lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db") + record_finished_run(control) + reports = wait_for_warehouse_sync_health( + api, held_lock["held"], location, log, after=observed_health_at + ) + observed_health_at = max( + datetime.fromisoformat(report["received_at"]) for report in reports + ) + lock.close() + record_finished_run(control) + reports = wait_for_warehouse_sync_health( + api, held_lock["released"], location, log, after=observed_health_at + ) + observed_health_at = max( + datetime.fromisoformat(report["received_at"]) for report in reports + ) + assert log.read_text().count("Warehouse file is in use by another process") == len( + held_locks + ) @pytest.mark.parametrize( From 86afe81661eaab752b77dcb8121404c8164a3c91 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:10:11 +0000 Subject: [PATCH 9/9] test(janitor): count audit results among the pruned state tables --- .../cli/commands/main/janitor/test_janitor_targets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py index ba85d4120e..ee4e9a2603 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py @@ -162,7 +162,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview prod", "prod.customers -> prod._sqb_archive__", - "\u2713 Janitor completed archived 1, deleted 1, pruned 3", + "\u2713 Janitor completed archived 1, deleted 1, pruned 4", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), @@ -176,7 +176,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview dev", "dev.customers -> dev._sqb_archive__", - "\u2713 Janitor completed archived 1, deleted 1, pruned 3", + "\u2713 Janitor completed archived 1, deleted 1, pruned 4", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",),