From 622acfdc1fc14035b09edec261cadc3f9c3c5a01 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:51:57 +0000 Subject: [PATCH 01/17] feat(plan): time project discovery, rules and the warehouse connection `sqb plan --timings`, and the timings a plan document carries, left discovering the project, evaluating the configured rules and opening the warehouse connection inside Other. Record them as their own steps, "Discovering the project", "Evaluating rules" and "Connecting to the warehouse", so Other is only time no step covers. A location's plan from its synced state uses the project it already discovered, compiled and checked, and opens no connection, so those steps do not appear there. --- .../src/planner_phases/constants.rs | 9 +++++++ .../_helpers/compile_driver/plan_pipeline.rs | 21 +++++++++------ .../bindings/_helpers/plan_command/plan.rs | 21 ++++++++------- .../sqlbuild/references/docs/cli/plan.md | 27 ++++++++++--------- .../cli/commands/main/plan/_test_types.py | 1 + .../commands/main/plan/test_plan_timings.py | 13 ++++++++- website/src/content/docs/docs/cli/plan.mdx | 27 ++++++++++--------- 7 files changed, 77 insertions(+), 42 deletions(-) diff --git a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs index 0a6b6af84f..3524cae4a7 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs @@ -73,9 +73,18 @@ pub const AUDIT_ENTRIES: &str = "audit_entries"; /// An audit plan entry's field saying its failure holds nothing back. pub const IGNORE_FAILURE: &str = "ignore_failure"; +/// Discovering the project's files and Python nodes; a location planning its published project skips it. +pub const DISCOVERING: &str = "Discovering the project"; + /// Compiling the project before planning; a location planning its published project skips it. pub const COMPILING: &str = "Compiling the project"; +/// Evaluating the configured rules; skipped where the rules ran when the project loaded. +pub const EVALUATING_RULES: &str = "Evaluating rules"; + +/// Opening the warehouse connection; a plan from a synced state opens none. +pub const CONNECTING: &str = "Connecting to the warehouse"; + /// The warehouse snapshot, model migrations and source relations, live or from a synced state. pub const READING_WAREHOUSE: &str = "Reading warehouse state"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index f4a1a822b6..87bc98fb83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -1,5 +1,7 @@ //! The compile-and-plan pipeline of execution commands: compile, rules gate, then planning. +use sqlbuild_python_planner::planner_phases::constants::CONNECTING; +use sqlbuild_python_planner::planner_phases::constants::EVALUATING_RULES; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; @@ -113,12 +115,14 @@ pub(crate) fn planned_project<'py>( }; if !rules_ran_at_load() { let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; - let graph: Bound<'py, PyAny> = project_graph(&project)?; - run_rules_gate( - (discovered, adapter, expansion_reuse.as_any()), - (&options, &on_progress), - (&project, &graph), - )?; + timed(EVALUATING_RULES, || { + let graph: Bound<'py, PyAny> = project_graph(&project)?; + run_rules_gate( + (discovered, adapter, expansion_reuse.as_any()), + (&options, &on_progress), + (&project, &graph), + ) + })?; } planned( (discovered, adapter), @@ -297,8 +301,9 @@ fn planned<'py>( ) } None => { - let opened: Bound<'py, PyAny> = - connection_with_hooks(adapter, &connection_config, hooks)?; + let opened: Bound<'py, PyAny> = timed(CONNECTING, || { + connection_with_hooks(adapter, &connection_config, hooks) + })?; connection = Some(opened.clone()); let reads = LiveReads::new(adapter, &opened, None).handle(adapter.py())?; (opened, reads) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index d2e0d8eeb5..fe3981687d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -6,6 +6,7 @@ use sqlbuild_python_planner::planner_phases::constants::WRITING; use sqlbuild_python_planner::planner_phases::main::timing::timed::timed; use sqlbuild_python_planner::planner_phases::models::PlanTimingScope; use sqlbuild_python_planner::planner_phases::models::PlanTimings; +use sqlbuild_python_planner::planner_phases::constants::DISCOVERING; use std::sync::Arc; use pyo3::prelude::{ @@ -164,16 +165,18 @@ fn resolve_invocation<'py>( }; let discovered: Bound<'py, PyAny> = match project { PlanProject::Held { discovered, .. } => discovered.clone(), - PlanProject::Compiled => discover_project_inputs( - py, - Some(keywords( + PlanProject::Compiled => timed(DISCOVERING, || { + discover_project_inputs( py, - &[ - ("project_dir", project_dir.clone()), - ("extract_output_column_locations", flag(py, false)), - ], - )?), - )?, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", flag(py, false)), + ], + )?), + ) + })?, }; validate_preview_target( py, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index a68f01bd5f..1c3f09f65e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -142,20 +142,23 @@ When a removed column and an added column look related but are not an exact rena `--timings` prints how long each planning step took, in pipeline order, after the plan: ``` -Plan timings seconds share - Compiling the project 2.1 20% - Reading warehouse state 3.3 31% - Detecting changes 2.4 23% - Checking source freshness 0.2 2% - Building plan entries 1.6 15% - Preparing test fixtures 0.1 1% - Assembling the plan 0.4 4% - Writing the plan 0.3 3% - Other 0.2 2% - Total 10.6 +Plan timings seconds share + Discovering the project 1.2 10% + Compiling the project 2.1 17% + Evaluating rules 0.6 5% + Connecting to the warehouse 0.3 2% + Reading warehouse state 3.3 26% + Detecting changes 2.4 19% + Checking source freshness 0.2 2% + Building plan entries 1.6 13% + Preparing test fixtures 0.1 1% + Assembling the plan 0.4 3% + Writing the plan 0.3 2% + Other 0.1 1% + Total 12.6 ``` -`Reading build history` appears with `--changed`. `Preparing test fixtures` counts only the time the plan waits for them, because they are prepared alongside the other steps. `Other` is the time no step covers. A plan a location makes for the web UI records the same steps in its plan document. +`Reading build history` appears with `--changed`. A plan a location makes for the web UI from its synced warehouse state uses the project it already discovered, compiled and checked against the rules, and opens no connection, so those steps do not appear there. `Preparing test fixtures` counts only the time the plan waits for them, because they are prepared alongside the other steps. `Other` is the time no step covers. A plan a location makes for the web UI records the same steps in its plan document. ## Exit codes diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index ac1eafad7f..dbc8b38c80 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -94,6 +94,7 @@ class PlanTimingsE2ETestCase: plan_args: tuple[str, ...] expected_first_step: str expected_last_step: str + expected_steps: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py index 1409cc61c8..76e3ebaefb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py @@ -24,8 +24,15 @@ PlanTimingsE2ETestCase( description="--timings prints each step with Other last, then the total", plan_args=("--timings",), - expected_first_step="Compiling the project", + expected_first_step="Discovering the project", expected_last_step="Other", + expected_steps=( + "Discovering the project", + "Compiling the project", + "Evaluating rules", + "Connecting to the warehouse", + "Reading warehouse state", + ), ), ], ids=lambda case: case.description, @@ -41,6 +48,10 @@ def test_given_timings_when_planning_then_the_steps_add_up_to_the_total( assert len(table.steps) == table.row_count, output assert table.steps[0][0] == test_case.expected_first_step, output assert table.steps[-1][0] == test_case.expected_last_step, output + step_names: list[str] = [name for name, _ in table.steps] + assert list(filter(set(test_case.expected_steps).__contains__, step_names)) == list( + test_case.expected_steps + ), output assert "Reading build history" not in output, output assert sum(seconds for _, seconds in table.steps) == pytest.approx( table.total_seconds, abs=0.05 * len(table.steps) diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 4c38bcaea4..1b3ade695c 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -131,20 +131,23 @@ When a removed column and an added column look related but are not an exact rena `--timings` prints how long each planning step took, in pipeline order, after the plan: ``` -Plan timings seconds share - Compiling the project 2.1 20% - Reading warehouse state 3.3 31% - Detecting changes 2.4 23% - Checking source freshness 0.2 2% - Building plan entries 1.6 15% - Preparing test fixtures 0.1 1% - Assembling the plan 0.4 4% - Writing the plan 0.3 3% - Other 0.2 2% - Total 10.6 +Plan timings seconds share + Discovering the project 1.2 10% + Compiling the project 2.1 17% + Evaluating rules 0.6 5% + Connecting to the warehouse 0.3 2% + Reading warehouse state 3.3 26% + Detecting changes 2.4 19% + Checking source freshness 0.2 2% + Building plan entries 1.6 13% + Preparing test fixtures 0.1 1% + Assembling the plan 0.4 3% + Writing the plan 0.3 2% + Other 0.1 1% + Total 12.6 ``` -`Reading build history` appears with `--changed`. `Preparing test fixtures` counts only the time the plan waits for them, because they are prepared alongside the other steps. `Other` is the time no step covers. A plan a location makes for the web UI records the same steps in its plan document. +`Reading build history` appears with `--changed`. A plan a location makes for the web UI from its synced warehouse state uses the project it already discovered, compiled and checked against the rules, and opens no connection, so those steps do not appear there. `Preparing test fixtures` counts only the time the plan waits for them, because they are prepared alongside the other steps. `Other` is the time no step covers. A plan a location makes for the web UI records the same steps in its plan document. ## Exit codes From de14f0016334c44ffb85f7200f4b8d3c5452475d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:51:51 +0000 Subject: [PATCH 02/17] perf(plan): list each schema's tables and views at the same time Planning listed every schema with SHOW TABLES and then SHOW VIEWS, one after the other. A schema-scoped reader can now hand the catalog its listing as independent reads and combine their results, and Snowflake does so, so the catalog runs every schema's TABLES and VIEWS reads together within the adapter's inspection concurrency. Builds keep the listing path that records schema listing versions. The schema existence check behind a failed SHOW is remembered per plan; callers asking while it runs now wait and share its answer, so two failed reads of a missing schema check it once. --- .../relations/classes/inspection_catalog.py | 109 ++++++++++++++++-- .../relations/classes/remembered_flight.py | 34 ++++++ src/sqlbuild/adapter/relations/types.py | 19 +++ .../snowflake/classes/snowflake_adapter.py | 43 +++++-- .../snowflake/inspection/_test_types.py | 13 ++- .../inspection/test_inspection_catalog.py | 66 ++++++++++- 6 files changed, 255 insertions(+), 29 deletions(-) create mode 100644 src/sqlbuild/adapter/relations/classes/remembered_flight.py diff --git a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py index 870d551b88..f02c12c41f 100644 --- a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py +++ b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py @@ -13,6 +13,7 @@ current_relation_metadata_cache, ) from sqlbuild.adapter.relations.classes.relation_metadata_cache import RelationMetadataCache +from sqlbuild.adapter.relations.classes.remembered_flight import RememberedFlight from sqlbuild.adapter.relations.constants import ( INSPECTION_DESCRIBED_NAME_LIMIT, INSPECTION_IN_LIST_LIMIT, @@ -25,6 +26,7 @@ SchemaRelationListing, ) from sqlbuild.adapter.relations.types import ( + PartedRelationListingReader, RelationListingScoper, SchemaScopedMetadataReader, ViewDefinitionBatchReader, @@ -50,19 +52,35 @@ def __init__(self, *, adapter: Any, connection: Any) -> None: self._capped_column_schemas: set[_SchemaKey] = set() self._remembered: dict[tuple[str, ...], object] = {} self._remember_lock: threading.Lock = threading.Lock() + self._in_flight: dict[tuple[str, ...], RememberedFlight] = {} self._relation_requests: dict[_RelationRequestKey, tuple[RelationInfo, ...]] = {} self._relation_columns: dict[_RelationIdentity, tuple[ColumnInfo, ...] | None] = {} self._view_definitions: dict[_RelationIdentity, str | None] = {} def remember[ValueT](self, *, key: tuple[str, ...], compute: Callable[[], ValueT]) -> ValueT: - """Compute one adapter-owned metadata fact at most once per planning invocation.""" + """Compute one adapter-owned metadata fact at most once; concurrent callers share it.""" with self._remember_lock: if key in self._remembered: return cast(ValueT, self._remembered[key]) - value: ValueT = compute() + flight: RememberedFlight | None = self._in_flight.get(key) + owner: bool = flight is None + if flight is None: + flight = RememberedFlight() + self._in_flight[key] = flight + if not owner: + return cast(ValueT, flight.wait()) + try: + value: ValueT = compute() + except BaseException as error: + with self._remember_lock: + del self._in_flight[key] + flight.fail(error) + raise with self._remember_lock: self._remembered.setdefault(key, value) + del self._in_flight[key] + flight.succeed(value) return value @property @@ -126,21 +144,73 @@ def prefetch_schema_listings( key: _SchemaKey = self._schema_key(database=database, schema=schema) if key not in self._relation_listings: missing.setdefault(key, (database, schema)) - listings: list[SchemaRelationListing | None] = run_bounded_inspections( - tasks=tuple( - self._bind_relation_listing( - reader=reader, database=database, schema=schema, best_effort=best_effort - ) - for database, schema in missing.values() - ), - concurrency=self._concurrency, - ) + listings: list[SchemaRelationListing | None] + if isinstance(reader, PartedRelationListingReader) and ( + current_relation_metadata_cache(adapter=self.adapter) is None + ): + listings = self._read_parted_listings( + reader=reader, scopes=tuple(missing.values()), best_effort=best_effort + ) + else: + listings = run_bounded_inspections( + tasks=tuple( + self._bind_relation_listing( + reader=reader, database=database, schema=schema, best_effort=best_effort + ) + for database, schema in missing.values() + ), + concurrency=self._concurrency, + ) missing_key: _SchemaKey listing: SchemaRelationListing | None for missing_key, listing in zip(missing, listings, strict=True): if listing is not None: self._relation_listings[missing_key] = listing + def _read_parted_listings( + self, + *, + reader: PartedRelationListingReader, + scopes: tuple[tuple[str | None, str], ...], + best_effort: bool, + ) -> list[SchemaRelationListing | None]: + """Run every schema's listing reads together; a best-effort failure loses its schema.""" + + parts: list[tuple[int, Callable[[], object]]] = [] + failed: set[int] = set() + index: int + database: str | None + schema: str + for index, (database, schema) in enumerate(scopes): + try: + reads: tuple[Callable[[], object], ...] = reader.relation_listing_reads( + connection=self.connection, database=database, schema=schema + ) + except Exception: + if not best_effort: + raise + failed.add(index) + continue + parts.extend((index, read) for read in reads) + results: list[object] = run_bounded_inspections( + tasks=tuple(_guarded_read(read=read, best_effort=best_effort) for _, read in parts), + concurrency=self._concurrency, + ) + by_scope: dict[int, list[object]] = {} + result: object + for (index, _), result in zip(parts, results, strict=True): + if result is _FAILED_READ: + failed.add(index) + by_scope.setdefault(index, []).append(result) + return [ + None + if index in failed + else reader.relation_listing_from_reads( + database=database, schema=schema, results=tuple(by_scope.get(index, ())) + ) + for index, (database, schema) in enumerate(scopes) + ] + def prefetch_view_definitions( self, *, requests: tuple[tuple[str | None, str, tuple[str, ...]], ...] ) -> None: @@ -471,6 +541,23 @@ def _schema_key(self, *, database: str | None, schema: str) -> _SchemaKey: ) +_FAILED_READ: object = object() + + +def _guarded_read(*, read: Callable[[], object], best_effort: bool) -> Callable[[], object]: + """``read``, answering a failure with ``_FAILED_READ`` when the listing is best effort.""" + + def run() -> object: + try: + return read() + except Exception: + if best_effort: + return _FAILED_READ + raise + + return run + + def _flatten_relations(chunks: list[tuple[RelationInfo, ...]]) -> tuple[RelationInfo, ...]: relations: list[RelationInfo] = [] chunk: tuple[RelationInfo, ...] diff --git a/src/sqlbuild/adapter/relations/classes/remembered_flight.py b/src/sqlbuild/adapter/relations/classes/remembered_flight.py new file mode 100644 index 0000000000..c53abe6212 --- /dev/null +++ b/src/sqlbuild/adapter/relations/classes/remembered_flight.py @@ -0,0 +1,34 @@ +"""One remembered catalog computation in progress, which callers of the same key wait for.""" + +from __future__ import annotations + +import threading + + +class RememberedFlight: + """A computation's value or failure, shared with callers that asked while it ran.""" + + def __init__(self) -> None: + self._done: threading.Event = threading.Event() + self._value: object = None + self._error: BaseException | None = None + + def succeed(self, value: object) -> None: + """Share ``value`` with every waiting caller.""" + + self._value = value + self._done.set() + + def fail(self, error: BaseException) -> None: + """Raise ``error`` in every waiting caller.""" + + self._error = error + self._done.set() + + def wait(self) -> object: + """The computation's value once it finishes, raising its failure.""" + + _ = self._done.wait() + if self._error is not None: + raise self._error + return self._value diff --git a/src/sqlbuild/adapter/relations/types.py b/src/sqlbuild/adapter/relations/types.py index 4863de5887..20bf2f654f 100644 --- a/src/sqlbuild/adapter/relations/types.py +++ b/src/sqlbuild/adapter/relations/types.py @@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable if TYPE_CHECKING: + from collections.abc import Callable + from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations.models import SchemaColumnListing, SchemaRelationListing @@ -69,6 +71,23 @@ def metadata_name_key(self, name: str) -> str: ... +@runtime_checkable +class PartedRelationListingReader(Protocol): + """Schema-scoped reader whose relation listing is independent reads that may run together.""" + + def relation_listing_reads( + self, *, connection: Any, database: str | None, schema: str + ) -> tuple[Callable[[], object], ...]: + """The reads ``read_schema_relation_listing`` runs one after another for one schema.""" + ... + + def relation_listing_from_reads( + self, *, database: str | None, schema: str, results: tuple[object, ...] + ) -> SchemaRelationListing: + """The schema's listing from its reads' results, in ``relation_listing_reads`` order.""" + ... + + @runtime_checkable class RelationChangeTokenReader(Protocol): """Adapter that reads a table's change token without starting warehouse compute.""" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 3317a3cb44..41d3d77767 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -2281,24 +2281,45 @@ def read_schema_relation_listing( ) -> SchemaRelationListing: """List one schema's tables and views with SHOW, which needs no running warehouse.""" + return self.relation_listing_from_reads( + database=database, + schema=schema, + results=tuple( + read() + for read in self.relation_listing_reads( + connection=connection, database=database, schema=schema + ) + ), + ) + + def relation_listing_reads( + self, *, connection: _SnowflakeConnection, database: str | None, schema: str + ) -> tuple[Callable[[], object], ...]: + """SHOW TABLES and SHOW VIEWS of one schema, independent reads that may run together.""" + show_database: str = self._show_database( connection=connection, database=database, schema=schema ) scope: str = self._show_schema_scope( connection=connection, database=database, schema=schema ) - tables: list[dict[str, object]] = self._fetch_paged_show_rows( - connection=connection, - query=f"SHOW TABLES IN SCHEMA {scope}", - show_database=show_database, - schema=schema, - ) - views: list[dict[str, object]] = self._fetch_paged_show_rows( - connection=connection, - query=f"SHOW VIEWS IN SCHEMA {scope}", - show_database=show_database, - schema=schema, + return tuple( + partial( + self._fetch_paged_show_rows, + connection=connection, + query=f"SHOW {kind} IN SCHEMA {scope}", + show_database=show_database, + schema=schema, + ) + for kind in ("TABLES", "VIEWS") ) + + def relation_listing_from_reads( + self, *, database: str | None, schema: str, results: tuple[object, ...] + ) -> SchemaRelationListing: + """One schema's listing from its SHOW TABLES and SHOW VIEWS rows.""" + + tables, views = cast(tuple[list[dict[str, object]], list[dict[str, object]]], results) entries: dict[str, ListedRelation] = {} row: dict[str, object] for row in tables: diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py index 3eb644e84e..a38573a919 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py @@ -60,7 +60,7 @@ class SpeculativePrefetchTestCase: description: str failing_schema: str expected_error_fragment: str - expected_failed_reads: int + expected_failed_reads: tuple[int, int] @dataclass(frozen=True) @@ -214,3 +214,14 @@ class SessionDatabaseSwitchTestCase: switch_sql: str expected_scopes: tuple[str, ...] expected_session_reads: int + + +@dataclass(frozen=True) +class PartedListingTestCase: + """Schemas whose tables and views are listed by separate reads running together.""" + + description: str + schema_count: int + relations_per_schema: int + statement_latency_seconds: float + expected_max_concurrent: int diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_inspection_catalog.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_inspection_catalog.py index 10fb349f1c..e63d00f698 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_inspection_catalog.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_inspection_catalog.py @@ -28,6 +28,7 @@ FreshnessReuseTestCase, MissingSchemaTestCase, NestedConcurrencyTestCase, + PartedListingTestCase, RelationRequestEquivalenceTestCase, RepeatedLookupTestCase, ShowResultCapTestCase, @@ -285,7 +286,9 @@ def test_given_repeated_lookups_when_catalog_is_open_then_lists_schema_once( ) assert counts == test_case.expected_relation_counts - assert tuple(query.kind for query in warehouse.queries) == test_case.expected_query_kinds + assert sorted(query.kind for query in warehouse.queries) == sorted( + test_case.expected_query_kinds + ) @pytest.mark.parametrize( @@ -295,7 +298,7 @@ def test_given_repeated_lookups_when_catalog_is_open_then_lists_schema_once( description="unreadable speculative schema fails only the request that needs it", failing_schema="RESTRICTED", expected_error_fragment="unavailable", - expected_failed_reads=2, + expected_failed_reads=(3, 4), ) ], ids=lambda case: case.description, @@ -324,7 +327,8 @@ def test_given_unreadable_schema_when_prefetching_best_effort_then_only_real_rea assert len(staging) == 1 assert len(warehouse.queries_of_kind("show_tables")) == 1 - assert len(warehouse.attempted_sql) - len(warehouse.queries) == test_case.expected_failed_reads + failed_reads: int = len(warehouse.attempted_sql) - len(warehouse.queries) + assert test_case.expected_failed_reads[0] <= failed_reads <= test_case.expected_failed_reads[1] @pytest.mark.parametrize( @@ -395,7 +399,9 @@ def test_given_schema_beyond_show_cap_when_listing_then_matches_direct_adapter( assert sorted_relation_reprs(served) == sorted_relation_reprs(direct) assert served_columns == direct_columns - assert tuple(query.kind for query in warehouse.queries) == test_case.expected_query_kinds + assert sorted(query.kind for query in warehouse.queries) == sorted( + test_case.expected_query_kinds + ) @pytest.mark.parametrize( @@ -434,7 +440,8 @@ def test_given_no_database_when_listing_schemas_then_show_uses_the_session_datab assert len(staging) + len(marts) == test_case.expected_relation_count assert {relation.database for relation in (*staging, *marts)} == {None} - assert tuple(warehouse.attempted_sql) == test_case.expected_attempted_sql + assert warehouse.attempted_sql[0] == test_case.expected_attempted_sql[0] + assert sorted(warehouse.attempted_sql[1:]) == sorted(test_case.expected_attempted_sql[1:]) @pytest.mark.parametrize( @@ -447,6 +454,7 @@ def test_given_no_database_when_listing_schemas_then_show_uses_the_session_datab expected_error_fragment="database restricted_catalog does not exist", expected_attempted_sql=( 'SHOW TABLES IN SCHEMA "RESTRICTED_CATALOG"."STAGING" LIMIT 10000', + 'SHOW VIEWS IN SCHEMA "RESTRICTED_CATALOG"."STAGING" LIMIT 10000', "SHOW SCHEMAS LIKE 'STAGING' IN DATABASE \"RESTRICTED_CATALOG\"", ), ) @@ -464,7 +472,7 @@ def test_given_missing_database_when_listing_schema_then_inspection_fails( with pytest.raises(AdapterUserError, match=test_case.expected_error_fragment or ""): _ = catalog.list_relations(database=test_case.database, schemas=("staging",)) - assert tuple(warehouse.attempted_sql) == test_case.expected_attempted_sql + assert set(warehouse.attempted_sql) == set(test_case.expected_attempted_sql) @pytest.mark.parametrize( @@ -667,5 +675,51 @@ def test_given_capped_schemas_when_reading_columns_then_nested_reads_do_not_mult assert len(warehouse.queries_of_kind("columns")) == test_case.expected_column_reads +@pytest.mark.parametrize( + "test_case", + [ + PartedListingTestCase( + description="each schema's tables and views are listed at once", + schema_count=2, + relations_per_schema=3, + statement_latency_seconds=0.05, + expected_max_concurrent=4, + ) + ], + ids=lambda case: case.description, +) +def test_given_schemas_when_prefetching_listings_then_tables_and_views_run_together( + test_case: PartedListingTestCase, +) -> None: + relations: tuple[FakeRelation, ...] = build_wide_schemas_relations( + schema_count=test_case.schema_count, + relation_count=test_case.relations_per_schema, + columns_per_relation=1, + ) + adapter, connection, warehouse = build_offline_snowflake( + relations=relations, statement_latency_seconds=test_case.statement_latency_seconds + ) + schemas: tuple[str, ...] = tuple( + f"warehouse_{index}" for index in range(test_case.schema_count) + ) + direct: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, database="analytics", schemas=schemas + ) + warehouse.reset() + + with open_inspection_catalog(adapter=adapter, connection=connection) as catalog: + catalog.prefetch_schema_listings(scopes=tuple(("analytics", schema) for schema in schemas)) + listed: tuple[RelationInfo, ...] = catalog.list_relations( + database="analytics", schemas=schemas + ) + + assert sorted(relation.identity for relation in listed) == sorted( + relation.identity for relation in direct + ) + assert warehouse.max_concurrent_metadata == test_case.expected_max_concurrent + assert len(warehouse.queries_of_kind("show_tables")) == test_case.schema_count + assert len(warehouse.queries_of_kind("show_views")) == test_case.schema_count + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From ab3ca1120e2530a7d3722091b90b846aa37f3654 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:51:43 +0000 Subject: [PATCH 03/17] perf(plan): read cursor bounds in batched UNION ALL statements A live plan read every incremental model's cursor bounds with its own statement, so a project with hundreds of cursor models ran hundreds of round trips, eight at a time on Snowflake. Read up to 50 physical cursor reads in one UNION ALL statement, one row per read tagged with its position, at the adapter's inspection concurrency. A read without a minimum repeats its maximum, so no branch casts a bare NULL. When a batch fails, its reads run their own statements, so one unreadable relation is still the only one reported unavailable. A single read keeps its single statement. The eligible-maximum reads stay one statement each: they filter rows rather than read metadata, and batched they ran one after another inside a single statement. --- .../main/cursor_batch_sql.rs | 27 +++ .../src/planner_warehouse/main/mod.rs | 1 + .../tests/test_cursor_queries.rs | 12 + .../src/planner_warehouse/tests/test_types.rs | 1 + .../src/live_reads/_helpers/bounded.rs | 47 ++-- .../src/live_reads/_helpers/cursor_batches.rs | 80 +++++++ .../src/live_reads/_helpers/mod.rs | 1 + .../src/live_reads/main/bound_task.rs | 2 +- .../src/live_reads/mod.rs | 1 + .../src/live_reads/models.rs | 70 ++++-- .../src/live_reads/types.rs | 6 + .../_helpers/inspection/cursors.rs | 206 +++++++++++++++--- .../src/planner_warehouse/constants.rs | 3 + .../src/planner_warehouse/models.rs | 18 +- .../src/planner_warehouse/types.rs | 17 +- .../_helpers/compile_driver/plan_pipeline.rs | 4 +- .../bindings/_helpers/plan_command/plan.rs | 2 +- fensu.toml | 2 +- .../main/plan/test_verbose_inspection.py | 4 +- .../compiler/planner/_helpers/_test_types.py | 2 + .../compiler/planner/_helpers/helpers.py | 6 +- .../_helpers/test_warehouse_cursor_reads.py | 21 +- .../_helpers/test_warehouse_snapshot.py | 8 +- .../sqlbuild/compiler/planner/main/helpers.py | 11 +- .../main/test_snowflake_inspection_budget.py | 11 +- .../adapters/snowflake/inspection/helpers.py | 33 ++- 26 files changed, 494 insertions(+), 102 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_batch_sql.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/_helpers/cursor_batches.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/types.rs diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_batch_sql.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_batch_sql.rs new file mode 100644 index 0000000000..97b31e537f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_batch_sql.rs @@ -0,0 +1,27 @@ +//! One `UNION ALL` statement reading the bounds of several physical cursor reads. + +use crate::planner_warehouse::models::PhysicalCursorQuery; + +/// One `(_read, _min, _max)` row per query; a missing minimum repeats the maximum, not `NULL`. +pub fn cursor_batch_sql( + queries: &[&PhysicalCursorQuery], + as_text: &dyn Fn(&PhysicalCursorQuery, &str) -> String, +) -> String { + queries + .iter() + .enumerate() + .map(|(read, query)| { + let maximum: String = as_text(query, &format!("MAX({})", query.cursor_column)); + let minimum: String = if query.min_tags.is_empty() { + maximum.clone() + } else { + as_text(query, &format!("MIN({})", query.cursor_column)) + }; + format!( + "SELECT {read} AS _read, {minimum} AS _min, {maximum} AS _max FROM {}", + query.relation + ) + }) + .collect::>() + .join("\nUNION ALL\n") +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs index 4cd8559b89..22e9b87248 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs @@ -1,5 +1,6 @@ //! Entry points of warehouse-inspection planning. +pub mod cursor_batch_sql; pub mod cursor_bounds; pub mod cursor_query_groups; pub mod cursor_query_identity; diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs index df78d914d7..a614f5b3bf 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs @@ -1,8 +1,10 @@ //! Logical cursor bounds share one physical read per relation column and type. +use crate::planner_warehouse::main::cursor_batch_sql::cursor_batch_sql; use crate::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use crate::planner_warehouse::main::cursor_query_identity::cursor_query_identity; use crate::planner_warehouse::main::cursor_query_sql::cursor_query_sql; +use crate::planner_warehouse::models::PhysicalCursorQuery; use crate::planner_warehouse::tests::helpers::request; use crate::planner_warehouse::tests::test_types::CursorGroupTestCase; @@ -37,16 +39,26 @@ fn given_logical_bounds_when_grouping_then_reads_each_relation_once_in_first_ord "(1/2): analytics.orders.updated_at [max]", "(2/2): raw.events.updated_at [min,max]", ], + expected_batch_sql: "SELECT 0 AS _read, CAST(MAX(updated_at) AS VARCHAR) AS _min, CAST(MAX(updated_at) AS VARCHAR) AS _max FROM analytics.orders\nUNION ALL\nSELECT 1 AS _read, CAST(MIN(updated_at) AS VARCHAR) AS _min, CAST(MAX(updated_at) AS VARCHAR) AS _max FROM raw.events", }, CursorGroupTestCase { description: "no requests read nothing", requests: Vec::new(), expected_sql: Vec::new(), expected_identities: Vec::new(), + expected_batch_sql: "", }, ]; for test_case in test_cases { let groups = cursor_query_groups(&test_case.requests); + let batch: Vec<&PhysicalCursorQuery> = groups.iter().collect(); + let batch_sql: String = + cursor_batch_sql(&batch, &|_query, bound| format!("CAST({bound} AS VARCHAR)")); + assert_eq!( + batch_sql, test_case.expected_batch_sql, + "{}", + test_case.description + ); let sql: Vec = groups .iter() .map(|group| cursor_query_sql(group, &|bound| format!("CAST({bound} AS VARCHAR)"))) diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs index f4ae048082..332339cdc2 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs @@ -5,6 +5,7 @@ pub(super) struct CursorGroupTestCase { pub requests: Vec, pub expected_sql: Vec<&'static str>, pub expected_identities: Vec<&'static str>, + pub expected_batch_sql: &'static str, } pub(super) struct RetentionDirectionTestCase { diff --git a/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs b/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs index ddf86cf94e..f9e3bffab9 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs @@ -19,12 +19,12 @@ pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } -/// Run one cursor statement; a failure leaves only that relation unavailable. -pub(crate) fn read_cursor_query<'py>( +/// Run one cursor statement and record it: `(rows, seconds, error)`, `rows` `None` on failure. +fn run_cursor_statement<'py>( py: Python<'py>, sql: &str, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { +) -> PyResult<(Bound<'py, PyAny>, f64, Bound<'py, PyAny>)> { let started = Instant::now(); let outcome = call_with( execute, @@ -32,19 +32,10 @@ pub(crate) fn read_cursor_query<'py>( ) .and_then(|result| result.call_method0("fetchall")); let elapsed = started.elapsed().as_secs_f64(); - let (row, error, count) = match outcome { + let (rows, error, count) = match outcome { Ok(rows) => { - let count = rows.len()?; - let row = if count > 0 { - rows.get_item(0)? - } else { - py.None().into_bound(py) - }; - ( - row, - py.None().into_bound(py), - count.into_pyobject(py)?.into_any(), - ) + let count = rows.len()?.into_pyobject(py)?.into_any(); + (rows, py.None().into_bound(py), count) } Err(error) if error.is_instance_of::(py) => { let value = error.value(py).clone().into_any(); @@ -67,14 +58,33 @@ pub(crate) fn read_cursor_query<'py>( ], )?; call_with(&host_entry(py, RECORD_QUERY)?, &[("record", record)])?; - Ok(PyTuple::new(py, [row, PyFloat::new(py, elapsed).into_any(), error])?.into_any()) + Ok((rows, elapsed, error)) +} + +/// Run and record one cursor statement: `(rows, seconds, error)`, rows cut to the first if `first_row`. +pub(crate) fn read_cursor_query<'py>( + py: Python<'py>, + sql: &str, + reader: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + first_row: bool, +) -> PyResult> { + let (rows, elapsed, error) = run_cursor_statement(py, sql, reader)?; + let rows = if !first_row { + rows + } else if !rows.is_none() && rows.len()? > 0 { + rows.get_item(0)? + } else { + py.None().into_bound(py) + }; + Ok(PyTuple::new(py, [rows, PyFloat::new(py, elapsed).into_any(), error])?.into_any()) } -/// A bounded-inspection task running one cursor statement. +/// A bounded-inspection task running `read_cursor_query` on one statement. pub(crate) fn bound_task<'py>( py: Python<'py>, sql: String, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + first_row: bool, ) -> PyResult> { let connection: Py = connection.clone().unbind(); let execute: Py = execute.clone().unbind(); @@ -86,7 +96,8 @@ pub(crate) fn bound_task<'py>( _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult> { let py = args.py(); - Ok(read_cursor_query(py, &sql, (connection.bind(py), execute.bind(py)))?.unbind()) + let reader = (connection.bind(py), execute.bind(py)); + Ok(read_cursor_query(py, &sql, reader, first_row)?.unbind()) }, )? .into_any()) diff --git a/crates/sqlbuild-python-planner/src/live_reads/_helpers/cursor_batches.rs b/crates/sqlbuild-python-planner/src/live_reads/_helpers/cursor_batches.rs new file mode 100644 index 0000000000..d36f05f790 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/_helpers/cursor_batches.rs @@ -0,0 +1,80 @@ +//! Cursor reads answered together in batched `UNION ALL` statements. + +use crate::live_reads::_helpers::bounded::bound_task; +use crate::live_reads::_helpers::bounded::run_bounded; +use crate::live_reads::types::CursorBatchOutcomes; +use crate::planner_warehouse::models::CursorBatch; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyFloat, PyTuple}; +use std::collections::HashMap; + +/// Run `batches` at most `concurrency` at a time: each answered read's outcome, and failed batches' reads. +pub(crate) fn run_cursor_batches<'py>( + py: Python<'py>, + batches: &[CursorBatch], + reader: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (concurrency, on_complete): (i64, Option>), +) -> PyResult> { + let tasks: Vec> = batches + .iter() + .map(|batch| bound_task(py, batch.sql.clone(), reader, false)) + .collect::>()?; + let results = run_bounded(py, tasks, concurrency.max(1), on_complete)?; + let mut answered: Vec<(usize, Bound<'py, PyAny>)> = Vec::new(); + let mut failed: Vec = Vec::new(); + for (batch, result) in batches.iter().zip(results) { + let rows: Bound<'py, PyAny> = result.get_item(0)?; + if !result.get_item(2)?.is_none() || rows.is_none() { + failed.extend(batch.reads.iter().map(|(index, _)| *index)); + continue; + } + let elapsed: f64 = result.get_item(1)?.extract()?; + let mut by_read: HashMap> = HashMap::new(); + for row in rows.try_iter()? { + let row: Bound<'py, PyAny> = row?; + let read: usize = read_position(&row.get_item(0)?)?; + by_read.insert(read, row); + } + for (position, (index, has_min)) in batch.reads.iter().enumerate() { + let row: Bound<'py, PyAny> = by_read + .remove(&position) + .unwrap_or_else(|| py.None().into_bound(py)); + let outcome = PyTuple::new( + py, + [ + bound_row(&row, *has_min)?, + PyFloat::new(py, elapsed).into_any(), + py.None().into_bound(py), + ], + )?; + answered.push((*index, outcome.into_any())); + } + } + Ok((answered, failed)) +} + +/// A batch row's `_read` position, which a driver may return as an integer or a decimal. +fn read_position(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(position) = value.extract::() { + return Ok(position); + } + let text: String = value.str()?.extract()?; + text.parse::().map_err(|_| { + PyValueError::new_err(format!("cursor batch row has no read position: {text}")) + }) +} + +/// A batch row `(_read, _min, _max)` as the row its read's own statement returns. +fn bound_row<'py>(row: &Bound<'py, PyAny>, has_min: bool) -> PyResult> { + let py = row.py(); + if row.is_none() { + return Ok(py.None().into_bound(py)); + } + let bounds: Vec> = if has_min { + vec![row.get_item(1)?, row.get_item(2)?] + } else { + vec![row.get_item(2)?] + }; + Ok(PyTuple::new(py, bounds)?.into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs index b3886f7bf0..ac232e5f26 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod bounded; +pub(crate) mod cursor_batches; diff --git a/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs b/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs index 8230420d86..44c7be36bb 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs @@ -10,5 +10,5 @@ pub fn bound_task<'py>( sql: String, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - bounded::bound_task(py, sql, (connection, execute)) + bounded::bound_task(py, sql, (connection, execute), true) } diff --git a/crates/sqlbuild-python-planner/src/live_reads/mod.rs b/crates/sqlbuild-python-planner/src/live_reads/mod.rs index 19d4b07806..3132bb75fc 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/mod.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-planner/src/live_reads/models.rs b/crates/sqlbuild-python-planner/src/live_reads/models.rs index 030c294279..6fb7db6992 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/models.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/models.rs @@ -6,6 +6,7 @@ use crate::live_reads::_helpers::bounded::eligible_task; use crate::live_reads::_helpers::bounded::read_cursor_query; use crate::live_reads::_helpers::bounded::run_bounded; use crate::live_reads::_helpers::bounded::text; +use crate::live_reads::_helpers::cursor_batches::run_cursor_batches; use crate::live_reads::constants::ACTIVE_CATALOG; use crate::live_reads::constants::EXECUTE; use crate::live_reads::constants::INSPECTION_CATALOG; @@ -14,7 +15,8 @@ use crate::live_reads::constants::RELATIONS_MAIN_LIST; use crate::live_reads::main::bound_task::bound_task; use crate::observations::main::capture_source_columns::capture_source_columns; use crate::planner_warehouse::models::{ - AdapterRead, EligibleCursorQuery, FingerprintRead, PlannerReads, Read, StateHistoryRead, + AdapterRead, CursorStatements, EligibleCursorQuery, FingerprintRead, PlannerReads, Read, + StateHistoryRead, }; use crate::planner_warehouse::types::CursorProgress; use crate::planner_warehouse::types::WarehouseReads; @@ -250,35 +252,61 @@ impl WarehouseReads for LiveReads { fn cursor_bounds<'py>( &self, py: Python<'py>, - statements: &dyn Fn() -> PyResult>, + statements: &dyn Fn() -> PyResult, progress: &dyn CursorProgress<'py>, ) -> PyResult>>> { - let statements: Vec = statements()?; + let CursorStatements { singles, batches } = statements()?; let connection: Bound<'py, PyAny> = self.connection(py); let execute: Bound<'py, PyAny> = self.method(py, "execute")?; let concurrency: i64 = self.inspection_concurrency(py)?; - let total: usize = statements.len(); - if concurrency <= 1 || total <= 1 { - let mut outcomes: Vec> = Vec::with_capacity(total); - for (index, sql) in statements.iter().enumerate() { + let total: usize = singles.len(); + let mut outcomes: Vec>> = vec![None; total]; + let pending: Vec = if batches.is_empty() || total <= 1 { + (0..total).collect() + } else { + progress.batched( + batches.len(), + (concurrency.max(1) as usize).min(batches.len()), + )?; + let sizes: Vec = batches.iter().map(|batch| batch.reads.len()).collect(); + let (answered, failed) = run_cursor_batches( + py, + &batches, + (&connection, &execute), + (concurrency, Some(progress.batch_callback(py, sizes)?)), + )?; + for (index, outcome) in answered { + outcomes[index] = Some(outcome); + } + failed + }; + if concurrency <= 1 || pending.len() <= 1 { + for index in pending { progress.starting(index)?; - let outcome = read_cursor_query(py, sql, (&connection, &execute))?; + let outcome = + read_cursor_query(py, &singles[index], (&connection, &execute), true)?; progress.finished(index, &outcome)?; - outcomes.push(outcome); + outcomes[index] = Some(outcome); + } + } else { + if batches.is_empty() { + progress.concurrent((concurrency as usize).min(pending.len()))?; + } + let tasks: Vec> = pending + .iter() + .map(|index| bound_task(py, singles[*index].clone(), (&connection, &execute))) + .collect::>()?; + let results = run_bounded( + py, + tasks, + concurrency, + Some(progress.completion_callback(py, pending.clone())?), + )?; + for (index, outcome) in pending.into_iter().zip(results) { + outcomes[index] = Some(outcome); } - return Ok(Read::Known(outcomes)); } - progress.concurrent((concurrency as usize).min(total))?; - let tasks: Vec> = statements - .into_iter() - .map(|sql| bound_task(py, sql, (&connection, &execute))) - .collect::>()?; - Ok(Read::Known(run_bounded( - py, - tasks, - concurrency, - Some(progress.completion_callback(py)?), - )?)) + Ok(Read::Known(outcomes.into_iter().flatten().collect())) } fn eligible_cursor_maxes<'py>( diff --git a/crates/sqlbuild-python-planner/src/live_reads/types.rs b/crates/sqlbuild-python-planner/src/live_reads/types.rs new file mode 100644 index 0000000000..4ed47b70cf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/types.rs @@ -0,0 +1,6 @@ +//! Type aliases of live warehouse reads. + +use pyo3::prelude::{Bound, PyAny}; + +/// Each answered read's position with its row, and the positions of reads whose batch failed. +pub type CursorBatchOutcomes<'py> = (Vec<(usize, Bound<'py, PyAny>)>, Vec); diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs index 877dabe529..90d51f4cfc 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs @@ -7,9 +7,11 @@ use crate::planner_resolution::main::safety::maximum_allowed_start::maximum_allo use crate::planner_strategy::main::lineage::resolve_lineage_reference::resolve_lineage_reference; use crate::planner_warehouse::_helpers::inspection::watermarks::validate_watermark_column; use crate::planner_warehouse::constants::{ - CURSOR_ALGEBRA_ERROR, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, LOG_DEBUG_EVENT, - PLANNER_LOGGER, + CURSOR_ALGEBRA_ERROR, CURSOR_BATCH_SIZE, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, + DATE_VALUE, LOG_DEBUG_EVENT, PLANNER_LOGGER, }; +use crate::planner_warehouse::models::CursorBatch; +use crate::planner_warehouse::models::CursorStatements; use crate::planner_warehouse::models::EligibleCursorQuery; use crate::planner_warehouse::models::Read; use crate::planner_warehouse::models::SnapshotRequest; @@ -19,6 +21,7 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_changes::constants::FORWARD_ONLY_POLICY; +use sqlbuild_analysis::planner_warehouse::main::cursor_batch_sql::cursor_batch_sql; use sqlbuild_analysis::planner_warehouse::main::cursor_bounds::cursor_bounds; use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use sqlbuild_analysis::planner_warehouse::main::cursor_query_identity::cursor_query_identity; @@ -586,17 +589,19 @@ fn report_cursor_query( let error = outcome.get_item(2)?; if error.is_none() { if !on_progress.is_none() { - on_progress.call1((format!( - "Inspected cursor bounds {identity} ({elapsed:.2}s)" - ),))?; + emit_progress( + on_progress, + format!("Inspected cursor bounds {identity} ({elapsed:.2}s)"), + )?; } return Ok(()); } let error_text: String = error.str()?.extract()?; if !on_progress.is_none() { - on_progress.call1((format!( - "Failed cursor bounds {identity} ({elapsed:.2}s): {error_text}" - ),))?; + emit_progress( + on_progress, + format!("Failed cursor bounds {identity} ({elapsed:.2}s): {error_text}"), + )?; } log_debug( py, @@ -614,9 +619,10 @@ fn report_cursor_query( ) } -fn completion_reporter<'py>( +/// The `on_complete` callback of statements where task `i` reads `queries[reads[i]]`. +fn reads_reporter<'py>( py: Python<'py>, - queries: Vec, + (queries, reads): (Vec, Vec), on_progress: &Bound<'py, PyAny>, ) -> PyResult> { let completed = Arc::new(AtomicUsize::new(0)); @@ -635,14 +641,68 @@ fn completion_reporter<'py>( .get_item("result")? .unwrap_or_else(|| py.None().into_bound(py)); let done = completed.fetch_add(1, Ordering::SeqCst) + 1; - let query = &queries[index]; - let identity = cursor_query_identity(query, (done, queries.len())); + let query = &queries[reads[index]]; + let identity = cursor_query_identity(query, (done, reads.len())); report_cursor_query(py, query, (&result, &identity), on_progress.bind(py)) }, )? .into_any()) } +/// The `on_complete` callback of batched statements answering `sizes[i]` reads each. +fn batch_reporter<'py>( + py: Python<'py>, + sizes: Vec, + on_progress: &Bound<'py, PyAny>, +) -> PyResult> { + let completed = Arc::new(AtomicUsize::new(0)); + let on_progress: Py = on_progress.clone().unbind(); + Ok(PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + let py = args.py(); + let on_progress = on_progress.bind(py); + if on_progress.is_none() { + return Ok(()); + } + let kwargs = kwargs.cloned().unwrap_or_else(|| PyDict::new(py)); + let index: usize = kwargs + .get_item("index")? + .map_or(Ok(0), |value| value.extract())?; + let Some(result) = kwargs.get_item("result")? else { + return Ok(()); + }; + let done = completed.fetch_add(1, Ordering::SeqCst) + 1; + let elapsed: f64 = result.get_item(1)?.extract()?; + let error = result.get_item(2)?; + let relations = sizes[index]; + let line = if error.is_none() { + format!( + "Inspected cursor bounds ({done}/{}): {relations} relations ({elapsed:.2}s)", + sizes.len() + ) + } else { + format!( + "Failed cursor bounds ({done}/{}): {relations} relations ({elapsed:.2}s): {}; reading each relation on its own", + sizes.len(), + error.str()? + ) + }; + emit_progress(on_progress, line)?; + Ok(()) + }, + )? + .into_any()) +} + +/// Write one progress `line` through the plan's `on_progress` callback. +fn emit_progress(on_progress: &Bound<'_, PyAny>, line: String) -> PyResult<()> { + on_progress.call1((line,))?; + Ok(()) +} + fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResult> { let mut values = Values::new(); if row.is_none() { @@ -675,11 +735,14 @@ fn execute_queries<'py>( queries: &[PhysicalCursorQuery], ) -> PyResult, HashSet)>> { let py = request.project.py(); - let statements = || -> PyResult> { - queries - .iter() - .map(|query| dialect_cursor_sql(&request.dialect, query)) - .collect() + let statements = || -> PyResult { + Ok(CursorStatements { + singles: queries + .iter() + .map(|query| dialect_cursor_sql(&request.dialect, query)) + .collect::>()?, + batches: dialect_cursor_batches(&request.dialect, queries)?, + }) }; let progress = (queries, &request.on_progress); let Read::Known(outcomes) = @@ -1075,12 +1138,15 @@ pub(crate) fn cursor_snapshots<'py>( .iter() .map(|query| query.min_tags.len() + query.max_tags.len()) .sum(); - request.on_progress.call1((format!( - "Gathered cursor bounds ({}/{logical} logical values; {} physical relation reads). ({:.2}s)", - values.len(), - queries.len(), - started.elapsed().as_secs_f64() - ),))?; + emit_progress( + &request.on_progress, + format!( + "Gathered cursor bounds ({}/{logical} logical values; {} physical relation reads). ({:.2}s)", + values.len(), + queries.len(), + started.elapsed().as_secs_f64() + ), + )?; } Ok((assemble(py, &models, (&values, &failed))?, false)) } @@ -1113,13 +1179,62 @@ pub(crate) fn dialect_cursor_sql( } } +/// `UNION ALL` batches answering `queries`, bounds read as `dialect_cursor_sql` reads them. +fn dialect_cursor_batches( + dialect: &Bound<'_, Dialect>, + queries: &[PhysicalCursorQuery], +) -> PyResult> { + if queries.len() <= 1 { + return Ok(Vec::new()); + } + let py = dialect.py(); + let render = dialect.syntax(CURSOR_TEXT_METHOD)?; + let failure: RefCell> = RefCell::new(None); + let as_text = |query: &PhysicalCursorQuery, expression: &str| -> String { + call_with( + &render, + &[ + ("expression", text(py, expression)), + ("cursor_type", text(py, &query.cursor_type)), + ], + ) + .and_then(|rendered| rendered.extract::()) + .unwrap_or_else(|error| { + failure.replace(Some(error)); + String::new() + }) + }; + let mut batches: Vec = Vec::new(); + for (chunk, start) in queries + .chunks(CURSOR_BATCH_SIZE) + .zip((0..).step_by(CURSOR_BATCH_SIZE)) + { + let members: Vec<&PhysicalCursorQuery> = chunk.iter().collect(); + batches.push(CursorBatch { + reads: chunk + .iter() + .enumerate() + .map(|(offset, query)| (start + offset, !query.min_tags.is_empty())) + .collect(), + sql: cursor_batch_sql(&members, &as_text), + }); + } + match failure.into_inner() { + Some(error) => Err(error), + None => Ok(batches), + } +} + /// Progress lines of one plan's cursor bound statements: the statements' queries and `on_progress`. impl<'py> CursorProgress<'py> for (&[PhysicalCursorQuery], &Bound<'py, PyAny>) { fn starting(&self, index: usize) -> PyResult<()> { let (queries, on_progress) = *self; if !on_progress.is_none() { let identity = cursor_query_identity(&queries[index], (index + 1, queries.len())); - on_progress.call1((format!("Inspecting cursor bounds {identity}..."),))?; + emit_progress( + on_progress, + format!("Inspecting cursor bounds {identity}..."), + )?; } Ok(()) } @@ -1127,10 +1242,32 @@ impl<'py> CursorProgress<'py> for (&[PhysicalCursorQuery], &Bound<'py, PyAny>) { fn concurrent(&self, concurrency: usize) -> PyResult<()> { let (queries, on_progress) = *self; if !on_progress.is_none() { - on_progress.call1((format!( - "Inspecting cursor bounds for {} relations ({concurrency} concurrent)...", - queries.len() - ),))?; + emit_progress( + on_progress, + format!( + "Inspecting cursor bounds for {} relations ({concurrency} concurrent)...", + queries.len() + ), + )?; + } + Ok(()) + } + + fn batched(&self, batches: usize, concurrency: usize) -> PyResult<()> { + let (queries, on_progress) = *self; + if !on_progress.is_none() { + let statements = if batches == 1 { + "statement" + } else { + "statements" + }; + emit_progress( + on_progress, + format!( + "Inspecting cursor bounds for {} relations in {batches} {statements} ({concurrency} concurrent)...", + queries.len() + ), + )?; } Ok(()) } @@ -1146,8 +1283,17 @@ impl<'py> CursorProgress<'py> for (&[PhysicalCursorQuery], &Bound<'py, PyAny>) { ) } - fn completion_callback(&self, py: Python<'py>) -> PyResult> { + fn completion_callback( + &self, + py: Python<'py>, + reads: Vec, + ) -> PyResult> { let (queries, on_progress) = *self; - completion_reporter(py, queries.to_vec(), on_progress) + reads_reporter(py, (queries.to_vec(), reads), on_progress) + } + + fn batch_callback(&self, py: Python<'py>, sizes: Vec) -> PyResult> { + let (_, on_progress) = *self; + batch_reporter(py, sizes, on_progress) } } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs index 7f31dc14a4..5da2153fcc 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs @@ -54,3 +54,6 @@ pub const PREVIEW_UNKNOWNS: [&str; 5] = [ /// What a preview's cursor reads leave to the run. pub const CURSOR_INPUTS: &str = "cursor inputs"; + +/// Cursor reads one `UNION ALL` statement answers, so a plan runs few statements, each small. +pub(crate) const CURSOR_BATCH_SIZE: usize = 50; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index 3f669b09b4..da5c4f3d27 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -427,6 +427,22 @@ pub struct EligibleCursorQuery { pub(crate) sql: String, } +/// One `UNION ALL` statement answering several cursor reads, one row per read led by `_read`. +#[derive(Debug)] +pub struct CursorBatch { + /// The reads it answers in `_read` order: each read's index and whether its row has a minimum. + pub(crate) reads: Vec<(usize, bool)>, + pub(crate) sql: String, +} + +/// A plan's cursor reads: one statement per read, and batches answering the reads together. +#[derive(Debug)] +pub struct CursorStatements { + pub(crate) singles: Vec, + /// Empty when the reads run one statement each. + pub(crate) batches: Vec, +} + /// The warehouse reads one plan runs through. #[pyclass(frozen, module = "sqlbuild._native", name = "WarehouseReads")] #[derive(Debug)] @@ -645,7 +661,7 @@ impl WarehouseReads for SyncedReads { fn cursor_bounds<'py>( &self, _py: Python<'py>, - _statements: &dyn Fn() -> PyResult>, + _statements: &dyn Fn() -> PyResult, _progress: &dyn CursorProgress<'py>, ) -> PyResult>>> { Ok(Read::Unknown(CURSOR_INPUTS.into())) diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs index a036a75f7f..cbf790687b 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs @@ -1,6 +1,7 @@ //! The warehouse reads planning consults, as traits. use crate::planner_warehouse::models::AdapterRead; +use crate::planner_warehouse::models::CursorStatements; use crate::planner_warehouse::models::EligibleCursorQuery; use crate::planner_warehouse::models::FingerprintRead; use crate::planner_warehouse::models::Read; @@ -15,10 +16,18 @@ pub trait CursorProgress<'py> { fn starting(&self, index: usize) -> PyResult<()>; /// Before statements run `concurrency` at a time. fn concurrent(&self, concurrency: usize) -> PyResult<()>; + /// Before `batches` statements answering every read run `concurrency` at a time. + fn batched(&self, batches: usize, concurrency: usize) -> PyResult<()>; /// After statement `index` ran, with its `(row, seconds, error)` outcome. fn finished(&self, index: usize, outcome: &Bound<'py, PyAny>) -> PyResult<()>; - /// The `on_complete(index=, result=)` callback concurrent statements report through. - fn completion_callback(&self, py: Python<'py>) -> PyResult>; + /// The `on_complete(index=, result=)` callback of statements where task `i` reads `reads[i]`. + fn completion_callback( + &self, + py: Python<'py>, + reads: Vec, + ) -> PyResult>; + /// The `on_complete(index=, result=)` callback of batches answering `sizes[i]` reads each. + fn batch_callback(&self, py: Python<'py>, sizes: Vec) -> PyResult>; } /// Everything a plan reads from the warehouse, live or from a synced state that may answer unknown. @@ -65,11 +74,11 @@ pub trait WarehouseReads: Send + Sync + std::fmt::Debug { /// Whether listing by schema and name finds relations a name-only listing would. fn lists_by_schema(&self, py: Python<'_>) -> PyResult; - /// Each statement's `(row, seconds, error)`; `statements` renders SQL only for reads that run it. + /// Each read's `(row, seconds, error)`; a failed batch's reads then run their own statements. fn cursor_bounds<'py>( &self, py: Python<'py>, - statements: &dyn Fn() -> PyResult>, + statements: &dyn Fn() -> PyResult, progress: &dyn CursorProgress<'py>, ) -> PyResult>>>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index 87bc98fb83..c625aa9534 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -1,13 +1,13 @@ //! The compile-and-plan pipeline of execution commands: compile, rules gate, then planning. -use sqlbuild_python_planner::planner_phases::constants::CONNECTING; -use sqlbuild_python_planner::planner_phases::constants::EVALUATING_RULES; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_planner::live_reads::models::LiveReads; use sqlbuild_python_planner::planner_phases::constants::COMPILING; +use sqlbuild_python_planner::planner_phases::constants::CONNECTING; +use sqlbuild_python_planner::planner_phases::constants::EVALUATING_RULES; use sqlbuild_python_planner::planner_phases::main::timing::timed::timed; use sqlbuild_python_planner::planner_warehouse::models::PlannerReads; use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index fe3981687d..8263242843 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -2,11 +2,11 @@ use crate::bindings::_helpers::command_runtime::audit_failures::resolve_ignore_audit_failures; use sqlbuild_python_planner::planner_phases::constants::BUILD_HISTORY; +use sqlbuild_python_planner::planner_phases::constants::DISCOVERING; use sqlbuild_python_planner::planner_phases::constants::WRITING; use sqlbuild_python_planner::planner_phases::main::timing::timed::timed; use sqlbuild_python_planner::planner_phases::models::PlanTimingScope; use sqlbuild_python_planner::planner_phases::models::PlanTimings; -use sqlbuild_python_planner::planner_phases::constants::DISCOVERING; use std::sync::Arc; use pyo3::prelude::{ diff --git a/fensu.toml b/fensu.toml index de2eb450d0..80802c155b 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2602,7 +2602,7 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs=0", "crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs=2", "crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs=1", - "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=11", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=7", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs=6", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs=3", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs=2", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_verbose_inspection.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_verbose_inspection.py index 5376a0ac92..ad243f0398 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_verbose_inspection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_verbose_inspection.py @@ -19,7 +19,9 @@ _INSPECTION_LINE: re.Pattern[str] = re.compile(r"^ inspect +\d+\.\d{2}s \d+ rows? \S", re.M) _METADATA_LINE: re.Pattern[str] = re.compile(r"^ inspect .*list_relations\(", re.M) -_CURSOR_LINE: re.Pattern[str] = re.compile(r"^ inspect .*SELECT CAST\(MAX\(", re.M) +_CURSOR_LINE: re.Pattern[str] = re.compile( + r"^ inspect .*SELECT (?:\d+ AS _read, )?CAST\((?:MIN|MAX)\(", re.M +) _TOTAL_LINE: re.Pattern[str] = re.compile(r"^Inspection queries: (\d+) \(", re.M) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 365c03d796..298c1326d7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -152,6 +152,8 @@ class ConcurrentCursorReadsTestCase: failing_relations: frozenset[str] expected_start_progress: str expected_unreadable: dict[str, tuple[str, ...]] + expected_statements: int + expected_max_active: int @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index c52642fb62..750a365eda 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -340,8 +340,10 @@ def _tracked(self, connection: Any, sql: str, eligible: bool) -> Any: self.cursor_sql.append(sql) try: time.sleep(0.01) - relation: str = sql.split(" FROM ", 1)[1].split(" ", 1)[0] - failing: bool = relation in self._failing[eligible] + relations: set[str] = { + part.split(None, 1)[0].rstrip(")") for part in sql.split(" FROM ")[1:] + } + failing: bool = not relations.isdisjoint(self._failing[eligible]) return _CURSOR_ANSWERS[(failing, self._fail_on_fetch)](connection, sql, self._lock) finally: with self._lock: diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py index ff7c1d48d2..98399cd1b4 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -39,20 +39,28 @@ "test_case", [ ConcurrentCursorReadsTestCase( - description="parallel reads stay within the bound", - relation_count=12, + description="batched reads run concurrently within the bound", + relation_count=120, concurrency=4, failing_relations=frozenset(), - expected_start_progress="Inspecting cursor bounds for 12 relations (4 concurrent)...", + expected_start_progress=( + "Inspecting cursor bounds for 120 relations in 3 statements (3 concurrent)..." + ), expected_unreadable={}, + expected_statements=3, + expected_max_active=3, ), ConcurrentCursorReadsTestCase( description="one failed relation stays unavailable and others succeed", relation_count=8, concurrency=4, failing_relations=frozenset({"staging.events_5"}), - expected_start_progress="Inspecting cursor bounds for 8 relations (4 concurrent)...", + expected_start_progress=( + "Inspecting cursor bounds for 8 relations in 1 statement (1 concurrent)..." + ), expected_unreadable={"daily_5": ("events_5.event_time",)}, + expected_statements=9, + expected_max_active=4, ), ], ids=lambda case: case.description, @@ -92,10 +100,11 @@ def test_given_concurrent_catalog_when_reading_cursor_bounds_then_matches_sequen reads: list[str] = cursor_progress(parallel_progress) assert parallel.cursor_snapshots == sequential.cursor_snapshots assert sequential_execute.max_active == 1 - assert 1 < parallel_execute.max_active <= test_case.concurrency + assert 1 < parallel_execute.max_active <= test_case.expected_max_active assert sorted(parallel_execute.cursor_sql) == sorted(sequential_execute.cursor_sql) + assert len(parallel_execute.cursor_sql) == test_case.expected_statements assert reads[0] == test_case.expected_start_progress - assert len(reads) == test_case.relation_count + 2 + assert len(reads) == test_case.expected_statements + 2 assert unreadable == test_case.expected_unreadable diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index 0ed6547cc9..4d3edc35c2 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -361,7 +361,7 @@ def test_given_sources_when_gathering_source_columns_then_filters_metadata_names upstream_maxes=("2024-02-01 00:00:00",), ), }, - expected_progress_calls=5, + expected_progress_calls=3, ), GatherCursorSnapshotTestCase( description="derives highest eligible target max without removing future rows", @@ -393,7 +393,7 @@ def test_given_sources_when_gathering_source_columns_then_filters_metadata_names upstream_maxes=("2024-02-01 00:00:00",), ), }, - expected_progress_calls=5, + expected_progress_calls=3, ), GatherCursorSnapshotTestCase( description="gathers cursor bounds for first run with no target table", @@ -710,7 +710,7 @@ def test_given_timezone_cursor_text_when_gathering_snapshot_then_normalizes_ware description="explicit timestamp start skips eligible target query", expected_target_max=TimestampValue(value=datetime(2026, 9, 3)), expected_eligible_max=None, - expected_statement_count=2, + expected_statement_count=1, ) ], ids=lambda case: case.description, @@ -845,7 +845,7 @@ def _record_execute(*, connection: Any, sql: str) -> Any: cursor_snapshot: ModelCursorSnapshot = snapshot.cursor_snapshots["hourly_events"] assert cursor_snapshot.target_eligible_max == test_case.expected_eligible_max assert test_case.expected_eligible_sql in statements - assert "UNION ALL" not in statements[0] + assert statements[0].count("\nUNION ALL\n") == 1 @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index ecde09533b..187971a336 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -7,6 +7,7 @@ from collections.abc import Callable from dataclasses import fields, replace from datetime import UTC, datetime +from itertools import chain from pathlib import Path from tempfile import gettempdir from types import MappingProxyType @@ -485,7 +486,7 @@ def grouped(model: type) -> dict[str, Any]: ) _OFFLINE_INFORMATION_SCHEMA_KINDS: frozenset[str] = frozenset({"tables", "columns"}) _SHOW_SCHEMA_SCOPE: re.Pattern[str] = re.compile(r' IN SCHEMA (?:"[^"]*"\.)?"([^"]*)"') -_CURSOR_BOUND_RELATION: re.Pattern[str] = re.compile(r"\sFROM\s+(\S+)\s*$", re.IGNORECASE) +_CURSOR_BOUND_RELATION: re.Pattern[str] = re.compile(r"\sFROM\s+(\S+)\s*$", re.IGNORECASE | re.M) def plan_offline_snowflake_project( @@ -538,11 +539,13 @@ def offline_information_schema_queries( def offline_cursor_bound_relations(warehouse: RecordingSnowflakeWarehouse) -> tuple[str, ...]: - """Return the single relation each cursor-bound statement reads.""" + """Return every relation the cursor-bound statements read, in statement order.""" return tuple( - _CURSOR_BOUND_RELATION.findall(query.sql)[0] - for query in warehouse.queries_of_kind("cursor_bounds") + chain.from_iterable( + _CURSOR_BOUND_RELATION.findall(query.sql) + for query in warehouse.queries_of_kind("cursor_bounds") + ) ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py index 67ce789fab..7d00289314 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py @@ -38,6 +38,7 @@ write_synthetic_snowflake_project, ) +_CURSOR_BATCH_SIZE: int = 50 _ONE_READ_PER_SCHEMA: dict[str, int] = {"STAGING": 1, "INTERMEDIATE": 1, "MARTS": 1, "RAW": 1} _SOURCES: int = 260 _SHOW_READS_PER_SCHEMA: int = 3 @@ -112,14 +113,14 @@ def test_given_large_multi_schema_project_when_planning_then_metadata_reads_stay "test_case", [ SnowflakeCursorBoundsBudgetTestCase( - description="one statement per relation with bounded parallelism", + description="batched statements read each relation once with bounded parallelism", statement_latency_seconds=0.002, expected_max_concurrency=OfflineSnowflakeAdapter.metadata_inspection_concurrency, ) ], ids=lambda case: case.description, ) -def test_given_incremental_models_when_planning_then_cursor_bounds_run_one_per_relation_in_parallel( +def test_given_incremental_models_when_planning_then_cursor_bounds_run_in_parallel_batches( test_case: SnowflakeCursorBoundsBudgetTestCase, tmp_path: Path ) -> None: project: SyntheticSnowflakeProject = write_synthetic_snowflake_project( @@ -136,9 +137,9 @@ def test_given_incremental_models_when_planning_then_cursor_bounds_run_one_per_r bounds: tuple[RecordedQuery, ...] = warehouse.queries_of_kind("cursor_bounds") relations: tuple[str, ...] = offline_cursor_bound_relations(warehouse) - assert len(bounds) >= len(project.incremental_model_names) - assert all("UNION" not in query.sql.upper() for query in bounds) - assert len(relations) == len(bounds) == len(set(relations)) + assert len(relations) >= len(project.incremental_model_names) + assert len(relations) == len(set(relations)) + assert len(bounds) == -(-len(relations) // _CURSOR_BATCH_SIZE) assert 1 < warehouse.max_concurrent_cursor_bounds <= test_case.expected_max_concurrency diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py index e049c88eac..2ff789c9e0 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py @@ -48,6 +48,11 @@ r"^SELECT\s+CAST\((?:MIN|MAX)\(.*\)\s+AS\s+VARCHAR\).*\s+FROM\s+(?P\S+)\s*$", re.IGNORECASE | re.DOTALL, ) +_CURSOR_BATCH_PATTERN: re.Pattern[str] = re.compile(r"^SELECT \d+ AS _read, ", re.IGNORECASE) +_CURSOR_BATCH_BRANCH: re.Pattern[str] = re.compile( + r"^SELECT (?P\d+) AS _read, (?P.*?)(?:\s+FROM\s+(?P[^\s()]+))?$", re.IGNORECASE | re.DOTALL, ) +_DESCRIBE_PATTERN: re.Pattern[str] = re.compile(r"\) AS __describe_source LIMIT 0$") _ANY_PATTERN: re.Pattern[str] = re.compile(r"") _CURRENT_DATABASE_PATTERN: re.Pattern[str] = re.compile(r"^SELECT CURRENT_DATABASE\(\)$") _USE_DATABASE_PATTERN: re.Pattern[str] = re.compile( @@ -329,6 +330,7 @@ def _answer(self, *, sql: str, params: tuple[object, ...]) -> tuple[list[tuple[A (_SHOW_COLUMNS_PATTERN, self._answer_show_columns), (_SHOW_SCHEMA_RELATIONS_PATTERN, self._answer_show_schema_relations), (_SHOW_SCHEMA_COLUMNS_PATTERN, self._answer_show_schema_columns), + (_DESCRIBE_PATTERN, self._answer_describe), (_CURSOR_BATCH_PATTERN, self._answer_cursor_batch), (_CURSOR_BOUND_PATTERN, self._answer_cursor_bounds), (_CURRENT_DATABASE_PATTERN, self._answer_current_database), @@ -517,6 +519,16 @@ def _answer_cursor_bounds( ) return [values], tuple((f"_{index}",) for index in range(len(values))) + def _answer_describe( + self, *, sql: str, params: tuple[object, ...] + ) -> tuple[list[tuple[Any, ...]], Any]: + """A query's columns, named as Snowflake folds unquoted identifiers, and no rows.""" + + del params + with self._lock: + description: Any = self._database.execute(sql).description + return [], tuple((str(column[0]).upper(),) for column in description) + def _answer_cursor_batch( self, *, sql: str, params: tuple[object, ...] ) -> tuple[list[tuple[Any, ...]], Any]: From 40b0124b8a0d761a85fad4395bfa63851167dc51 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:16:19 +0000 Subject: [PATCH 07/17] fix(plan): compare inspected source column types as the warehouse stores them A source declared `VARCHAR` lists as `VARCHAR(16777216)` on Snowflake, and one declared `INTEGER` or `NUMBER` lists as `NUMBER(38,0)`, so every such source counted as changed and planning analysed every selected model reading it again. Compare a declared type with the listed one through the analysis dialect's engine types instead of their text: synonyms, default string lengths, a timestamp precision left unstated, and Snowflake's bare NUMBER or DECIMAL as NUMBER(38,0) all match. A shorter VARCHAR or a different scale still counts as a change. Dialects without a native analysis module keep the exact text comparison. The 10K live-plan guard's offline warehouse now lists source columns with Snowflake's own spellings. --- .../src/analysis_route/constants.rs | 2 + .../src/analysis_route/main/mod.rs | 1 + .../analysis_route/main/same_column_type.rs | 48 +++++++ .../src/analysis_route/tests.rs | 2 + .../tests/test_same_column_type.rs | 132 ++++++++++++++++++ .../_helpers/source_bindings.rs | 21 ++- .../cli/commands/main/plan/helpers.py | 9 +- .../pipeline/test_native_binding_catalog.py | 14 ++ 8 files changed, 220 insertions(+), 9 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/main/same_column_type.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/tests/test_same_column_type.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index d6331e5dd2..2baa56243d 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -490,3 +490,5 @@ pub(crate) const CLICKHOUSE_DECIMAL_WIDTHS: [(&str, u32); 4] = [ ("Decimal128", 38), ("Decimal256", 76), ]; +/// Snowflake's precision of a fixed-point type written without one. +pub(crate) const SNOWFLAKE_DEFAULT_PRECISION: i64 = 38; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 7824a37df3..7bf841082f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -5,6 +5,7 @@ pub mod native_gap; pub(crate) mod native_outcomes; pub mod native_revalidation; pub mod query_expression_ranges; +pub mod same_column_type; pub mod schema_column; pub mod skipped_analysis; pub mod unapplied_nullability_rules; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/same_column_type.rs b/crates/sqlbuild-analysis/src/analysis_route/main/same_column_type.rs new file mode 100644 index 0000000000..9796803da1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/same_column_type.rs @@ -0,0 +1,48 @@ +//! Whether a declared column type and a warehouse-listed one name the type the engine stores. + +use crate::analysis_route::constants::{SNOWFLAKE_DEFAULT_PRECISION, SNOWFLAKE_DIALECT}; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::type_system::main::same_type::same_type; +use crate::type_system::models::{NormalizedType, PythonInteger, TypeFamily}; + +/// Whether `declared` and `listed` are one stored type in `dialect`, its defaults and synonyms met. +#[must_use] +pub fn same_column_type(dialect: Option<&str>, declared: &str, listed: &str) -> bool { + if declared.trim().eq_ignore_ascii_case(listed.trim()) { + return true; + } + let AnalysisRoute::Native(native) = analysis_route(dialect) else { + return false; + }; + let (Some(declared), Some(listed)) = ( + native.normalize_type(declared), + native.normalize_type(listed), + ) else { + return false; + }; + let (declared, listed): (NormalizedType, NormalizedType) = match native.name() { + SNOWFLAKE_DIALECT => (snowflake_defaults(declared), snowflake_defaults(listed)), + _ => (declared, listed), + }; + native.same_type(&declared, &listed) || same_type(&declared, &listed, native.name()) +} + +/// A Snowflake fixed-point type written without precision as the `NUMBER(38,0)` it stores. +fn snowflake_defaults(normalized: NormalizedType) -> NormalizedType { + if normalized.family != TypeFamily::Decimal || normalized.precision.is_some() { + return normalized; + } + let precision: PythonInteger = PythonInteger::from(SNOWFLAKE_DEFAULT_PRECISION); + let scale: PythonInteger = normalized + .scale + .clone() + .unwrap_or_else(|| PythonInteger::from(0)); + NormalizedType { + normalized_name: format!("DECIMAL({precision},{scale})"), + family: TypeFamily::Decimal, + precision: Some(precision), + scale: Some(scale), + length: None, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index 660e64b4ed..4c488bd8ed 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -24,6 +24,8 @@ mod test_native_outcome; mod test_native_revalidation; #[path = "tests/test_postgres_dialect.rs"] mod test_postgres_dialect; +#[path = "tests/test_same_column_type.rs"] +mod test_same_column_type; #[path = "tests/test_type_spelling.rs"] mod test_type_spelling; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_same_column_type.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_same_column_type.rs new file mode 100644 index 0000000000..50908556d4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_same_column_type.rs @@ -0,0 +1,132 @@ +use crate::analysis_route::main::same_column_type::same_column_type; +use crate::analysis_route::tests::test_types::EngineTypeMatchTestCase; + +#[test] +fn given_declared_and_snowflake_listed_types_when_comparing_then_default_spellings_match() { + let test_cases = [ + EngineTypeMatchTestCase { + description: "an unsized VARCHAR is VARCHAR(16777216)", + declared: "VARCHAR", + inferred: "VARCHAR(16777216)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "TEXT and STRING are VARCHAR(16777216)", + declared: "TEXT", + inferred: "VARCHAR(16777216)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "STRING is VARCHAR(16777216)", + declared: "STRING", + inferred: "VARCHAR(16777216)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a bare NUMBER is NUMBER(38,0)", + declared: "NUMBER", + inferred: "NUMBER(38,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a bare DECIMAL is NUMBER(38,0)", + declared: "DECIMAL", + inferred: "NUMBER(38,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "INT is NUMBER(38,0)", + declared: "INT", + inferred: "NUMBER(38,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "BIGINT is NUMBER(38,0)", + declared: "BIGINT", + inferred: "NUMBER(38,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "NUMBER(18) is NUMBER(18,0)", + declared: "NUMBER(18)", + inferred: "NUMBER(18,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "DOUBLE is FLOAT", + declared: "DOUBLE", + inferred: "FLOAT", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a stated timestamp precision meets the listed unstated one", + declared: "TIMESTAMP_NTZ(9)", + inferred: "TIMESTAMP_NTZ", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "spelling case does not matter", + declared: "varchar", + inferred: "VARCHAR(16777216)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a shorter VARCHAR is a different type", + declared: "VARCHAR(10)", + inferred: "VARCHAR(16777216)", + expected_same_type: false, + }, + EngineTypeMatchTestCase { + description: "a scaled NUMBER is not an integer", + declared: "NUMBER(18,2)", + inferred: "NUMBER(38,0)", + expected_same_type: false, + }, + EngineTypeMatchTestCase { + description: "a string is not a number", + declared: "VARCHAR", + inferred: "NUMBER(38,0)", + expected_same_type: false, + }, + EngineTypeMatchTestCase { + description: "TIMESTAMP_TZ is not TIMESTAMP_NTZ", + declared: "TIMESTAMP_TZ", + inferred: "TIMESTAMP_NTZ", + expected_same_type: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + same_column_type(Some("snowflake"), test_case.declared, test_case.inferred), + test_case.expected_same_type, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_dialect_without_a_native_module_when_comparing_then_only_equal_text_matches() { + let test_cases = [ + EngineTypeMatchTestCase { + description: "equal text matches", + declared: "VARCHAR", + inferred: "varchar", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a default length is not assumed", + declared: "VARCHAR", + inferred: "VARCHAR(16777216)", + expected_same_type: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + same_column_type(None, test_case.declared, test_case.inferred), + test_case.expected_same_type, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs index 990651c72b..fa45995209 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs @@ -30,6 +30,7 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; +use sqlbuild_analysis::analysis_route::main::same_column_type::same_column_type; use sqlbuild_sqltext::resolution::main::unquoted_column_name::unquoted_column_name; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; @@ -91,6 +92,7 @@ pub(crate) fn source_binding_diagnostics<'py>( && EXACT_UNQUOTED_DIALECTS.contains(&dialect.extract::()?.as_str()); let folds_upper: bool = !dialect.is_none() && dialect.extract::()?.as_str() == UPPERCASE_FOLDING_DIALECT; + let type_dialect: Option = dialect.extract()?; let mut changed: HashSet = HashSet::new(); for item in mapping_items(&columns)?.try_iter()? { let (name, value): (String, Object<'py>) = item?.extract()?; @@ -123,12 +125,17 @@ pub(crate) fn source_binding_diagnostics<'py>( }) .collect(); let known: Option> = shapes.get_item(&name)?; + let same = |declared: &str, listed: &str| { + same_column_type(type_dialect.as_deref(), declared, listed) + }; let evidence: bool = if sensitive { - adds_physical_evidence(&bound, known.as_ref(), |column| { - bound_identity(column, folds_upper) - })? + adds_physical_evidence( + &bound, + known.as_ref(), + (|column| bound_identity(column, folds_upper), same), + )? } else { - adds_physical_evidence(&physical, known.as_ref(), str::to_owned)? + adds_physical_evidence(&physical, known.as_ref(), (str::to_owned, same))? }; if evidence { changed.insert(name.clone()); @@ -424,11 +431,11 @@ fn positions<'py>( Ok(Bound::new(py, BindingPositions::new(request.extract()?)?)?.into_any()) } -/// Whether inspected columns differ from the known shape by `identity`; untyped names match any. +/// Whether inspected columns differ from the known shape by `identity` and stored type `same`. fn adds_physical_evidence( physical: &[(String, Option)], known: Option<&Object<'_>>, - identity: impl Fn(&str) -> String, + (identity, same): (impl Fn(&str) -> String, impl Fn(&str, &str) -> bool), ) -> PyResult { let Some(known) = known else { return Ok(true); @@ -461,7 +468,7 @@ fn adds_physical_evidence( .rev() .find(|(known_name, _)| known_name == name) .map(|(_, known_type)| known_type); - if known_type.is_some_and(|known_type| known_type != physical_type) { + if known_type.is_some_and(|known_type| !same(known_type, physical_type)) { return Ok(true); } } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index b237443780..1fd65c5fa5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -507,6 +507,9 @@ def timed_plan( _DRIFT_COLUMN: FakeColumn = FakeColumn(name="_LOADED_AT", data_type="TIMESTAMP_NTZ") _NUMBER_TYPE: re.Pattern[str] = re.compile(r"NUMBER\((\d+),(\d+)\)") _RELATION_TYPES: dict[str, str] = {"TABLE": "BASE TABLE", "VIEW": "VIEW"} +#: Declared string types Snowflake stores, and lists, as TEXT of its default length. +_LISTED_TYPES: dict[str, str] = {"VARCHAR": "TEXT", "STRING": "TEXT", "TEXT": "TEXT"} +_TEXT_LENGTHS: dict[str, int] = dict.fromkeys(_LISTED_TYPES, 16_777_216) @dataclass(frozen=True) @@ -533,14 +536,16 @@ def _number_column(name: str, declared: re.Match[str]) -> FakeColumn: def _column(column: dict[str, str]) -> FakeColumn: - """The warehouse column a declared column lists as, its type spelled as declared.""" + """The warehouse column a declared column lists as, its type spelled as Snowflake lists it.""" declared: str = column["data_type"].upper() number: re.Match[str] | None = _NUMBER_TYPE.fullmatch(declared) builders: dict[bool, Callable[[], FakeColumn]] = { True: lambda: _number_column(column["name"], cast(re.Match[str], number)), False: lambda: FakeColumn( - name=column["name"], data_type=declared, character_maximum_length=None + name=column["name"], + data_type=_LISTED_TYPES.get(declared, declared), + character_maximum_length=_TEXT_LENGTHS.get(declared), ), } return builders[number is not None]() diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 71117ecbe2..7b9010c111 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -223,6 +223,20 @@ def test_given_case_sensitive_snowflake_when_rebinding_listed_columns_then_unquo expected_rebinding=True, expected_codes=("B002",), ), + PhysicalEvidenceCase( + "Snowflake default length and precision spellings of the declared types", + SnowflakeAdapter, + "snowflake", + (("order_id", "NUMBER(38,0)"), ("status", "VARCHAR(16777216)")), + expected_rebinding=False, + ), + PhysicalEvidenceCase( + "Snowflake shorter listed VARCHAR", + SnowflakeAdapter, + "snowflake", + (("order_id", "NUMBER(38,0)"), ("status", "VARCHAR(10)")), + expected_rebinding=True, + ), PhysicalEvidenceCase( "Snowflake changed column type", SnowflakeAdapter, From 20b0782c6a887f86c2a0e20784effd79de2038c7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:46:40 +0000 Subject: [PATCH 08/17] test(plan): rebind the quoted-case session case on a retyped source column Physical rebinding now analyses a source's readers again only when its listed columns differ from the declared ones, so the session-setting case listed the declared column unchanged and saw nothing to rebind; the plan still stopped on the compile-time B002. List the column with a different length so the case rebinds and checks the setting again. --- .../src/sqlbuild/compiler/pipeline/test_semantic_triage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index ad9eeb940c..f4656ffeb9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -104,7 +104,7 @@ def test_given_effective_target_when_binding_quotes_then_honours_session_setting ) physical: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, - columns={"orders": (ColumnInfo(name="customer", type="VARCHAR"),)}, + columns={"orders": (ColumnInfo(name="customer", type="VARCHAR(10)"),)}, profile=ExpressionInferenceProfile(sql_analysis_dialect="snowflake"), selected_keys=frozenset(model.key for model in project.models), ) From ca9468b22e71ce5bb578da772be004ba014d8907 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:01:16 +0000 Subject: [PATCH 09/17] perf(plan): analyse readers again only when a source's new columns can change them When a source's warehouse columns differed from its declared ones, planning analysed every selected model reading it again, and every model downstream of those. Warehouses routinely carry columns a source does not declare, so on a real project this meant re-analysing about 1,600 models on every plan, almost all of them unchanged. When the listing only adds columns, every declared column present with the type it stores, analyse a reader again only if the new columns can change its result: it projects the source's columns through a star, directly or through CTEs, subqueries and set operations; it binds an unqualified name in a scope where the source sits beside another relation; it names one of the added columns anywhere; or it uses a construct the walk does not follow, or SQL the parser cannot read. Everything else keeps its compile-time analysis. A re-analysed model passes on only the difference its own new shape makes, so its readers are judged the same way, and a model whose shape did not change stops the cascade. A missing or retyped column still re-analyses every reader. On a 10K-model fixture with one source in fifty carrying an extra column, re-analysis drops from 519 models to 52; the guard's expected range follows. --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/source_drift/_helpers/mod.rs | 1 + .../src/source_drift/_helpers/walk.rs | 340 ++++++++++++++++++ .../src/source_drift/constants.rs | 8 + .../src/source_drift/main/mod.rs | 1 + .../source_drift/main/reads_added_columns.rs | 44 +++ .../sqlbuild-analysis/src/source_drift/mod.rs | 8 + .../src/source_drift/tests.rs | 4 + .../tests/test_reads_added_columns.rs | 104 ++++++ .../src/source_drift/tests/test_types.rs | 6 + .../_helpers/source_bindings.rs | 286 +++++++++------ .../test_duckshop_10k_preview_performance.py | 6 +- .../sqlbuild/compiler/pipeline/_test_types.py | 9 + .../pipeline/test_native_binding_catalog.py | 114 +++++- 14 files changed, 825 insertions(+), 107 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/source_drift/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs create mode 100644 crates/sqlbuild-analysis/src/source_drift/constants.rs create mode 100644 crates/sqlbuild-analysis/src/source_drift/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/source_drift/main/reads_added_columns.rs create mode 100644 crates/sqlbuild-analysis/src/source_drift/mod.rs create mode 100644 crates/sqlbuild-analysis/src/source_drift/tests.rs create mode 100644 crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs create mode 100644 crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 848c4de883..ca5a7c5549 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -46,6 +46,7 @@ pub mod semantic_checks; pub mod semantic_usage; pub mod semantic_validation; pub mod snowflake_cost; +pub mod source_drift; pub mod sql_macros; pub mod sql_test_attachment; pub mod sql_tokens; diff --git a/crates/sqlbuild-analysis/src/source_drift/_helpers/mod.rs b/crates/sqlbuild-analysis/src/source_drift/_helpers/mod.rs new file mode 100644 index 0000000000..208de6bb67 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod walk; diff --git a/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs b/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs new file mode 100644 index 0000000000..04078968dd --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs @@ -0,0 +1,340 @@ +//! The walk deciding whether new columns on some relations could change a query's result. + +use std::collections::HashSet; + +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlquack_syntax::expressions::queries::models::{JoinKind, Select, With}; +use sqlquack_syntax::{Expression, ExpressionWalk}; + +use crate::source_drift::constants::{COUNT_FUNCTION, OPAQUE_KINDS, SHARED_SCOPE_RELATIONS}; + +/// The relations that gained columns and the names they gained, casefolded. +pub(crate) struct Drift<'a> { + pub(crate) relations: &'a HashSet, + pub(crate) added: &'a HashSet, +} + +/// One query scope's visible relations: whether each exposes a drifted relation's columns. +type Visible = Vec; + +/// CTE names in scope and whether each exposes a drifted relation's columns, innermost last. +type Ctes = Vec<(String, bool)>; + +pub(crate) struct Walk<'a> { + drift: Drift<'a>, + /// Set once anything shows the result could change. + pub(crate) affected: bool, +} + +impl<'a> Walk<'a> { + pub(crate) fn new(drift: Drift<'a>) -> Self { + Self { + drift, + affected: false, + } + } + + /// Mark `statement` affected when new columns could change its result. + pub(crate) fn statement(&mut self, statement: &Expression) { + self.names(statement); + if self.query(statement, &Vec::new(), &Vec::new()) { + self.affected = true; + } + } + + /// Any construct whose meaning depends on every column, and any name a new column could take. + fn names(&mut self, statement: &Expression) { + for node in statement.dfs() { + let name: Option<&str> = match node { + Expression::Column(column) => Some(column.name.name.as_str()), + Expression::Identifier(identifier) => Some(identifier.name.as_str()), + Expression::Pivot(_) + | Expression::Unpivot(_) + | Expression::PipeOperator(_) + | Expression::With(_) + | Expression::Cte(_) => { + self.affected = true; + None + } + _ if OPAQUE_KINDS.contains(&node.variant_name()) => { + self.affected = true; + None + } + _ => None, + }; + if name.is_some_and(|name| self.drift.added.contains(&python_casefold(name))) { + self.affected = true; + } + } + } + + /// Whether `query`'s output columns include a drifted relation's columns, walking its scopes. + fn query(&mut self, query: &Expression, ctes: &Ctes, outer: &Visible) -> bool { + match query { + Expression::Select(select) => self.select((query, select), ctes, outer), + Expression::Union(union) => self.set_operation( + (&union.left, &union.right), + union.with.as_ref(), + ctes, + outer, + ), + Expression::Intersect(intersect) => self.set_operation( + (&intersect.left, &intersect.right), + intersect.with.as_ref(), + ctes, + outer, + ), + Expression::Except(except) => self.set_operation( + (&except.left, &except.right), + except.with.as_ref(), + ctes, + outer, + ), + Expression::Subquery(subquery) => self.query(&subquery.this, ctes, outer), + Expression::Paren(paren) => self.query(&paren.this, ctes, outer), + other => { + let mut exposes: bool = false; + for nested in nested_queries(other) { + exposes |= self.query(nested, ctes, outer); + } + exposes + } + } + } + + /// A set operation pairs columns by position, so a branch exposing new columns changes it. + fn set_operation( + &mut self, + (left, right): (&Expression, &Expression), + with: Option<&With>, + ctes: &Ctes, + outer: &Visible, + ) -> bool { + let ctes: Ctes = self.with(with, ctes, outer); + let exposes: bool = self.query(left, &ctes, outer) | self.query(right, &ctes, outer); + if exposes { + self.affected = true; + } + exposes + } + + fn with(&mut self, with: Option<&With>, ctes: &Ctes, outer: &Visible) -> Ctes { + let mut scoped: Ctes = ctes.clone(); + for cte in with.map(|with| with.ctes.as_slice()).unwrap_or_default() { + let exposes: bool = self.query(&cte.this, &scoped, outer); + scoped.push((python_casefold(&cte.alias.name), exposes)); + } + scoped + } + + fn select( + &mut self, + (node, select): (&Expression, &Select), + ctes: &Ctes, + outer: &Visible, + ) -> bool { + let ctes: Ctes = self.with(select.with.as_ref(), ctes, outer); + let mut walked: HashSet<*const Expression> = select + .with + .iter() + .flat_map(|with| with.ctes.iter()) + .map(|cte| std::ptr::from_ref(&cte.this)) + .collect(); + let mut local: Vec<(Option, bool)> = Vec::new(); + let relations = select + .from + .iter() + .flat_map(|from| from.expressions.iter()) + .chain(select.joins.iter().map(|join| &join.this)); + for relation in relations { + walked.insert(std::ptr::from_ref(relation)); + local.extend(self.relation(relation, &ctes, outer)); + } + let drifted_here: bool = local.iter().any(|(_, exposes)| *exposes); + for join in &select.joins { + let natural: bool = matches!( + join.kind, + JoinKind::Natural + | JoinKind::NaturalLeft + | JoinKind::NaturalRight + | JoinKind::NaturalFull + ); + let using_added: bool = join + .using + .iter() + .any(|name| self.drift.added.contains(&python_casefold(&name.name))); + if (natural && drifted_here) || using_added { + self.affected = true; + } + } + let visible: Visible = outer + .iter() + .copied() + .chain(local.iter().map(|(_, exposes)| *exposes)) + .collect(); + let shared: bool = + visible.len() >= SHARED_SCOPE_RELATIONS && visible.iter().any(|exposes| *exposes); + let mut star: bool = false; + let mut unqualified: bool = false; + let mut nested: Vec<&Expression> = Vec::new(); + let mut pending: Vec<(&Expression, bool)> = Vec::new(); + for item in &select.expressions { + pending.push((item, true)); + } + let projected: HashSet<*const Expression> = + select.expressions.iter().map(std::ptr::from_ref).collect(); + for child in node.children() { + if !walked.contains(&std::ptr::from_ref(child)) + && !projected.contains(&std::ptr::from_ref(child)) + { + pending.push((child, false)); + } + } + while let Some((node, in_projection)) = pending.pop() { + match node { + Expression::Select(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) + | Expression::Subquery(_) => nested.push(node), + Expression::Star(expanded) if in_projection => { + star |= match &expanded.table { + None => drifted_here, + Some(qualifier) => { + let qualifier: String = python_casefold(&qualifier.name); + local + .iter() + .find(|(alias, _)| alias.as_deref() == Some(qualifier.as_str())) + .is_none_or(|(_, exposes)| *exposes) + } + }; + } + Expression::BracedWildcard(_) if in_projection => star |= drifted_here, + Expression::Count(_) + | Expression::AggregateFunction(_) + | Expression::Function(_) + if counts_rows(node) => + { + pending.extend( + node.children() + .into_iter() + .filter(|child| !matches!(child, Expression::Star(_))) + .map(|child| (child, in_projection)), + ); + } + Expression::Column(column) if column.table.is_none() => unqualified = true, + _ => pending.extend( + node.children() + .into_iter() + .map(|child| (child, in_projection)), + ), + } + } + if shared && unqualified { + self.affected = true; + } + for query in nested { + if self.query(query, &ctes, &visible) { + self.affected = true; + } + } + star + } + + /// Whether each relation a `FROM` item brings into scope exposes a drifted relation's columns. + fn relation( + &mut self, + relation: &Expression, + ctes: &Ctes, + outer: &Visible, + ) -> Vec<(Option, bool)> { + match relation { + Expression::Table(table) => { + let alias: String = + python_casefold(&table.alias.as_ref().unwrap_or(&table.name).name); + let name: String = python_casefold(&table.name.name); + let qualified: String = python_casefold( + &[&table.catalog, &table.schema] + .into_iter() + .flatten() + .map(|part| part.name.as_str()) + .chain([table.name.name.as_str()]) + .collect::>() + .join("."), + ); + let unqualified: bool = table.schema.is_none() && table.catalog.is_none(); + let cte: Option = unqualified + .then(|| { + ctes.iter() + .rev() + .find(|(cte, _)| *cte == name) + .map(|(_, exposes)| *exposes) + }) + .flatten(); + vec![( + Some(alias), + cte.unwrap_or_else(|| { + self.drift.relations.contains(&name) + || self.drift.relations.contains(&qualified) + }), + )] + } + Expression::Subquery(subquery) => vec![( + subquery + .alias + .as_ref() + .map(|alias| python_casefold(&alias.name)), + self.query(&subquery.this, ctes, outer), + )], + other => { + let mut found: Vec<(Option, bool)> = Vec::new(); + let mut pending: Vec<&Expression> = other.children(); + while let Some(node) = pending.pop() { + match node { + Expression::Table(_) | Expression::Subquery(_) => { + found.extend(self.relation(node, ctes, outer)); + } + Expression::Select(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) => { + found.push((None, self.query(node, ctes, outer))) + } + _ => pending.extend(node.children()), + } + } + if found.is_empty() { + found.push((None, false)); + } + found + } + } + } +} + +/// Whether `node` is `COUNT`, whose `*` counts rows rather than listing columns. +fn counts_rows(node: &Expression) -> bool { + match node { + Expression::Count(_) => true, + Expression::AggregateFunction(call) => call.name.eq_ignore_ascii_case(COUNT_FUNCTION), + Expression::Function(call) => call.name.eq_ignore_ascii_case(COUNT_FUNCTION), + _ => false, + } +} + +/// The outermost queries nested anywhere inside `expression`. +fn nested_queries(expression: &Expression) -> Vec<&Expression> { + let mut found: Vec<&Expression> = Vec::new(); + let mut pending: Vec<&Expression> = expression.children(); + while let Some(node) = pending.pop() { + match node { + Expression::Select(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) + | Expression::Subquery(_) => found.push(node), + _ => pending.extend(node.children()), + } + } + found +} diff --git a/crates/sqlbuild-analysis/src/source_drift/constants.rs b/crates/sqlbuild-analysis/src/source_drift/constants.rs new file mode 100644 index 0000000000..7ca9ea23c5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/constants.rs @@ -0,0 +1,8 @@ +//! Constants of the added-column walk. + +/// Node kinds the parser keeps as unparsed text, whose columns the walk cannot see. +pub(crate) const OPAQUE_KINDS: [&str; 2] = ["command", "raw"]; +/// The aggregate whose `*` argument counts rows. +pub(crate) const COUNT_FUNCTION: &str = "COUNT"; +/// Relations in one scope from which an unqualified name could bind to either of two. +pub(crate) const SHARED_SCOPE_RELATIONS: usize = 2; diff --git a/crates/sqlbuild-analysis/src/source_drift/main/mod.rs b/crates/sqlbuild-analysis/src/source_drift/main/mod.rs new file mode 100644 index 0000000000..5daf588a3c --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/main/mod.rs @@ -0,0 +1 @@ +pub mod reads_added_columns; diff --git a/crates/sqlbuild-analysis/src/source_drift/main/reads_added_columns.rs b/crates/sqlbuild-analysis/src/source_drift/main/reads_added_columns.rs new file mode 100644 index 0000000000..6fdbf7c388 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/main/reads_added_columns.rs @@ -0,0 +1,44 @@ +//! `reads_added_columns`: whether a model must be analysed again when its inputs gain columns. + +use std::collections::HashSet; + +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlquack_syntax::Parsed; + +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; +use crate::source_drift::_helpers::walk::{Drift, Walk}; +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; + +/// Whether `query_sql` could return something else once the `drifted` relations gain `added`. +#[must_use] +pub fn reads_added_columns( + dialect: &str, + (query_sql, placeholders): (&str, &[(String, String)]), + drifted: &[String], + added: &[String], +) -> bool { + let AnalysisRoute::Native(native) = analysis_route(Some(dialect)) else { + return true; + }; + let Ok(normalized) = normalize_native_sql(query_sql, native.name(), native.stub_prefix()) + else { + return true; + }; + let sql: String = placeholder_defaults(&normalized, placeholders); + let Ok(Parsed { statements, .. }) = native.parse(&sql) else { + return true; + }; + let [statement] = statements.as_slice() else { + return true; + }; + let relations: HashSet = drifted.iter().map(|name| python_casefold(name)).collect(); + let added: HashSet = added.iter().map(|name| python_casefold(name)).collect(); + let mut walk: Walk<'_> = Walk::new(Drift { + relations: &relations, + added: &added, + }); + walk.statement(statement); + walk.affected +} diff --git a/crates/sqlbuild-analysis/src/source_drift/mod.rs b/crates/sqlbuild-analysis/src/source_drift/mod.rs new file mode 100644 index 0000000000..f484bfc4da --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/mod.rs @@ -0,0 +1,8 @@ +//! Whether columns a warehouse added to a model's inputs could change what the model returns. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/source_drift/tests.rs b/crates/sqlbuild-analysis/src/source_drift/tests.rs new file mode 100644 index 0000000000..3958450839 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_reads_added_columns.rs"] +mod test_reads_added_columns; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs b/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs new file mode 100644 index 0000000000..df91ab1e75 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs @@ -0,0 +1,104 @@ +use crate::source_drift::main::reads_added_columns::reads_added_columns; +use crate::source_drift::tests::test_types::AddedColumnsTestCase; + +#[test] +fn given_a_source_gaining_columns_when_reading_models_then_only_column_list_readers_are_affected() { + let test_cases = [ + AddedColumnsTestCase { + description: "a star over the source", + sql: r#"SELECT * FROM __source("orders")"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "explicit columns from the source alone", + sql: r#"SELECT order_id, status FROM __source("orders")"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "an unqualified name in a join with the source", + sql: r#"SELECT order_id FROM __source("orders") AS o + JOIN __ref("customers") AS c ON o.customer_id = c.customer_id"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "qualified names in a join with the source", + sql: r#"SELECT o.order_id, c.customer_name FROM __source("orders") AS o + JOIN __ref("customers") AS c ON o.customer_id = c.customer_id"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "a star reaching the output through a CTE", + sql: r#"WITH s AS (SELECT * FROM __source("orders")) SELECT * FROM s"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a star through a subquery", + sql: r#"SELECT * FROM (SELECT * FROM __source("orders")) AS s"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a starred CTE read by explicit columns", + sql: r#"WITH source_rows AS (SELECT * FROM __source("orders")), + final AS (SELECT s.order_id, s.status FROM source_rows AS s) + SELECT * FROM final"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "a starred CTE joined with an unqualified name", + sql: r#"WITH s AS (SELECT * FROM __source("orders")) + SELECT customer_name FROM s JOIN __ref("customers") AS c ON s.customer_id = c.customer_id"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a set operation with a starred branch", + sql: r#"SELECT * FROM __source("orders") UNION ALL SELECT * FROM __source("orders")"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a name the source gains, qualified", + sql: r#"SELECT o.order_id FROM __source("orders") AS o WHERE o.region = 'EU'"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "an unqualified name a correlated subquery may now bind to the source", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c + WHERE EXISTS (SELECT 1 FROM __source("orders") AS o WHERE o.customer_id = customer_id)"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a count of every row", + sql: r#"SELECT COUNT(*) AS orders FROM __source("orders")"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "a star over another relation in a join with the source", + sql: r#"SELECT c.*, o.order_id FROM __ref("customers") AS c + JOIN __source("orders") AS o ON o.customer_id = c.customer_id"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "a star over the source in a join", + sql: r#"SELECT o.*, c.customer_name FROM __ref("customers") AS c + JOIN __source("orders") AS o ON o.customer_id = c.customer_id"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "SQL the parser cannot read", + sql: r#"SELECT FROM WHERE __source("orders")"#, + expected_affected: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + reads_added_columns( + "snowflake", + (test_case.sql, &[]), + &["orders".to_owned()], + &["region".to_owned()], + ), + test_case.expected_affected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs b/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs new file mode 100644 index 0000000000..daf66cf1a3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs @@ -0,0 +1,6 @@ +/// A model query, and whether `orders` gaining a `region` column could change its result. +pub(super) struct AddedColumnsTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_affected: bool, +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs index fa45995209..48d875394f 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs @@ -2,7 +2,7 @@ use crate::semantic_shapes::main::declared_types::declared_types; use crate::semantic_shapes::main::function_names::function_names; -use std::collections::HashSet; +use std::collections::{BTreeSet, HashMap, HashSet}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -19,6 +19,8 @@ use crate::semantic_shapes::_helpers::shapes::model_inputs; use crate::semantic_shapes::_helpers::shapes::project_shapes; use crate::semantic_shapes::_helpers::shapes::published_shape; use crate::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; +use sqlbuild_analysis::analysis_route::main::same_column_type::same_column_type; +use sqlbuild_analysis::source_drift::main::reads_added_columns::reads_added_columns; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; @@ -30,7 +32,6 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; -use sqlbuild_analysis::analysis_route::main::same_column_type::same_column_type; use sqlbuild_sqltext::resolution::main::unquoted_column_name::unquoted_column_name; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; @@ -92,8 +93,12 @@ pub(crate) fn source_binding_diagnostics<'py>( && EXACT_UNQUOTED_DIALECTS.contains(&dialect.extract::()?.as_str()); let folds_upper: bool = !dialect.is_none() && dialect.extract::()?.as_str() == UPPERCASE_FOLDING_DIALECT; - let type_dialect: Option = dialect.extract()?; - let mut changed: HashSet = HashSet::new(); + let comparison: Comparison = Comparison { + dialect: dialect.extract()?, + sensitive, + folds_upper, + }; + let mut changed: HashMap = HashMap::new(); for item in mapping_items(&columns)?.try_iter()? { let (name, value): (String, Object<'py>) = item?.extract()?; if !value.is_truthy()? { @@ -125,20 +130,9 @@ pub(crate) fn source_binding_diagnostics<'py>( }) .collect(); let known: Option> = shapes.get_item(&name)?; - let same = |declared: &str, listed: &str| { - same_column_type(type_dialect.as_deref(), declared, listed) - }; - let evidence: bool = if sensitive { - adds_physical_evidence( - &bound, - known.as_ref(), - (|column| bound_identity(column, folds_upper), same), - )? - } else { - adds_physical_evidence(&physical, known.as_ref(), (str::to_owned, same))? - }; - if evidence { - changed.insert(name.clone()); + let listed: &[(String, Option)] = if sensitive { &bound } else { &physical }; + if let Some(drift) = comparison.drift(listed, known.as_ref())? { + changed.insert(name.clone(), drift); } let quoted: Bound<'py, PyDict> = PyDict::new(py); for (column, column_type) in bound { @@ -158,7 +152,7 @@ pub(crate) fn source_binding_diagnostics<'py>( &project, &request.get("selected_keys")?, (&changed, &shapes), - &profile, + (&profile, &comparison), )?; if !physical_catalog.is_none() { physical_catalog = physical_catalog.call_method1("with_relations", (&shapes,))?; @@ -166,40 +160,18 @@ pub(crate) fn source_binding_diagnostics<'py>( validated_diagnostics(&project, &models, (&shapes, &physical_catalog), &dialect) } -/// Selected models reading changed sources, re-analysed in dependency order; shapes updated. +/// Selected models whose result inspected columns could change, re-analysed in dependency order. fn rebound_models<'py>( project: &Object<'py>, selected_keys: &Object<'py>, - (changed, shapes): (&HashSet, &Bound<'py, PyDict>), - profile: &Object<'py>, + (changed, shapes): (&HashMap, &Bound<'py, PyDict>), + (profile, comparison): (&Object<'py>, &Comparison), ) -> PyResult>> { let py: Python<'py> = project.py(); - let by_name: Bound<'py, PyDict> = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { - let model: Object<'py> = model?; - by_name.set_item(native_field(&model, "name")?, &model)?; - } - let graph: Bound<'py, PyDict> = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { - let model: Object<'py> = model?; - let mut upstream: Vec> = Vec::new(); - for reference in native_field(&model, "references")?.try_iter()? { - let reference: Object<'py> = reference?; - let name: Object<'py> = native_field(&reference, "ref_name")?; - if native_field(&reference, "ref_kind")?.eq(REF_KIND)? && by_name.contains(&name)? { - upstream.push(name); - } - } - graph.set_item(native_field(&model, "name")?, PyTuple::new(py, upstream)?)?; - } - let order: Object<'py> = py - .import(GRAPHLIB_MODULE)? - .getattr("TopologicalSorter")? - .call1((graph,))? - .call_method0("static_order")?; + let (by_name, order) = dependency_order(project)?; let query_class: Object<'py> = record_class(py, RecordId::NativeColumnQuery)?; let unknown: Object<'py> = enum_member(py, RecordId::InferredNullability, "UNKNOWN")?; - let mut affected: HashSet = HashSet::new(); + let mut affected: HashMap = HashMap::new(); let mut models: Vec> = Vec::new(); for name in order.try_iter()? { let model: Object<'py> = by_name.as_any().get_item(name?)?; @@ -210,23 +182,11 @@ fn rebound_models<'py>( { continue; } - let mut reads_changed: bool = false; - for reference in native_field(&model, "references")?.try_iter()? { - let reference: Object<'py> = reference?; - let kind: String = native_field(&reference, "ref_kind")?.str()?.extract()?; - let upstream: String = native_field(&reference, "ref_name")?.extract()?; - if (kind == SOURCE_KIND && changed.contains(&upstream)) - || (kind == REF_KIND && affected.contains(&upstream)) - { - reads_changed = true; - break; - } - } - if !reads_changed { + if !needs_reanalysis(&model, (changed, &affected), comparison)? { continue; } let model_name: String = native_field(&model, "name")?.extract()?; - affected.insert(model_name.clone()); + let known: Option> = shapes.get_item(&model_name)?; models.push(model.clone()); let inputs: Bound<'py, PyDict> = model_inputs(&model, shapes)?; let nullability: Bound<'py, PyDict> = PyDict::new(py); @@ -279,17 +239,86 @@ fn rebound_models<'py>( }; columns.set_item(native_field(&column, "name")?, column_type)?; } - shapes.set_item( - &model_name, - published_shape(&model, &columns, &inputs, profile)?, - )?; - } else if shapes.contains(&model_name)? { - shapes.del_item(&model_name)?; + let shape: Bound<'py, PyDict> = published_shape(&model, &columns, &inputs, profile)?; + let listed: Vec<(String, Option)> = mapping_items(&shape)? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()?; + if let Some(drift) = comparison.drift(&listed, known.as_ref())? { + affected.insert(model_name.clone(), drift); + } + shapes.set_item(&model_name, shape)?; + } else { + affected.insert(model_name.clone(), Drift::Changed); + if shapes.contains(&model_name)? { + shapes.del_item(&model_name)?; + } } } Ok(models) } +/// The project's models by name, and their names in dependency order. +fn dependency_order<'py>(project: &Object<'py>) -> PyResult<(Bound<'py, PyDict>, Object<'py>)> { + let py: Python<'py> = project.py(); + let by_name: Bound<'py, PyDict> = PyDict::new(py); + for model in native_field(project, "models")?.try_iter()? { + let model: Object<'py> = model?; + by_name.set_item(native_field(&model, "name")?, &model)?; + } + let graph: Bound<'py, PyDict> = PyDict::new(py); + for model in native_field(project, "models")?.try_iter()? { + let model: Object<'py> = model?; + let mut upstream: Vec> = Vec::new(); + for reference in native_field(&model, "references")?.try_iter()? { + let reference: Object<'py> = reference?; + let name: Object<'py> = native_field(&reference, "ref_name")?; + if native_field(&reference, "ref_kind")?.eq(REF_KIND)? && by_name.contains(&name)? { + upstream.push(name); + } + } + graph.set_item(native_field(&model, "name")?, PyTuple::new(py, upstream)?)?; + } + let order: Object<'py> = py + .import(GRAPHLIB_MODULE)? + .getattr("TopologicalSorter")? + .call1((graph,))? + .call_method0("static_order")?; + Ok((by_name, order)) +} + +/// Whether `model` reads a changed input, or inputs whose added columns could change its result. +fn needs_reanalysis( + model: &Object<'_>, + (changed, affected): (&HashMap, &HashMap), + comparison: &Comparison, +) -> PyResult { + let mut drifted: Vec = Vec::new(); + let mut added: BTreeSet = BTreeSet::new(); + for reference in native_field(model, "references")?.try_iter()? { + let reference: Object<'_> = reference?; + let kind: String = native_field(&reference, "ref_kind")?.str()?.extract()?; + let upstream: String = native_field(&reference, "ref_name")?.extract()?; + let drift: Option<&Drift> = match kind.as_str() { + SOURCE_KIND => changed.get(&upstream), + REF_KIND => affected.get(&upstream), + _ => None, + }; + match drift { + Some(Drift::Changed) => return Ok(true), + Some(Drift::Added(names)) => { + drifted.push(upstream); + added.extend(names.iter().cloned()); + } + None => {} + } + } + if drifted.is_empty() { + return Ok(false); + } + comparison.reads_added(model, &drifted, &added.into_iter().collect::>()) +} + /// Each rebound model's binding diagnostics, located in its authored file. fn validated_diagnostics<'py>( project: &Object<'py>, @@ -431,48 +460,99 @@ fn positions<'py>( Ok(Bound::new(py, BindingPositions::new(request.extract()?)?)?.into_any()) } -/// Whether inspected columns differ from the known shape by `identity` and stored type `same`. -fn adds_physical_evidence( - physical: &[(String, Option)], - known: Option<&Object<'_>>, - (identity, same): (impl Fn(&str) -> String, impl Fn(&str, &str) -> bool), -) -> PyResult { - let Some(known) = known else { - return Ok(true); - }; - let known: Vec<(String, String)> = mapping_items(known)? - .try_iter()? - .map(|item| { - let (name, column_type): (String, String) = item?.extract()?; - Ok((identity(&name), column_type)) - }) - .collect::>()?; - let physical: Vec<(String, Option<&str>)> = physical - .iter() - .map(|(name, column_type)| (identity(name), column_type.as_deref())) - .collect(); - let physical_names: HashSet<&str> = physical.iter().map(|(name, _)| name.as_str()).collect(); - let known_names: HashSet<&str> = known.iter().map(|(name, _)| name.as_str()).collect(); - if physical_names != known_names || physical_names.len() != physical.len() { - return Ok(true); +/// How a relation's columns differ from what compiling assumed of them. +enum Drift { + /// Every known column is there as it was, and these columns are new. + Added(Vec), + /// A known column is missing, retyped or listed twice, or nothing was known. + Changed, +} + +/// How this dialect binds and stores columns, to compare listed columns with known ones. +struct Comparison { + dialect: Option, + /// Whether names bind case-sensitively, compared by `bound_identity`. + sensitive: bool, + folds_upper: bool, +} + +impl Comparison { + fn identity(&self, name: &str) -> String { + if self.sensitive { + bound_identity(name, self.folds_upper) + } else { + name.to_owned() + } } - for (name, physical_type) in &physical { - let Some(physical_type) = *physical_type else { - continue; + + /// How `listed` columns differ from the `known` shape; `None` when they bind and store alike. + fn drift( + &self, + listed: &[(String, Option)], + known: Option<&Object<'_>>, + ) -> PyResult> { + let Some(known) = known else { + return Ok(Some(Drift::Changed)); }; - if physical_type.is_empty() { - continue; + let known: HashMap = mapping_items(known)? + .try_iter()? + .map(|item| { + let (name, column_type): (String, String) = item?.extract()?; + Ok((self.identity(&name), column_type)) + }) + .collect::>()?; + let mut seen: HashSet = HashSet::new(); + let mut added: Vec = Vec::new(); + for (name, listed_type) in listed { + let identity: String = self.identity(name); + if !seen.insert(identity.clone()) { + return Ok(Some(Drift::Changed)); + } + match known.get(&identity) { + None => added.push(identity), + Some(known_type) => { + let listed_type: &str = listed_type.as_deref().unwrap_or_default(); + if !listed_type.is_empty() + && !same_column_type(self.dialect.as_deref(), known_type, listed_type) + { + return Ok(Some(Drift::Changed)); + } + } + } } - let known_type: Option<&String> = known - .iter() - .rev() - .find(|(known_name, _)| known_name == name) - .map(|(_, known_type)| known_type); - if known_type.is_some_and(|known_type| !same(known_type, physical_type)) { - return Ok(true); + if known.keys().any(|name| !seen.contains(name)) { + return Ok(Some(Drift::Changed)); } + Ok((!added.is_empty()).then_some(Drift::Added(added))) + } + + /// Whether `model`'s result could change once the `drifted` relations gain `added` columns. + fn reads_added( + &self, + model: &Object<'_>, + drifted: &[String], + added: &[String], + ) -> PyResult { + let Some(dialect) = self.dialect.as_deref() else { + return Ok(true); + }; + let query_sql: String = native_field(model, "query_sql")?.extract()?; + let placeholders: Object<'_> = placeholders(model)?; + let placeholders: Vec<(String, String)> = if placeholders.is_none() { + Vec::new() + } else { + mapping_items(&placeholders)? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()? + }; + Ok(reads_added_columns( + dialect, + (&query_sql, &placeholders), + drifted, + added, + )) } - Ok(false) } /// `{str(key): str(value)}` of a dict `placeholders` config, else None. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py index a17369cfdb..f43f7d80ba 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py @@ -61,7 +61,7 @@ expected_max_warm_to_publish=0.45, expected_max_check_seconds=0.5, expected_max_live_to_warm=6.0, - expected_source_rebinding=(300, 1_000), + expected_source_rebinding=(25, 150), expected_max_warehouse_statements=1_150, ), PreviewPerformanceTestCase( @@ -73,7 +73,7 @@ expected_max_warm_to_publish=0.45, expected_max_check_seconds=0.5, expected_max_live_to_warm=6.0, - expected_source_rebinding=(300, 1_000), + expected_source_rebinding=(25, 150), expected_max_warehouse_statements=1_150, ), ), @@ -169,7 +169,7 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then test_case.expected_source_rebinding[0] <= live.source_rebinding_analyses <= test_case.expected_source_rebinding[1] - ), "only models reading a source whose warehouse columns differ are analysed again" + ), "only models whose result a source's new or changed columns can change are analysed again" assert live.cursor_statements == -(-live.cursor_relations // _CURSOR_BATCH_SIZE), ( "cursor bounds are read in full batches" ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index 754aff80de..be3b05377d 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -74,6 +74,15 @@ class PhysicalEvidenceCase: expected_codes: tuple[str, ...] = () +@dataclass(frozen=True) +class AddedColumnsCase: + """Models over a source the warehouse lists with one more column, and which are re-analysed.""" + + description: str + models: tuple[tuple[str, str], ...] + expected_reanalysed: int + + @dataclass(frozen=True) class SemanticCompileCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 7b9010c111..46abce3a20 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -20,12 +20,17 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + AddedColumnsCase, NativeCatalogCase, PhysicalEvidenceCase, ValidationCacheCase, ) _DECLARED_ORDERS: tuple[tuple[str, str], ...] = (("order_id", "INTEGER"), ("status", "VARCHAR")) +_CUSTOMERS_MODEL: tuple[str, str] = ( + "customers", + "SELECT 1 AS customer_id, 'Ada' AS customer_name", +) @pytest.mark.parametrize( @@ -209,11 +214,19 @@ def test_given_case_sensitive_snowflake_when_rebinding_listed_columns_then_unquo expected_rebinding=False, ), PhysicalEvidenceCase( - "Snowflake new listed column", + "Snowflake new listed column an explicit reader cannot see", SnowflakeAdapter, "snowflake", (*_DECLARED_ORDERS, ("region", "VARCHAR")), + expected_rebinding=False, + ), + PhysicalEvidenceCase( + "Snowflake declared column missing from the listing", + SnowflakeAdapter, + "snowflake", + (("order_id", "INTEGER"), ("region", "VARCHAR")), expected_rebinding=True, + expected_codes=("B002",), ), PhysicalEvidenceCase( "Snowflake lowercase stored name rejects unquoted reader", @@ -300,6 +313,105 @@ def test_given_case_sensitive_dialect_when_listed_columns_bind_as_declared_then_ assert tuple(item.code for item in diagnostics) == test_case.expected_codes +@pytest.mark.parametrize( + "test_case", + [ + AddedColumnsCase( + "a star reader is analysed again", + (("report", 'SELECT * FROM __source("orders")'),), + expected_reanalysed=1, + ), + AddedColumnsCase( + "an explicit reader of the source alone keeps its analysis", + (("report", 'SELECT order_id, status FROM __source("orders")'),), + expected_reanalysed=0, + ), + AddedColumnsCase( + "an unqualified name in a join with the source is analysed again", + ( + _CUSTOMERS_MODEL, + ( + "report", + 'SELECT order_id, customer_name FROM __source("orders") AS o ' + 'JOIN __ref("customers") AS c ON o.order_id = c.customer_id', + ), + ), + expected_reanalysed=1, + ), + AddedColumnsCase( + "qualified names in a join with the source keep their analysis", + ( + _CUSTOMERS_MODEL, + ( + "report", + 'SELECT o.order_id, c.customer_name FROM __source("orders") AS o ' + 'JOIN __ref("customers") AS c ON o.order_id = c.customer_id', + ), + ), + expected_reanalysed=0, + ), + AddedColumnsCase( + "an explicit reader of a star reader keeps its analysis", + ( + ("staged", 'SELECT * FROM __source("orders")'), + ("report", 'SELECT order_id FROM __ref("staged")'), + ), + expected_reanalysed=1, + ), + AddedColumnsCase( + "a star reader of a star reader is analysed again", + ( + ("staged", 'SELECT * FROM __source("orders")'), + ("report", 'SELECT * FROM __ref("staged")'), + ), + expected_reanalysed=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_source_gaining_columns_when_rebinding_then_reanalyses_models_they_can_change( + test_case: AddedColumnsCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n" + " contract: enforced\n columns:\n" + + "".join( + f" - name: {name}\n type: {kind}\n" for name, kind in _DECLARED_ORDERS + ) + ) + for name, sql in test_case.models: + (tmp_path / f"models/{name}.sql").write_text( + f"MODEL (description 'Test model {name}.', database warehouse, schema analytics);\n" + + sql + ) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=SnowflakeAdapter() + ) + _ = sql_analysis_models.compile_stage_work_info(reset=True) + + diagnostics: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( + project=project, + columns={ + "orders": tuple( + ColumnInfo(name, kind) for name, kind in (*_DECLARED_ORDERS, ("region", "VARCHAR")) + ) + }, + profile=ExpressionInferenceProfile(sql_analysis_dialect="snowflake"), + selected_keys=frozenset(model.key for model in project.models), + ) + + assert ( + sql_analysis_models.compile_stage_work_info().get("source_rebinding_analyses", 0) + == test_case.expected_reanalysed + ) + assert diagnostics == () + + @pytest.mark.parametrize( "test_case", [ From ddb023ab4cca749589cce6d18924c716c2efacf6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 02:23:01 +0000 Subject: [PATCH 10/17] feat(plan): let a target plan from its synced copy of warehouse state Every plan lists and describes the warehouse's schemas live, which is most of a plan's time on a large project, while `sqb dev` already keeps a synced copy of the catalog in the project's control store. A target can now set `plan_from = "synced_state"`, and `--plan-from live|synced_state` on `sqb plan` and `sqb build` overrides it for one invocation. The default stays `live`. Planning from synced state reads the target's mirrored catalog from the control store, then checks it with one INFORMATION_SCHEMA.TABLES read per database: the listing it returns is used as is, so created and dropped tables are seen, and columns come from the copy except for tables whose LAST_DDL is later than the copy observed them (less five minutes for clock skew), which are described again. With more than 128 such tables the schema's columns are read live. Cursor bounds and incremental state are still read live. When the project has no control store, no location syncs the target for the project, the copy is empty or the check fails, the plan reads live and says why on stderr. Otherwise it reports how long the check took and how many tables it described again. Planning never writes to the control store. --- .../sqlbuild-cli/src/argv/command_table.json | 28 +++ crates/sqlbuild-cli/src/argv/constants.rs | 3 +- .../argv/tests/test_parse_cli_arguments.rs | 18 ++ crates/sqlbuild-cli/src/dispatch/constants.rs | 6 +- .../src/effective/constants.rs | 7 + .../src/project/_helpers/targets.rs | 24 ++- .../sqlbuild-config/src/project/constants.rs | 7 +- .../src/project/tests/test_load_config.rs | 22 +++ .../src/entries/_helpers/adapter.rs | 5 + crates/sqlbuild-host/src/entries/models.rs | 1 + .../src/warehouse_metadata/constants.rs | 6 + .../main/location_reads/mod.rs | 2 + .../read_development_location.rs | 35 ++++ .../location_reads/read_synced_catalog.rs | 41 ++++ .../src/warehouse_metadata/models.rs | 7 + .../_helpers/compiler_records/constants.rs | 1 + .../_helpers/contract_records/constants.rs | 2 + .../src/records/constants.rs | 2 + .../src/native_host/_helpers/mod.rs | 1 + .../native_host/_helpers/synced_catalog.rs | 115 ++++++++++++ .../src/native_host/constants.rs | 12 ++ .../native_host/main/attach_synced_catalog.rs | 66 +++++++ .../src/native_host/main/mod.rs | 2 + .../native_host/main/report_synced_catalog.rs | 22 +++ .../src/native_host/mod.rs | 1 + .../_helpers/build_command/planning.rs | 1 + .../_helpers/compile_driver/plan_pipeline.rs | 4 + .../bindings/_helpers/plan_command/plan.rs | 3 +- fensu.toml | 6 + .../sqlbuild/references/docs/cli/build.md | 1 + .../sqlbuild/references/docs/cli/plan.md | 1 + .../docs/concepts/project-configuration.md | 32 ++++ src/sqlbuild/_native.pyi | 10 + src/sqlbuild/_native_host.py | 1 + .../relations/classes/inspection_catalog.py | 95 ++++++++++ .../relations/classes/synced_catalog.py | 121 ++++++++++++ src/sqlbuild/adapter/relations/constants.py | 5 + src/sqlbuild/adapter/relations/models.py | 30 +++ src/sqlbuild/adapter/relations/types.py | 18 +- .../snowflake/_helpers/relation_changes.py | 50 +++++ .../snowflake/classes/snowflake_adapter.py | 34 ++++ .../cli/commands/main/plan/_test_types.py | 10 + .../commands/main/plan/test_diamond_plan.py | 35 ++++ .../compiler/planner/main/_test_types.py | 13 ++ .../sqlbuild/compiler/planner/main/helpers.py | 109 ++++++++++- .../main/test_plan_from_synced_state.py | 123 ++++++++++++ .../relations/test_metadata_sql_ownership.py | 1 + .../snowflake/inspection/_test_types.py | 25 +++ .../adapters/snowflake/inspection/helpers.py | 78 +++++++- .../inspection/test_synced_catalog.py | 176 ++++++++++++++++++ website/src/content/docs/docs/cli/build.mdx | 1 + website/src/content/docs/docs/cli/plan.mdx | 1 + .../docs/concepts/project-configuration.mdx | 32 ++++ 53 files changed, 1438 insertions(+), 14 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/_helpers/synced_catalog.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/constants.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/main/attach_synced_catalog.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/main/report_synced_catalog.rs create mode 100644 src/sqlbuild/adapter/relations/classes/synced_catalog.py create mode 100644 src/sqlbuild/adapters/snowflake/_helpers/relation_changes.py create mode 100644 tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py create mode 100644 tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_synced_catalog.py diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index e89e7360e8..0a0e5235bd 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1835,6 +1835,20 @@ "dest": "defer_sources_to", "group": 1 }, + { + "kind": "store", + "options": [ + "--plan-from" + ], + "dest": "plan_from", + "choices": [ + "live", + "synced_state" + ], + "default": null, + "help": "read warehouse state live, or from the target's synced copy checked against the warehouse first (default: the target's plan_from, else live)", + "group": 1 + }, { "kind": "store", "options": [ @@ -2277,6 +2291,20 @@ "dest": "defer_sources_to", "group": 1 }, + { + "kind": "store", + "options": [ + "--plan-from" + ], + "dest": "plan_from", + "choices": [ + "live", + "synced_state" + ], + "default": null, + "help": "read warehouse state live, or from the target's synced copy checked against the warehouse first (default: the target's plan_from, else live)", + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 88339e681a..991c78bae2 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -77,7 +77,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 191] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 192] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -141,6 +141,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 191] = [ ("as_target", NamespaceDefault::None), ("include_python", NamespaceDefault::Bool(true)), ("load_sources", NamespaceDefault::None), + ("plan_from", NamespaceDefault::None), ("reload", NamespaceDefault::Bool(false)), ("run_tests", NamespaceDefault::Bool(true)), ("run_audits", NamespaceDefault::Bool(true)), diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index c136da4699..8e3f224b8a 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -49,6 +49,18 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( attribute: "reach", expected_value: Value::Str("all".to_owned()), }, + ParsedValueTestCase { + description: "--plan-from names where a plan reads warehouse state", + argv: &["plan", "--plan-from", "synced_state"], + attribute: "plan_from", + expected_value: Value::Str("synced_state".to_owned()), + }, + ParsedValueTestCase { + description: "an unset --plan-from leaves the source to the target", + argv: &["build"], + attribute: "plan_from", + expected_value: Value::None, + }, ParsedValueTestCase { description: "an unset concurrency stays None", argv: &["build"], @@ -149,6 +161,12 @@ fn given_help_version_or_rejected_arguments_when_parsing_then_exits_after_writin expected_exit: 2, expected_last_write: "sqb build: error[C900]: argument --concurrency: invalid int value: 'x1'\n\n", }, + ExitTestCase { + description: "--plan-from names only live or synced_state", + argv: &["build", "--plan-from", "cache"], + expected_exit: 2, + expected_last_write: "sqb build: error[C900]: argument --plan-from: invalid choice: 'cache' (choose from 'live', 'synced_state')\n\n", + }, ExitTestCase { description: "an ambiguous abbreviation lists its matches", argv: &["build", "--allow-s"], diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 008107a2f0..2910f50607 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -110,7 +110,7 @@ pub(crate) const DAG_ROUTE: Route = routed( }, ); -const PLAN_FIELDS: [Field; 24] = [ +const PLAN_FIELDS: [Field; 25] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -135,6 +135,7 @@ const PLAN_FIELDS: [Field; 24] = [ same("selection_diagnostics"), same("timings"), same("ignore_audit_failures"), + same("plan_from"), ]; const PLAN_ARGUMENTS: [Field; 1] = [request(class("PlanCommandRequest"), &PLAN_FIELDS)]; pub(crate) const PLAN_ROUTE: Route = routed( @@ -144,13 +145,14 @@ pub(crate) const PLAN_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const BUILD_FIELDS: [Field; 39] = [ +const BUILD_FIELDS: [Field; 40] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), same("defer_to"), same("defer_clone_from"), same("defer_sources_to"), + same("plan_from"), SELECTED_TARGET, CURSOR_OVERRIDES, NO_COLOR, diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index a11bd722b1..683223e6dd 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -231,4 +231,11 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ decided_by: &["ignore_audit_failures"], }, }, + TargetField { + group: None, + name: "plan_from", + rule: MergeRule::Overlay { + decided_by: &["plan_from"], + }, + }, ]; diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index c83f1f4cc7..e520bb835e 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -5,9 +5,9 @@ use crate::project::_helpers::defaults::{ InheritableField, optional_retention_policy, optional_table_type, }; use crate::project::_helpers::fields::{ - bool_or_none, coerce_mapping, get, key_name, keyed, object, optional_nullable_bool, - optional_str, optional_strict_int, optional_text_field, string_mapping, text_or_none, - value_or_none, + bool_or_none, coerce_mapping, get, key_name, keyed, object, optional_choice, + optional_nullable_bool, optional_str, optional_strict_int, optional_text_field, string_mapping, + text_or_none, value_or_none, }; use crate::project::_helpers::keys::{ Place, reject_removed_virtual_keys, reject_unknown_mapping_keys, validate_allowed_keys, @@ -16,8 +16,8 @@ use crate::project::_helpers::sections::{fixed_duration_seconds, item_timeout}; use crate::project::_helpers::values::{int_below, is_int}; use crate::project::constants::{ CLONE_KEYS, CONCURRENCY_KEY, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, - IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, POOLS_KEY, RETENTION_MATERIALIZATIONS, - TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, + IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PLAN_FROM_KEY, PLAN_FROM_SOURCES, POOLS_KEY, + RETENTION_MATERIALIZATIONS, TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; @@ -432,9 +432,23 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded Loaded { + Ok(match get(mapping, PLAN_FROM_KEY) { + None => text_or_none(None), + Some(_) => text_or_none(Some(optional_choice( + mapping, + PLAN_FROM_KEY, + &PLAN_FROM_SOURCES, + PLAN_FROM_SOURCES[0], + )?)), + }) +} + /// `_load_local_targets`: each target's fields by name. pub(crate) fn load_target_fields( payload: Option<&ConfigValue>, diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index ad8150d254..6619b8aec3 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -192,7 +192,11 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 74] = [ pub(crate) const POOLS_KEY: &str = "pools"; /// The highest limit a pool may declare. pub(crate) const MAX_POOL_LIMIT: i64 = 10_000; -pub(crate) const TARGET_KEYS: [&str; 23] = [ +/// Where a target's plans read warehouse state: live, or a synced copy checked first. +pub(crate) const PLAN_FROM_KEY: &str = "plan_from"; +/// What `plan_from` may name; `live` is the default. +pub(crate) const PLAN_FROM_SOURCES: [&str; 2] = ["live", "synced_state"]; +pub(crate) const TARGET_KEYS: [&str; 24] = [ "connection", "catalog_connection", "vars", @@ -216,6 +220,7 @@ pub(crate) const TARGET_KEYS: [&str; 23] = [ "concurrency", POOLS_KEY, IGNORE_AUDIT_FAILURES_KEY, + PLAN_FROM_KEY, ]; pub(crate) const CONTROL_KEYS: [&str; 3] = ["location", "server", "sync_at_least_every"]; pub(crate) const CHANGES_KEYS: [&str; 2] = ["counts", "reach"]; diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 7cb96749fb..8c8a3fc77a 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -40,6 +40,28 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { "Expected 'catalog_connection' to be a non-empty string when provided", )), }, + ProjectFailureTestCase { + description: "a target may plan from its synced state", + project: "name = 'orders'\nadapter = 'duckdb'\n[targets.dev]\nplan_from = 'synced_state'\n", + local: "[targets.dev]\nplan_from = 'live'\n", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "plan_from names only live or synced_state", + project: "name = 'orders'\nadapter = 'duckdb'\n[targets.dev]\nplan_from = 'cache'\n", + local: "", + expected_failure: Some(ConfigFailure::plain( + "Expected 'plan_from' to be one of: live, synced_state", + )), + }, + ProjectFailureTestCase { + description: "a local target's plan_from names only live or synced_state", + project: "name = 'orders'\nadapter = 'duckdb'\n", + local: "[targets.dev]\nplan_from = 'cache'\n", + expected_failure: Some(ConfigFailure::plain( + "Expected 'plan_from' to be one of: live, synced_state", + )), + }, ProjectFailureTestCase { description: "an unknown key suggests the nearest supported key", project: "name = 'orders'\nadapter = 'duckdb'\n[scopes]\nenforce_placment = true\n", diff --git a/crates/sqlbuild-host/src/entries/_helpers/adapter.rs b/crates/sqlbuild-host/src/entries/_helpers/adapter.rs index d3cf753392..cdddda6eef 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/adapter.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/adapter.rs @@ -164,6 +164,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::Adapter, }, + HostEntry { + name: "SyncedCatalog", + kind: HostKind::Class, + edge: HostEdge::Adapter, + }, HostEntry { name: "InspectionQueryRecord", kind: HostKind::Class, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 1922abc77d..8fca0613e7 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -94,6 +94,7 @@ pub enum AdapterHost { ResolveRelationLocationQualifiedName, RunBoundedInspections, RunRecordedInspectionQuery, + SyncedCatalog, InspectionQueryRecord, StatementMetadataEffect, ConditionalResultNullability, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index b1eb03479b..1f753dee9c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -53,3 +53,9 @@ pub(crate) const INGESTION_CHUNK_RECORDS: usize = 200; /// The audit result metadata field marking a failure its build let through. pub(crate) const IGNORED_KEY: &str = "ignored"; +/// The mirrored catalog of one location's target, with when each relation was observed. +pub(crate) const SYNCED_CATALOG_SQL: &str = "SELECT record_json, observed_at FROM sqb_warehouse_catalog \ + WHERE location_id=$1 AND target=$2 ORDER BY state_database, state_schema, event_id"; +/// The development locations of one target, by project path. +pub(crate) const DEVELOPMENT_LOCATIONS_SQL: &str = + "SELECT project_path, location_id FROM sqb_development_locations WHERE target=$1"; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs index e38b47279c..86928a28c0 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs @@ -1,5 +1,7 @@ pub mod preview_state_parts; +pub mod read_development_location; pub mod read_preview_copy; pub mod read_statistics_due; pub mod read_sync_due; +pub mod read_synced_catalog; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs new file mode 100644 index 0000000000..2ee11e3f77 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs @@ -0,0 +1,35 @@ +//! The location `sqb dev` keeps for a project's target, read without creating one. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::DEVELOPMENT_LOCATIONS_SQL; + +/// The development location of `target` whose project path `same_project` accepts, if any. +pub async fn read_development_location( + store: &ControlStore, + target: &str, + same_project: impl Fn(&str) -> bool, +) -> Result, ControlError> { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await?, + DEVELOPMENT_LOCATIONS_SQL, + vec![Parameter::Text(target.into())], + ) + .await?; + commit(transaction).await?; + rows.iter() + .find(|row| row["project_path"].as_str().is_some_and(&same_project)) + .map(|row| { + row["location_id"] + .as_str() + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Schema("invalid development location identity".into())) + }) + .transpose() +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs new file mode 100644 index 0000000000..c3104ec656 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs @@ -0,0 +1,41 @@ +//! A plan from synced state reads the target's mirrored catalog and when each relation was seen. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::SYNCED_CATALOG_SQL; +use crate::warehouse_metadata::models::{SyncedCatalogRelation, WarehouseCatalogRecord}; + +/// Every mirrored relation of the location's target, with the time its catalog scan observed it. +pub async fn read_synced_catalog( + store: &ControlStore, + (location_id, target): (Uuid, &str), +) -> Result, ControlError> { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await?, + SYNCED_CATALOG_SQL, + vec![Parameter::Id(location_id), Parameter::Text(target.into())], + ) + .await?; + commit(transaction).await?; + rows.into_iter() + .map(|mut row| { + let record: WarehouseCatalogRecord = + serde_json::from_value(row.remove("record_json").unwrap_or_default()).map_err( + |error| ControlError::Invalid(format!("invalid mirrored relation: {error}")), + )?; + let observed_at = serde_json::from_value(row.remove("observed_at").unwrap_or_default()) + .map_err(|error| { + ControlError::Invalid(format!("invalid mirrored observation time: {error}")) + })?; + Ok(SyncedCatalogRelation { + record, + observed_at, + }) + }) + .collect() +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 7a40c9a009..fc24cb88a2 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -738,6 +738,13 @@ pub struct WarehouseCatalogColumn { pub data_type: String, } +/// One mirrored relation and the time the catalog scan that recorded it observed it. +#[derive(Clone, Debug, PartialEq)] +pub struct SyncedCatalogRelation { + pub record: WarehouseCatalogRecord, + pub observed_at: DateTime, +} + /// Physical identity and ordered columns come from the adapter catalog, not fingerprints. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index 530161c02c..ca785c24e1 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -407,6 +407,7 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { FieldDefault::NativeCall(RecordId::SqlTestSelection), ), FieldSpec::new("source_lexical_syntax", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), ], }; const R_PYTHON_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index c041a0efc2..78bdbf09b9 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -150,6 +150,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), ], }; const R_TARGET_SETTINGS: RecordSpec = RecordSpec { @@ -208,6 +209,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index f7f2d5cde5..ca6de97cfa 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -238,6 +238,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("defer_to", FieldDefault::None), FieldSpec::new("defer_clone_from", FieldDefault::None), FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("cursor_overrides", FieldDefault::None), FieldSpec::new("no_color", FieldDefault::False), @@ -560,6 +561,7 @@ const PLAN_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("no_sql_validation", FieldDefault::False), FieldSpec::new("defer_to", FieldDefault::None), FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("as_target", FieldDefault::None), FieldSpec::new("cursor_overrides", FieldDefault::None), diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs index b336f64d4d..18d68318d9 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod orchestration_triggers; +pub(crate) mod synced_catalog; diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/synced_catalog.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/synced_catalog.rs new file mode 100644 index 0000000000..5810d1f104 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/synced_catalog.rs @@ -0,0 +1,115 @@ +//! Reads and writes a plan from synced state shares: the control store's mirrored catalog and stderr notes. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::warehouse_metadata::main::location_reads::read_development_location::read_development_location; +use sqlbuild_orchestration::warehouse_metadata::main::location_reads::read_synced_catalog::read_synced_catalog; +use sqlbuild_orchestration::warehouse_metadata::models::SyncedCatalogRelation; + +use crate::native_host::constants::{CONTROL_STORE_FILE, CONTROL_STORE_VARIABLE, SYNCED_STATE}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; + +/// Whether `--plan-from`, or else the target's `plan_from`, asks for the synced state. +pub(crate) fn plans_from_synced_state( + discovered: &Bound<'_, PyAny>, + options: &Bound<'_, PyAny>, + target: &Bound<'_, PyAny>, +) -> PyResult { + let requested: Option = native_field(options, "plan_from")?.extract()?; + if let Some(requested) = requested { + return Ok(requested == SYNCED_STATE); + } + if target.is_none() { + return Ok(false); + } + let config: Bound<'_, PyAny> = resolve_target_config( + discovered.py(), + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + target, + )?; + let configured: Option = native_field(&config, "plan_from")?.extract()?; + Ok(configured.as_deref() == Some(SYNCED_STATE)) +} + +/// Write `text` to Python's stderr, where the plan's other progress goes, and flush it. +pub(crate) fn write_stderr(py: Python<'_>, text: &str) -> PyResult<()> { + let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; + stderr.call_method1("write", (text,))?; + stderr.call_method0("flush")?; + Ok(()) +} + +pub(crate) fn planning_live(py: Python<'_>, reason: &str) -> PyResult<()> { + write_stderr(py, &format!("Planning live: {reason}\n")) +} + +/// The mirrored catalog the project's control store holds for `target`'s development location. +pub(crate) fn synced_relations( + project_dir: &Path, + target: &str, +) -> Result, String> { + let url: String = match std::env::var(CONTROL_STORE_VARIABLE) { + Ok(url) => url, + Err(_) => { + let path: PathBuf = project_dir.join(CONTROL_STORE_FILE); + if !path.is_file() { + return Err("the project has no control store".to_owned()); + } + path.to_string_lossy().into_owned() + } + }; + let project: PathBuf = + std::fs::canonicalize(project_dir).unwrap_or_else(|_| project_dir.to_path_buf()); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| error.to_string())?; + let read = runtime.block_on(async { + let store = open_store(&url).await.map_err(|error| error.to_string())?; + let location = read_development_location(&store, target, |path| { + std::fs::canonicalize(path).is_ok_and(|path| path == project) + }) + .await + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("no location syncs target '{target}' for this project"))?; + read_synced_catalog(&store, (location, target)) + .await + .map_err(|error| error.to_string()) + }); + runtime.shutdown_timeout(Duration::from_secs(1)); + read +} + +/// `(database, schema, name, columns, observed_at)` of one mirrored relation. +pub(crate) fn synced_row<'py>( + py: Python<'py>, + relation: &SyncedCatalogRelation, +) -> PyResult> { + let record = &relation.record; + let columns: Vec<(String, String)> = record + .columns + .iter() + .map(|column| (column.name.clone(), column.data_type.clone())) + .collect(); + PyTuple::new( + py, + [ + record.state_database.clone().into_pyobject(py)?.into_any(), + record.state_schema.clone().into_pyobject(py)?.into_any(), + record.name.clone().into_pyobject(py)?.into_any(), + columns.into_pyobject(py)?.into_any(), + relation + .observed_at + .to_rfc3339() + .into_pyobject(py)? + .into_any(), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/native_host/constants.rs b/crates/sqlbuild-python-project/src/native_host/constants.rs new file mode 100644 index 0000000000..2c58c12343 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/constants.rs @@ -0,0 +1,12 @@ +//! Names a plan from synced state uses to find and attach the target's mirrored catalog. + +/// The `plan_from` source that plans from the target's synced copy of warehouse state. +pub(crate) const SYNCED_STATE: &str = "synced_state"; +/// The variable naming a control store other than the project's own. +pub(crate) const CONTROL_STORE_VARIABLE: &str = "SQLBUILD_CONTROL_DATABASE_URL"; +/// The project's own control store, beside its other `sqb dev` state. +pub(crate) const CONTROL_STORE_FILE: &str = ".sqlbuild/control.sqlite3"; +/// The adapter method that checks a synced copy against the warehouse; only Snowflake has it. +pub(crate) const RELATION_CHANGES_METHOD: &str = "read_relation_changes"; +/// The adapter attribute inspection reads the synced catalog from. +pub(crate) const SYNCED_CATALOG_ATTRIBUTE: &str = "synced_catalog"; diff --git a/crates/sqlbuild-python-project/src/native_host/main/attach_synced_catalog.rs b/crates/sqlbuild-python-project/src/native_host/main/attach_synced_catalog.rs new file mode 100644 index 0000000000..8ed31f9f80 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/attach_synced_catalog.rs @@ -0,0 +1,66 @@ +//! Give the adapter the target's synced catalog when a plan reads from synced state. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyList, PyTuple}; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_orchestration::warehouse_metadata::models::SyncedCatalogRelation; + +use crate::native_host::_helpers::synced_catalog::{ + planning_live, plans_from_synced_state, synced_relations, synced_row, +}; +use crate::native_host::constants::{RELATION_CHANGES_METHOD, SYNCED_CATALOG_ATTRIBUTE}; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Give `adapter` the target's synced catalog when the plan reads from synced state. +pub fn attach_synced_catalog( + adapter: &Bound<'_, PyAny>, + (discovered, compiled, options): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + if adapter.hasattr(SYNCED_CATALOG_ATTRIBUTE)? { + adapter.setattr(SYNCED_CATALOG_ATTRIBUTE, py.None())?; + } + let target: Bound<'_, PyAny> = + native_field(&native_field(compiled, "project")?, "effective_target_name")?; + if !plans_from_synced_state(discovered, options, &target)? { + return Ok(()); + } + let checks: bool = adapter.hasattr(RELATION_CHANGES_METHOD)? + && adapter.getattr(RELATION_CHANGES_METHOD)?.is_callable(); + if !checks { + let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; + return planning_live( + py, + &format!("the {adapter_name} adapter cannot check a synced copy"), + ); + } + let project_dir: Option = native_field(discovered, "project_dir")?.extract()?; + let target: Option = target.extract()?; + let (Some(project_dir), Some(target)) = (project_dir, target) else { + return planning_live(py, "the project directory or target is unknown"); + }; + let loaded: Result, String> = + py.detach(|| synced_relations(&project_dir, &target)); + match loaded { + Ok(relations) if !relations.is_empty() => { + let rows: Vec> = relations + .iter() + .map(|relation| synced_row(py, relation)) + .collect::>()?; + let catalog: Bound<'_, PyAny> = call_with( + &host_entry(py, AdapterHost::SyncedCatalog)?.getattr("from_rows")?, + &[("rows", PyList::new(py, rows)?.into_any())], + )?; + adapter.setattr(SYNCED_CATALOG_ATTRIBUTE, catalog) + } + Ok(_) => planning_live( + py, + "the control store holds no synced catalog for this target", + ), + Err(reason) => planning_live(py, &reason), + } +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/mod.rs b/crates/sqlbuild-python-project/src/native_host/main/mod.rs index 9d80042f90..0ecca1637c 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/mod.rs @@ -1,2 +1,4 @@ +pub mod attach_synced_catalog; pub mod compile_calendars; pub mod compile_sensors; +pub mod report_synced_catalog; diff --git a/crates/sqlbuild-python-project/src/native_host/main/report_synced_catalog.rs b/crates/sqlbuild-python-project/src/native_host/main/report_synced_catalog.rs new file mode 100644 index 0000000000..7fd881d59e --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/report_synced_catalog.rs @@ -0,0 +1,22 @@ +//! Write what a plan from synced state found when it checked the catalog. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +use crate::native_host::_helpers::synced_catalog::write_stderr; +use crate::native_host::constants::SYNCED_CATALOG_ATTRIBUTE; + +/// Write what the synced state checks found, once the plan is made. +pub fn report_synced_catalog(adapter: &Bound<'_, PyAny>) -> PyResult<()> { + if !adapter.hasattr(SYNCED_CATALOG_ATTRIBUTE)? { + return Ok(()); + } + let catalog: Bound<'_, PyAny> = adapter.getattr(SYNCED_CATALOG_ATTRIBUTE)?; + if catalog.is_none() { + return Ok(()); + } + let report: Option = catalog.call_method0("report")?.extract()?; + match report { + Some(line) => write_stderr(adapter.py(), &format!("{line}\n")), + None => Ok(()), + } +} diff --git a/crates/sqlbuild-python-project/src/native_host/mod.rs b/crates/sqlbuild-python-project/src/native_host/mod.rs index 9fbdf0b3ef..cbe3fe33b0 100644 --- a/crates/sqlbuild-python-project/src/native_host/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/mod.rs @@ -1,5 +1,6 @@ //! Model lookup and orchestration triggers. pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index d16b491703..9cbf89ba27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -67,6 +67,7 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult( })?; connection = Some(opened.clone()); let reads = LiveReads::new(adapter, &opened, None).handle(adapter.py())?; + attach_synced_catalog(adapter, (discovered, compiled, options))?; (opened, reads) } }; @@ -330,6 +333,7 @@ fn planned<'py>( tracker.setattr("planning_seconds", planning_seconds)?; } let result: Bound<'py, PyAny> = result?; + report_synced_catalog(adapter)?; write_plan_capture(&native_field(&result, "plan_output")?)?; replaced( &result, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 8263242843..6f4644d36d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -67,7 +67,7 @@ const DEFAULT_SECTION_ENTRIES: i64 = 50; const ERROR_SEVERITY: &str = "error"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; /// Request fields passed to the compile pipeline under the same name. -const PIPELINE_FIELDS: [&str; 12] = [ +const PIPELINE_FIELDS: [&str; 13] = [ "no_sql_validation", "no_cache", "verbose", @@ -80,6 +80,7 @@ const PIPELINE_FIELDS: [&str; 12] = [ "cli_vars", "max_microbatches", "selection_diagnostics", + "plan_from", ]; fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { diff --git a/fensu.toml b/fensu.toml index 80802c155b..5537713324 100644 --- a/fensu.toml +++ b/fensu.toml @@ -92,6 +92,11 @@ modules = ["sqlbuild.adapter.discovery.main.resolve_adapter"] symbols = ["resolve_adapter"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.classes.synced_catalog"] +symbols = ["SyncedCatalog"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.adapter.relations.main._get_columns_for_inspection"] symbols = ["get_columns_for_inspection"] @@ -3149,6 +3154,7 @@ module_layers = [ "native_host/live_reads=plan", "native_host/orchestration_sensors=commands", "native_host/setup_context=commands", + "native_host/synced_catalog=plan", "native_host/warehouse_metadata=commands", "observations=plan", "output_capture=registry", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 68a98b4599..eed303fed9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -63,6 +63,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--reach` | With `--changed`: `job` stops at models another scheduled job refreshes, `all` builds everything downstream | | `--defer-to` | Resolve unselected model references against another target | | `--defer-sources-to` | Read managed source data from another target | +| `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](../concepts/project-configuration.md#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](../concepts/testing-reference.md#time-limits) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 1c3f09f65e..66d69eead9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -34,6 +34,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--no-python` | Exclude read-side Python tasks and assets from the plan | | `--defer-to` | Resolve unselected model references against another target | +| `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](../concepts/project-configuration.md#planning-from-synced-state) | | `--as` | Compile for another target and show its plan, including [model migration](../concepts/models/migrations.md) decisions, using the current connection. Inspection only. | | `--json` | Output the plan as JSON | | `--timings` | After the plan, print how long each planning step took; see [Timings](#timings) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 63c1f46108..4b14e2650e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -188,6 +188,7 @@ A target is a named build context - the database and schema you build into, plus | `warehouses` | Snowflake warehouse per command group: `build`, `query` and `sync` (a location's warehouse syncs). See [Command-group warehouses](adapters/snowflake.md#command-group-warehouses) | | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](audits.md#ignoring-audit-failures) | +| `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | ```toml [connections.warehouse] @@ -281,6 +282,37 @@ The value must be an integer of at least `1`; anything else fails with `D001`. ` `8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and uses the active target's value. +### Planning from synced state + +Every plan reads the warehouse's catalog: which tables and views exist and their columns. On a +large project that is most of a plan's time. A target can instead plan from the copy of the +catalog that `sqb dev` keeps synced in the project's control store, after checking it against +the warehouse: + +```toml +[targets.dev] +schema = "analytics_dev" +plan_from = "synced_state" +``` + +`--plan-from live` or `--plan-from synced_state` on `sqb plan` or `sqb build` overrides the target +for one invocation. The default is `live`; SQLBuild cannot tell which target is production, so +planning from synced state is something a target opts into. + +At the start of the plan, one `INFORMATION_SCHEMA.TABLES` read per database lists the schemas the +copy holds, with each table's last DDL. The plan uses that listing, so tables created or dropped +since the last sync are seen. It reads columns from the copy, except for tables whose last DDL is +later than the copy's observation of them (allowing five minutes for clock skew), which it +describes afresh. When more than 128 tables changed, it reads the schema's columns live instead. +Cursor bounds and incremental state are still read live. + +The plan reads live, and says why on stderr (`Planning live: ...`), when the target's adapter +isn't Snowflake (the only adapter that can check the copy), the project has no control store, no +location syncs the target for this project, the copy is empty, or the check fails. Otherwise it reports `Planned from synced state: checked in 1.2s, 3 tables refreshed`. The +control store is `.sqlbuild/control.sqlite3` in the project, or the database +`SQLBUILD_CONTROL_DATABASE_URL` names. Planning never writes to it; only the location's sync +refreshes the copy. + ### Clone policies Targets can declare whether they allow cloning to or from: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7ab8e77d84..cd504f2230 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6560,6 +6560,7 @@ class BuildCommandRequest(NativeRecord): defer_to: str | None defer_clone_from: str | None defer_sources_to: str | None + plan_from: str | None selected_target: str | None cursor_overrides: CursorOverrides | None no_color: bool @@ -6603,6 +6604,7 @@ class BuildCommandRequest(NativeRecord): defer_to: str | None = None, defer_clone_from: str | None = None, defer_sources_to: str | None = None, + plan_from: str | None = None, selected_target: str | None = None, cursor_overrides: CursorOverrides | None = None, no_color: bool = False, @@ -6971,6 +6973,7 @@ class PlanCommandRequest(NativeRecord): no_sql_validation: bool defer_to: str | None defer_sources_to: str | None + plan_from: str | None selected_target: str | None as_target: str | None cursor_overrides: CursorOverrides | None @@ -6997,6 +7000,7 @@ class PlanCommandRequest(NativeRecord): no_sql_validation: bool = False, defer_to: str | None = None, defer_sources_to: str | None = None, + plan_from: str | None = None, selected_target: str | None = None, as_target: str | None = None, cursor_overrides: CursorOverrides | None = None, @@ -10234,6 +10238,7 @@ class CompilePipelineOptions(NativeRecord): accepts_unit_test_selectors: bool sql_test_selection: SqlTestSelection source_lexical_syntax: SqlLexicalSyntax | None + plan_from: str | None ignore_audit_failures: bool def __init__( self, @@ -10262,6 +10267,7 @@ class CompilePipelineOptions(NativeRecord): accepts_unit_test_selectors: bool = ..., sql_test_selection: SqlTestSelection = ..., source_lexical_syntax: SqlLexicalSyntax | None = ..., + plan_from: str | None = ..., ignore_audit_failures: bool = ..., ) -> None: ... @@ -11991,6 +11997,7 @@ class TargetConfig(NativeRecord): concurrency: int | None catalog_connection: str | None ignore_audit_failures: bool | None + plan_from: str | None def __init__( self, connection: dict[str, object] = ..., @@ -12016,6 +12023,7 @@ class TargetConfig(NativeRecord): concurrency: int | None = ..., catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., + plan_from: str | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -12044,6 +12052,7 @@ class LocalTargetConfig(NativeRecord): concurrency: int | None catalog_connection: str | None ignore_audit_failures: bool | None + plan_from: str | None def __init__( self, connection: dict[str, object] = ..., @@ -12070,6 +12079,7 @@ class LocalTargetConfig(NativeRecord): concurrency: int | None = ..., catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., + plan_from: str | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index c6f7428a5a..6e4615534d 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -63,6 +63,7 @@ "sqlbuild.adapter.relations.main.record_inspection_query", "record_inspection_query", ), + "SyncedCatalog": ("sqlbuild.adapter.relations.classes.synced_catalog", "SyncedCatalog"), "build_relation_lookup": ( "sqlbuild.adapter.relations.main.relation_lookup", "build_relation_lookup", diff --git a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py index f02c12c41f..bfb387c150 100644 --- a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py +++ b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py @@ -5,6 +5,7 @@ import threading import time from collections.abc import Callable +from dataclasses import replace from typing import Any, cast from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo @@ -14,19 +15,28 @@ ) from sqlbuild.adapter.relations.classes.relation_metadata_cache import RelationMetadataCache from sqlbuild.adapter.relations.classes.remembered_flight import RememberedFlight +from sqlbuild.adapter.relations.classes.synced_catalog import ( + SyncedCatalog, +) from sqlbuild.adapter.relations.constants import ( INSPECTION_DESCRIBED_NAME_LIMIT, INSPECTION_IN_LIST_LIMIT, SCHEMA_LISTING_MAX_AGE_SECONDS, + SYNCED_CATALOG_ATTRIBUTE, + SYNCED_REFRESH_LIMIT, ) from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections from sqlbuild.adapter.relations.models import ( InspectionQueryRecord, + ListedRelation, + RelationChange, SchemaColumnListing, SchemaRelationListing, + SeededSchema, ) from sqlbuild.adapter.relations.types import ( PartedRelationListingReader, + RelationChangeReader, RelationListingScoper, SchemaScopedMetadataReader, ViewDefinitionBatchReader, @@ -56,6 +66,8 @@ def __init__(self, *, adapter: Any, connection: Any) -> None: self._relation_requests: dict[_RelationRequestKey, tuple[RelationInfo, ...]] = {} self._relation_columns: dict[_RelationIdentity, tuple[ColumnInfo, ...] | None] = {} self._view_definitions: dict[_RelationIdentity, str | None] = {} + synced: object = getattr(adapter, SYNCED_CATALOG_ATTRIBUTE, None) + self._synced: SyncedCatalog | None = synced if isinstance(synced, SyncedCatalog) else None def remember[ValueT](self, *, key: tuple[str, ...], compute: Callable[[], ValueT]) -> ValueT: """Compute one adapter-owned metadata fact at most once; concurrent callers share it.""" @@ -144,6 +156,8 @@ def prefetch_schema_listings( key: _SchemaKey = self._schema_key(database=database, schema=schema) if key not in self._relation_listings: missing.setdefault(key, (database, schema)) + for key in self._seed_synced(reader=reader, missing=missing): + del missing[key] listings: list[SchemaRelationListing | None] if isinstance(reader, PartedRelationListingReader) and ( current_relation_metadata_cache(adapter=self.adapter) is None @@ -167,6 +181,87 @@ def prefetch_schema_listings( if listing is not None: self._relation_listings[missing_key] = listing + def _seed_synced( + self, + *, + reader: SchemaScopedMetadataReader, + missing: dict[_SchemaKey, tuple[str | None, str]], + ) -> tuple[_SchemaKey, ...]: + """List the synced copy's schemas with one check per database, columns from the copy.""" + + synced: SyncedCatalog | None = self._synced + if synced is None or not isinstance(self.adapter, RelationChangeReader): + return () + by_database: dict[str | None, list[tuple[_SchemaKey, str]]] = {} + key: _SchemaKey + database: str | None + schema: str + for key, (database, schema) in missing.items(): + if synced.covers(database=database, schema=schema): + by_database.setdefault(database, []).append((key, schema)) + seeded: list[_SchemaKey] = [] + scoped: list[tuple[_SchemaKey, str]] + for database, scoped in by_database.items(): + started: float = time.monotonic() + try: + changes: dict[str, tuple[RelationChange, ...]] = self.adapter.read_relation_changes( + connection=self.connection, + database=database, + schemas=tuple(schema for _, schema in scoped), + ) + except Exception as error: + synced.record_failure(reason=f"the synced state check failed ({error})") + self._synced = None + return tuple(seeded) + for key, schema in scoped: + listed: tuple[RelationChange, ...] = changes.get( + reader.metadata_name_key(schema), () + ) + if not listed: + continue + seed: SeededSchema = synced.seed(database=database, schema=schema, changes=listed) + self._relation_listings[key] = replace(seed.relations, database=database) + if len(seed.stale) <= SYNCED_REFRESH_LIMIT: + try: + columns: dict[str, tuple[ColumnInfo, ...]] = self._refreshed_columns( + reader=reader, seed=seed + ) + except Exception as error: + del self._relation_listings[key] + synced.record_failure(reason=f"the synced state refresh failed ({error})") + self._synced = None + return tuple(seeded) + self._column_listings[key] = SchemaColumnListing( + database=database, schema=schema, columns_by_stored_name=columns + ) + seeded.append(key) + synced.record_check( + seconds=time.monotonic() - started, + relations=len(seed.relations.entries), + refreshed=len(seed.stale), + ) + return tuple(seeded) + + def _refreshed_columns( + self, *, reader: SchemaScopedMetadataReader, seed: SeededSchema + ) -> dict[str, tuple[ColumnInfo, ...]]: + """The synced columns, each relation whose definition changed since then read afresh.""" + + fresh: list[tuple[ColumnInfo, ...] | None] = run_bounded_inspections( + tasks=tuple( + self._bind_relation_columns(reader=reader, relation=stale.relation) + for stale in seed.stale + ), + concurrency=self._concurrency, + ) + columns: dict[str, tuple[ColumnInfo, ...]] = dict(seed.columns) + stale: ListedRelation + read: tuple[ColumnInfo, ...] | None + for stale, read in zip(seed.stale, fresh, strict=True): + if read is not None: + columns[stale.stored_name] = read + return columns + def _read_parted_listings( self, *, diff --git a/src/sqlbuild/adapter/relations/classes/synced_catalog.py b/src/sqlbuild/adapter/relations/classes/synced_catalog.py new file mode 100644 index 0000000000..a05bcc153f --- /dev/null +++ b/src/sqlbuild/adapter/relations/classes/synced_catalog.py @@ -0,0 +1,121 @@ +"""A target's mirrored catalog, which a plan uses once the warehouse confirms it is current.""" + +from __future__ import annotations + +import sys +import threading +from datetime import datetime + +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.relations.constants import SYNCED_DDL_CLOCK_SKEW +from sqlbuild.adapter.relations.models import ( + ListedRelation, + RelationChange, + SchemaRelationListing, + SeededSchema, + SyncedRelation, +) +from sqlbuild.adapter.relations.types import SyncedRow + + +class SyncedCatalog: + """The mirror of one target's catalog, by lowercased database and schema.""" + + def __init__(self, *, relations: tuple[SyncedRelation, ...]) -> None: + self._schemas: dict[tuple[str | None, str], dict[str, list[SyncedRelation]]] = {} + relation: SyncedRelation + for relation in relations: + self._schemas.setdefault( + _scope_key(database=relation.database, schema=relation.schema), {} + ).setdefault(relation.name.lower(), []).append(relation) + self._lock: threading.Lock = threading.Lock() + self._checked: list[tuple[float, int, int]] = [] + self._failure: str | None = None + + @classmethod + def from_rows(cls, *, rows: list[SyncedRow]) -> SyncedCatalog: + """Each mirrored `(database, schema, name, columns, observed_at)` row as one catalog.""" + + return cls(relations=tuple(_synced_relation(row=row) for row in rows)) + + def covers(self, *, database: str | None, schema: str) -> bool: + """Whether the mirror holds any relation of this schema.""" + + return _scope_key(database=database, schema=schema) in self._schemas + + def seed( + self, *, database: str | None, schema: str, changes: tuple[RelationChange, ...] + ) -> SeededSchema: + """The schema as the check lists it, with mirrored columns where no DDL followed them.""" + + mirrored: dict[str, list[SyncedRelation]] = self._schemas.get( + _scope_key(database=database, schema=schema), {} + ) + entries: list[ListedRelation] = [] + columns: dict[str, tuple[ColumnInfo, ...]] = {} + stale: list[ListedRelation] = [] + names: dict[str, int] = {} + change: RelationChange + for change in changes: + names[change.listed.stored_name.lower()] = ( + names.get(change.listed.stored_name.lower(), 0) + 1 + ) + for change in changes: + entries.append(change.listed) + candidates: list[SyncedRelation] = mirrored.get(change.listed.stored_name.lower(), []) + current: bool = ( + len(candidates) == 1 + and names[change.listed.stored_name.lower()] == 1 + and change.last_ddl is not None + and change.last_ddl <= candidates[0].observed_at - SYNCED_DDL_CLOCK_SKEW + ) + if current: + columns[change.listed.stored_name] = candidates[0].columns + else: + stale.append(change.listed) + return SeededSchema( + relations=SchemaRelationListing( + database=database, schema=schema, entries=tuple(entries) + ), + columns=columns, + stale=tuple(stale), + ) + + def record_check(self, *, seconds: float, relations: int, refreshed: int) -> None: + """Note one check: how long it took, the relations it listed and those re-read.""" + + with self._lock: + self._checked.append((seconds, relations, refreshed)) + + def record_failure(self, *, reason: str) -> None: + """Note that the warehouse could not confirm the mirror, so the plan reads it live.""" + + with self._lock: + self._failure = self._failure or reason + print(f"Planning live: {reason}", file=sys.stderr, flush=True) + + def report(self) -> str | None: + """One line on what the checks found, or None when nothing was checked.""" + + with self._lock: + if not self._checked: + return None + seconds: float = sum(check[0] for check in self._checked) + refreshed: int = sum(check[2] for check in self._checked) + noun: str = "table" if refreshed == 1 else "tables" + return f"Planned from synced state: checked in {seconds:.1f}s, {refreshed} {noun} refreshed" + + +def _scope_key(*, database: str | None, schema: str) -> tuple[str | None, str]: + return (None if database is None else database.lower(), schema.lower()) + + +def _synced_relation(*, row: SyncedRow) -> SyncedRelation: + database, schema, name, columns, observed_at = row + return SyncedRelation( + database=database, + schema=schema, + name=name, + columns=tuple(ColumnInfo(name=column, type=kind) for column, kind in columns), + observed_at=datetime.fromisoformat(observed_at), + ) diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 89549a0934..6afe58c72e 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -1,5 +1,7 @@ """Shared relation metadata limits.""" +from datetime import timedelta + INSPECTION_DESCRIBED_NAME_LIMIT: int = 3 INSPECTION_IN_LIST_LIMIT: int = 50 METADATA_NAME_FILTER_LIMIT: int = 250 @@ -9,3 +11,6 @@ SCHEMA_LISTING_LOOKUP_THRESHOLD: int = 2 SCHEMA_LISTING_MAX_AGE_SECONDS: float = 60.0 UNKNOWN_RELATION_READER: str = "unknown user" +SYNCED_CATALOG_ATTRIBUTE: str = "synced_catalog" +SYNCED_REFRESH_LIMIT: int = 128 +SYNCED_DDL_CLOCK_SKEW: timedelta = timedelta(minutes=5) diff --git a/src/sqlbuild/adapter/relations/models.py b/src/sqlbuild/adapter/relations/models.py index d9762f4f60..9689310e37 100644 --- a/src/sqlbuild/adapter/relations/models.py +++ b/src/sqlbuild/adapter/relations/models.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import datetime from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo @@ -15,6 +16,15 @@ class ListedRelation: relation: RelationInfo +@dataclass(frozen=True) +class RelationChange: + """One listed relation and when its data and its definition last changed, as UTC.""" + + listed: ListedRelation + last_altered: datetime | None + last_ddl: datetime | None + + @dataclass(frozen=True) class SchemaRelationListing: """Every relation one database schema exposes to the inspecting role.""" @@ -61,3 +71,23 @@ class RunSchemaListing: global_version: int name_versions: dict[str, int] expires_at: float | None = None + + +@dataclass(frozen=True) +class SyncedRelation: + """One mirrored relation: its columns and when the catalog scan observed them.""" + + database: str | None + schema: str + name: str + columns: tuple[ColumnInfo, ...] + observed_at: datetime + + +@dataclass(frozen=True) +class SeededSchema: + """One checked schema's listing, its columns, and the relations whose columns are stale.""" + + relations: SchemaRelationListing + columns: dict[str, tuple[ColumnInfo, ...]] + stale: tuple[ListedRelation, ...] diff --git a/src/sqlbuild/adapter/relations/types.py b/src/sqlbuild/adapter/relations/types.py index 20bf2f654f..f5e9f0db57 100644 --- a/src/sqlbuild/adapter/relations/types.py +++ b/src/sqlbuild/adapter/relations/types.py @@ -8,12 +8,17 @@ from collections.abc import Callable from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo - from sqlbuild.adapter.relations.models import SchemaColumnListing, SchemaRelationListing + from sqlbuild.adapter.relations.models import ( + RelationChange, + SchemaColumnListing, + SchemaRelationListing, + ) type RelationCacheKey = tuple[str, str | None, str | None, str] type RelationCacheVersion = tuple[int, int] type SchemaListingScope = tuple[str | None, str] +type SyncedRow = tuple[str | None, str, str, list[tuple[str, str]], str] class RelationLocation(Protocol): @@ -88,6 +93,17 @@ def relation_listing_from_reads( ... +@runtime_checkable +class RelationChangeReader(Protocol): + """Adapter that lists schemas with when each relation's data and definition last changed.""" + + def read_relation_changes( + self, *, connection: Any, database: str | None, schemas: tuple[str, ...] + ) -> dict[str, tuple[RelationChange, ...]]: + """Each schema's relations, by the schema's metadata name key, in one read.""" + ... + + @runtime_checkable class RelationChangeTokenReader(Protocol): """Adapter that reads a table's change token without starting warehouse compute.""" diff --git a/src/sqlbuild/adapters/snowflake/_helpers/relation_changes.py b/src/sqlbuild/adapters/snowflake/_helpers/relation_changes.py new file mode 100644 index 0000000000..0d64f0ac0a --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/relation_changes.py @@ -0,0 +1,50 @@ +"""Map INFORMATION_SCHEMA.TABLES rows onto the relations SHOW TABLES and SHOW VIEWS list.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc +from sqlbuild.adapter.relations.models import ListedRelation, RelationChange + +_RELATION_TYPES: dict[str, str] = { + "BASE TABLE": "base table", + "LOCAL TEMPORARY": "temporary table", + "TEMPORARY TABLE": "temporary table", + "EXTERNAL TABLE": "external table", + "EVENT TABLE": "event table", + "VIEW": "view", + "MATERIALIZED VIEW": "materialized view", +} +_VIEW_TYPES: frozenset[str] = frozenset({"view", "materialized view"}) +_YES: str = "YES" + + +def relation_change_from_row(*, row: tuple[Any, ...], database: str | None) -> RelationChange: + """One relation as SHOW would list it, with its last data change and last DDL in UTC.""" + + stored_name: str = str(row[0]) + table_type: str = str(row[2] or "").upper() + relation_type: str = _RELATION_TYPES.get(table_type, table_type.lower()) + view: bool = relation_type in _VIEW_TYPES + retention: str = str(row[7] if row[7] is not None else "").strip() + return RelationChange( + listed=ListedRelation( + stored_name=stored_name, + relation=RelationInfo( + database=database, + schema=str(row[1]).lower(), + name=stored_name.lower(), + relation_type=relation_type, + is_transient=None if view else str(row[3] or "").upper() == _YES, + created_at=relation_age_timestamp_utc(row[4]), + last_altered_at=None, + retention_days=None + if view or not retention.lstrip("-").isdigit() + else int(retention), + ), + ), + last_altered=relation_age_timestamp_utc(row[5]), + last_ddl=relation_age_timestamp_utc(row[6]), + ) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 41d3d77767..f0fb30be60 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -121,6 +121,7 @@ ) from sqlbuild.adapter.relations.models import ( ListedRelation, + RelationChange, SchemaColumnListing, SchemaRelationListing, ) @@ -143,6 +144,7 @@ information_schema_type, show_columns_type, ) +from sqlbuild.adapters.snowflake._helpers.relation_changes import relation_change_from_row from sqlbuild.adapters.snowflake._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.snowflake._helpers.show_metadata import ( is_missing_object_error, @@ -201,6 +203,10 @@ "table_name, table_schema, table_type, is_transient, created, last_altered, retention_time" ) _BULK_COLUMN_RELATION_CHUNK_SIZE: int = 2_000 +_RELATION_CHANGE_COLUMNS: str = ( + "table_name, table_schema, table_type, is_transient, created, last_altered, last_ddl, " + "retention_time" +) class SnowflakeAdapter( @@ -2260,6 +2266,34 @@ def _relation_info_from_row(*, row: tuple[Any, ...], database: str | None) -> Re retention_days=None if row[6] is None else int(row[6]), ) + def read_relation_changes( + self, *, connection: _SnowflakeConnection, database: str | None, schemas: tuple[str, ...] + ) -> dict[str, tuple[RelationChange, ...]]: + """List schemas from INFORMATION_SCHEMA.TABLES, whose LAST_DDL moves with each DDL.""" + + placeholders: str = ", ".join(["%s"] * len(schemas)) + query: str = ( + f"SELECT {_RELATION_CHANGE_COLUMNS} " + f"FROM {self._information_schema_relation(database=database, name='tables')} " + f"WHERE table_schema IN ({placeholders})" + ) + params: list[str] = [self._information_schema_identifier(schema) for schema in schemas] + if database is not None: + query += " AND table_catalog = %s" + params.append(self._information_schema_identifier(database)) + rows: list[tuple[Any, ...]] = self._fetch_inspection_rows( + connection=connection, query=query, params=tuple(params) + ) + changes: dict[str, list[RelationChange]] = { + self.metadata_name_key(schema): [] for schema in schemas + } + row: tuple[Any, ...] + for row in rows: + changes.setdefault(self.metadata_name_key(str(row[1])), []).append( + relation_change_from_row(row=row, database=database) + ) + return {schema: tuple(listed) for schema, listed in changes.items()} + def read_relation_change_token( self, *, connection: _SnowflakeConnection, database: str | None, schema: str, name: str ) -> str | None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index 5a72309c50..68e56474f0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -109,3 +109,13 @@ class PlanWithoutTimingsE2ETestCase: description: str plan_args: tuple[str, ...] expected_table_printed: bool + + +@dataclass(frozen=True) +class PlanFromE2ETestCase: + """Where a DuckDB plan is asked to read warehouse state, and what it says on stderr.""" + + description: str + plan_args: tuple[str, ...] + expected_stderr: str + unexpected_stderr: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py index 42845dc31e..8b622abd76 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py @@ -16,6 +16,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( DiamondPlanE2ETestCase, DiamondPlanJsonE2ETestCase, + PlanFromE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( prepare_changed_incremental_diamond_project, @@ -91,3 +92,37 @@ def test_given_changed_diamond_root_when_planning_json_then_lists_each_model_onc reasons: dict[str, object] = {str(model["name"]): model["reason"] for model in models} assert len(models) == len(reasons) assert reasons == test_case.expected_reasons + + +@pytest.mark.parametrize( + "test_case", + [ + PlanFromE2ETestCase( + description="planning from synced state on DuckDB plans live and says why", + plan_args=("--plan-from", "synced_state"), + expected_stderr="Planning live: the duckdb adapter cannot check a synced copy\n", + unexpected_stderr="Planned from synced state", + ), + PlanFromE2ETestCase( + description="planning live says nothing about synced state", + plan_args=(), + expected_stderr="", + unexpected_stderr="Planning live", + ), + ], + ids=lambda case: case.description, +) +def test_given_duckdb_target_when_planning_from_synced_state_then_plans_live_and_says_why( + test_case: PlanFromE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_changed_incremental_diamond_project(tmp_path=tmp_path) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", *test_case.plan_args), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert test_case.expected_stderr in result.stderr, result.stderr + assert test_case.unexpected_stderr not in result.stderr, result.stderr + assert "Query changed (1)" in result.stdout, result.stdout diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index efc338a23f..c44c4047ec 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -173,3 +173,16 @@ class UnlistedMigrationOriginTestCase: select: tuple[str, ...] expected_warnings: tuple[str, ...] expected_unlisted_reads: int + + +@dataclass(frozen=True) +class PlanFromTestCase: + """Where a target and the command ask a plan to read from, and what the plan reports.""" + + description: str + target_plan_from: str | None + requested: str | None + store: bool + expected_report: str + expected_absent: tuple[str, ...] + expected_live_listing: bool diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 187971a336..8c0c472404 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -2,9 +2,12 @@ from __future__ import annotations +import json import re +import sqlite3 from collections import Counter from collections.abc import Callable +from contextlib import closing from dataclasses import fields, replace from datetime import UTC, datetime from itertools import chain @@ -12,6 +15,7 @@ from tempfile import gettempdir from types import MappingProxyType from typing import Any, cast +from uuid import uuid4 from sqlbuild._native import ( CompiledFunction, @@ -52,7 +56,8 @@ run_compile_pipeline, write_fingerprint, ) -from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationReadProbe +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import ( CompiledFunctionView, @@ -74,6 +79,8 @@ RecordedQuery, RecordingSnowflakeWarehouse, SyntheticSnowflakeProject, + live_inspection_catalog, + write_synthetic_snowflake_project, ) @@ -494,6 +501,7 @@ def plan_offline_snowflake_project( project: SyntheticSnowflakeProject, warehouse: RecordingSnowflakeWarehouse, no_cache: bool = True, + plan_from: str | None = None, ) -> CompilePipelineResult: """Compile and plan a synthetic Snowflake project against the recording warehouse.""" @@ -505,6 +513,7 @@ def plan_offline_snowflake_project( no_sql_validation=True, no_cache=no_cache, connection_config=dict(inputs.project_config.connection), + plan_from=plan_from, ), hooks=ConnectionHooks(), ) @@ -683,3 +692,101 @@ def plan_source_reading_project( adapter=adapter, options=CompilePipelineOptions(no_sql_validation=True), ) + + +def write_targeted_snowflake_project( + *, project_dir: Path, target: str, plan_from: str | None +) -> SyntheticSnowflakeProject: + """A small synthetic project whose default target sets `plan_from`, or leaves it unset.""" + + project: SyntheticSnowflakeProject = write_synthetic_snowflake_project( + project_dir=project_dir, + models_per_schema=6, + incremental_every=3, + sources=4, + unmanaged_relations_per_schema=5, + ) + toml: Path = project_dir / "sqlbuild_project.toml" + setting: str = ("", f'plan_from = "{plan_from}"\n')[plan_from is not None] + toml.write_text( + toml.read_text().replace( + 'adapter = "snowflake"\n', f'adapter = "snowflake"\ndefault_target = "{target}"\n', 1 + ) + + f'\n[targets.{target}]\nschema = "preserve"\n{setting}', + encoding="utf-8", + ) + return project + + +def write_synced_control_store( + *, + project_dir: Path, + target: str, + warehouse: RecordingSnowflakeWarehouse, + schemas: tuple[str, ...], +) -> None: + """A control store whose development location mirrored `schemas` of the warehouse just now.""" + + catalog: InspectionCatalog = live_inspection_catalog(warehouse) + relations: tuple[RelationInfo, ...] = catalog.list_relations( + database="analytics", schemas=schemas + ) + columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( + catalog.get_columns_for_relations(relations=relations) + ) + location: str = str(uuid4()) + observed: str = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + (project_dir / ".sqlbuild").mkdir(parents=True) + with closing(sqlite3.connect(project_dir / ".sqlbuild/control.sqlite3")) as store: + _ = store.execute( + "CREATE TABLE sqb_development_locations (project_path TEXT, target TEXT, " + "location_id TEXT)" + ) + _ = store.execute( + "CREATE TABLE sqb_warehouse_catalog (event_id TEXT, location_id TEXT, target TEXT, " + "state_database TEXT, state_schema TEXT, observed_at TEXT, record_json JSON)" + ) + _ = store.execute( + "INSERT INTO sqb_development_locations VALUES (?, ?, ?)", + (str(project_dir), target, location), + ) + _ = store.executemany( + "INSERT INTO sqb_warehouse_catalog VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + _mirrored_row( + relation=relation, + columns=columns.get(relation.identity, ()), + scope=(location, target, observed), + ) + for relation in relations + ], + ) + store.commit() + + +def _mirrored_row( + *, relation: RelationInfo, columns: tuple[ColumnInfo, ...], scope: tuple[str, str, str] +) -> tuple[str, ...]: + location, target, observed = scope + record: dict[str, object] = { + "state_database": "ANALYTICS", + "state_schema": str(relation.schema).upper(), + "name": relation.name, + "relation_type": relation.relation_type, + "columns": [{"name": column.name, "data_type": column.type} for column in columns], + } + return ( + str(uuid4()), + location, + target, + "ANALYTICS", + str(relation.schema).upper(), + observed, + json.dumps(record), + ) + + +def planned_entries(result: CompilePipelineResult) -> list[tuple[str, str]]: + """Each planned model's name and reason, in name order.""" + + return sorted((entry.name, str(entry.reason)) for entry in result.plan_output.model_entries) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py new file mode 100644 index 0000000000..f92caf9741 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py @@ -0,0 +1,123 @@ +"""A target opts into planning from its synced copy of warehouse state; other plans read live.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import CompilePipelineResult +from tests.integration.src.sqlbuild.compiler.planner.main._test_types import PlanFromTestCase +from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( + plan_offline_snowflake_project, + planned_entries, + write_synced_control_store, + write_targeted_snowflake_project, +) +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + RecordingSnowflakeWarehouse, + SyntheticSnowflakeProject, +) + +_TARGET: str = "dev" +_MODEL_SCHEMAS: tuple[str, ...] = ("staging", "intermediate", "marts") +_SYNCED: str = "Planned from synced state: checked in " +_NO_STORE: str = "Planning live: the project has no control store" +_LIVE: str = "Planning live:" + + +@pytest.mark.parametrize( + "test_case", + [ + PlanFromTestCase( + description="a target without the setting plans live and says nothing", + target_plan_from=None, + requested=None, + store=True, + expected_report="", + expected_absent=(_LIVE, _SYNCED), + expected_live_listing=True, + ), + PlanFromTestCase( + description="the target's setting plans from the synced copy after one check", + target_plan_from="synced_state", + requested=None, + store=True, + expected_report=_SYNCED, + expected_absent=(_LIVE,), + expected_live_listing=False, + ), + PlanFromTestCase( + description="the flag plans from the synced copy for a target without the setting", + target_plan_from=None, + requested="synced_state", + store=True, + expected_report=_SYNCED, + expected_absent=(_LIVE,), + expected_live_listing=False, + ), + PlanFromTestCase( + description="the flag plans live for a target with the setting", + target_plan_from="synced_state", + requested="live", + store=True, + expected_report="", + expected_absent=(_LIVE, _SYNCED), + expected_live_listing=True, + ), + PlanFromTestCase( + description="a project without a control store plans live and says why", + target_plan_from="synced_state", + requested=None, + store=False, + expected_report=_NO_STORE, + expected_absent=(_SYNCED,), + expected_live_listing=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_plan_from_when_planning_then_reads_the_synced_copy_only_when_asked( + test_case: PlanFromTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_CONTROL_DATABASE_URL", raising=False) + project: SyntheticSnowflakeProject = write_targeted_snowflake_project( + project_dir=tmp_path / "orders_platform", + target=_TARGET, + plan_from=test_case.target_plan_from, + ) + live: CompilePipelineResult = plan_offline_snowflake_project( + project=project, + warehouse=RecordingSnowflakeWarehouse(relations=project.relations), + plan_from="live", + ) + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse( + relations=project.relations + ) + write_synced_control_store( + project_dir=(tmp_path / "no_store", project.project_dir)[test_case.store], + target=_TARGET, + warehouse=warehouse, + schemas=_MODEL_SCHEMAS, + ) + warehouse.queries.clear() + _ = capsys.readouterr() + + result: CompilePipelineResult = plan_offline_snowflake_project( + project=project, warehouse=warehouse, plan_from=test_case.requested + ) + + err: str = capsys.readouterr().err + assert planned_entries(result) == planned_entries(live) + assert ( + any( + query.kind == "show_tables" and "STAGING" in f"{query.sql} {query.params}" + for query in warehouse.queries + ) + == test_case.expected_live_listing + ) + assert test_case.expected_report in err + assert not any(text in err for text in test_case.expected_absent) diff --git a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py index e00348448c..578de6b0cc 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py +++ b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py @@ -21,6 +21,7 @@ "adapters/databricks/constants.py", "adapters/postgres/classes/postgres_adapter.py", "adapters/snowflake/_helpers/metadata_types.py", + "adapters/snowflake/_helpers/relation_changes.py", "adapters/snowflake/_helpers/show_metadata.py", "adapters/snowflake/classes/snowflake_adapter.py", "adapters/sqlserver/classes/sqlserver_adapter.py", diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py index a38573a919..d0814cd101 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py @@ -1,6 +1,10 @@ from __future__ import annotations from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import FakeRelation @dataclass(frozen=True) @@ -225,3 +229,24 @@ class PartedListingTestCase: relations_per_schema: int statement_latency_seconds: float expected_max_concurrent: int + + +@dataclass(frozen=True) +class SyncedCatalogTestCase: + """A warehouse that moved on after the synced copy, and what a plan from the copy reads.""" + + description: str + changed: tuple[FakeRelation, ...] + dropped: tuple[str, ...] + expected_query_kinds: tuple[str, ...] + expected_refreshed: int + + +@dataclass(frozen=True) +class SyncedFallbackTestCase: + """A plan from synced state that cannot use the copy for a schema, and reads it live.""" + + description: str + empty_copy: bool + failing_check: bool + expected_message: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py index 4f9b934bb0..4fd598c1f5 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py @@ -21,7 +21,10 @@ import duckdb import pyarrow -from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog +from sqlbuild.adapter.relations.classes.synced_catalog import SyncedCatalog +from sqlbuild.adapter.relations.models import SyncedRelation from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection @@ -170,6 +173,7 @@ class FakeRelation: is_transient: bool = False columns: tuple[FakeColumn, ...] = (FakeColumn(name="ID", data_type="NUMBER"),) retention_time: int = 1 + last_ddl: datetime = datetime(2026, 1, 1, tzinfo=UTC) @dataclass(frozen=True) @@ -222,7 +226,8 @@ def __post_init__(self) -> None: self._database.execute( "CREATE TABLE fake_tables (table_catalog VARCHAR, table_schema VARCHAR, " "table_name VARCHAR, table_type VARCHAR, is_transient VARCHAR, " - "created TIMESTAMPTZ, last_altered TIMESTAMPTZ, retention_time INTEGER)" + "created TIMESTAMPTZ, last_altered TIMESTAMPTZ, last_ddl TIMESTAMPTZ, " + "retention_time INTEGER)" ) self._database.execute( "CREATE TABLE fake_columns (table_catalog VARCHAR, table_schema VARCHAR, " @@ -704,6 +709,7 @@ def _table_row(relation: FakeRelation) -> dict[str, object]: ], "created": _FIXED_CREATED_AT, "last_altered": _FIXED_CREATED_AT, + "last_ddl": relation.last_ddl, "retention_time": (relation.retention_time, None)[relation.table_type in _VIEW_TABLE_TYPES], } @@ -1081,3 +1087,71 @@ def attempted_query_kinds(warehouse: RecordingSnowflakeWarehouse) -> tuple[str, ).kind for sql in warehouse.attempted_sql ) + + +type ListedSchema = tuple[ + list[str], dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] +] + + +def live_inspection_catalog(warehouse: RecordingSnowflakeWarehouse) -> InspectionCatalog: + """A fresh catalog over its own adapter, reading `warehouse` live.""" + + adapter: OfflineSnowflakeAdapter = OfflineSnowflakeAdapter(warehouse=warehouse) + return InspectionCatalog(adapter=adapter, connection=adapter.connect({})) + + +def unavailable_relation_changes(**_: object) -> object: + """A synced state check whose warehouse read fails.""" + + raise RuntimeError("information schema unavailable") + + +def listed_schema(*, catalog: InspectionCatalog, database: str, schema: str) -> ListedSchema: + """One schema's listing, in an order-independent form, and its relations' columns.""" + + relations: tuple[RelationInfo, ...] = catalog.list_relations( + database=database, schemas=(schema,) + ) + return sorted_relation_reprs(relations), catalog.get_columns_for_relations(relations=relations) + + +def synced_copy( + *, warehouse: RecordingSnowflakeWarehouse, database: str, schema: str, observed_at: datetime +) -> SyncedCatalog: + """The copy a location synced: the live listing and columns, observed at `observed_at`.""" + + catalog: InspectionCatalog = live_inspection_catalog(warehouse) + relations: tuple[RelationInfo, ...] = catalog.list_relations( + database=database, schemas=(schema,) + ) + columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( + catalog.get_columns_for_relations(relations=relations) + ) + return SyncedCatalog( + relations=tuple( + SyncedRelation( + database=database.upper(), + schema=schema.upper(), + name=relation.name, + columns=columns.get(relation.identity, ()), + observed_at=observed_at, + ) + for relation in relations + ) + ) + + +def relations_after_sync( + *, + relations: tuple[FakeRelation, ...], + changed: tuple[FakeRelation, ...], + dropped: tuple[str, ...], +) -> tuple[FakeRelation, ...]: + """`relations` with `changed` replacing or joining them and `dropped` names removed.""" + + replaced: dict[str, FakeRelation] = {relation.name: relation for relation in relations} + replaced.update({relation.name: relation for relation in changed}) + for name in dropped: + replaced.pop(name) + return tuple(replaced.values()) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_synced_catalog.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_synced_catalog.py new file mode 100644 index 0000000000..d8421687ad --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_synced_catalog.py @@ -0,0 +1,176 @@ +"""A plan from synced state lists and describes schemas as a live read would, checking first.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog +from sqlbuild.adapter.relations.classes.synced_catalog import SyncedCatalog +from sqlbuild.adapter.relations.constants import SYNCED_CATALOG_ATTRIBUTE +from tests.unit.src.sqlbuild.adapters.snowflake.inspection._test_types import ( + SyncedCatalogTestCase, + SyncedFallbackTestCase, +) +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + FakeColumn, + FakeRelation, + ListedSchema, + build_offline_snowflake, + listed_schema, + live_inspection_catalog, + relations_after_sync, + synced_copy, + unavailable_relation_changes, +) + +_DATABASE: str = "analytics" +_SCHEMA: str = "staging" +_SYNCED_AT: datetime = datetime(2026, 2, 1, tzinfo=UTC) +_AFTER_SYNC: datetime = datetime(2026, 3, 1, tzinfo=UTC) +_RELATIONS: tuple[FakeRelation, ...] = ( + FakeRelation( + database="ANALYTICS", + schema="STAGING", + name="ORDERS", + columns=(FakeColumn(name="ID", data_type="NUMBER"), FakeColumn(name="STATUS")), + ), + FakeRelation(database="ANALYTICS", schema="STAGING", name="CUSTOMERS"), + FakeRelation(database="ANALYTICS", schema="STAGING", name="ORDERS_V", table_type="VIEW"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + SyncedCatalogTestCase( + description="an unchanged warehouse is checked once and described from the copy", + changed=(), + dropped=(), + expected_query_kinds=("tables",), + expected_refreshed=0, + ), + SyncedCatalogTestCase( + description="a table altered after the copy is the only one described again", + changed=( + FakeRelation( + database="ANALYTICS", + schema="STAGING", + name="ORDERS", + columns=( + FakeColumn(name="ID", data_type="NUMBER"), + FakeColumn(name="STATUS"), + FakeColumn(name="REGION"), + ), + last_ddl=_AFTER_SYNC, + ), + ), + dropped=(), + expected_query_kinds=("tables", "show_columns"), + expected_refreshed=1, + ), + SyncedCatalogTestCase( + description="a table created after the copy is listed and described", + changed=( + FakeRelation( + database="ANALYTICS", schema="STAGING", name="PAYMENTS", last_ddl=_AFTER_SYNC + ), + ), + dropped=(), + expected_query_kinds=("tables", "show_columns"), + expected_refreshed=1, + ), + SyncedCatalogTestCase( + description="a table dropped after the copy is no longer listed", + changed=(), + dropped=("CUSTOMERS",), + expected_query_kinds=("tables",), + expected_refreshed=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_synced_copy_when_listing_a_schema_then_matches_a_live_read_after_one_check( + test_case: SyncedCatalogTestCase, +) -> None: + adapter, connection, warehouse = build_offline_snowflake(relations=_RELATIONS) + synced: SyncedCatalog = synced_copy( + warehouse=warehouse, database=_DATABASE, schema=_SCHEMA, observed_at=_SYNCED_AT + ) + warehouse.replace_relations( + relations=relations_after_sync( + relations=_RELATIONS, changed=test_case.changed, dropped=test_case.dropped + ) + ) + warehouse.queries.clear() + setattr(adapter, SYNCED_CATALOG_ATTRIBUTE, synced) + + planned: ListedSchema = listed_schema( + catalog=InspectionCatalog(adapter=adapter, connection=connection), + database=_DATABASE, + schema=_SCHEMA, + ) + + assert sorted(query.kind for query in warehouse.queries) == sorted( + test_case.expected_query_kinds + ) + assert planned == listed_schema( + catalog=live_inspection_catalog(warehouse), database=_DATABASE, schema=_SCHEMA + ) + noun: str = ("tables", "table")[test_case.expected_refreshed == 1] + report: str | None = synced.report() + assert report is not None and report.startswith("Planned from synced state: checked in ") + assert report.endswith(f", {test_case.expected_refreshed} {noun} refreshed") + + +@pytest.mark.parametrize( + "test_case", + [ + SyncedFallbackTestCase( + description="a failing check reads the schema live and says why", + empty_copy=False, + failing_check=True, + expected_message="Planning live: the synced state check failed", + ), + SyncedFallbackTestCase( + description="a schema outside the copy is read live without a check", + empty_copy=True, + failing_check=False, + expected_message="", + ), + ], + ids=lambda case: case.description, +) +def test_given_unusable_copy_when_listing_then_reads_the_schema_live( + test_case: SyncedFallbackTestCase, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter, connection, warehouse = build_offline_snowflake(relations=_RELATIONS) + copies: tuple[SyncedCatalog, SyncedCatalog] = ( + synced_copy( + warehouse=warehouse, database=_DATABASE, schema=_SCHEMA, observed_at=_SYNCED_AT + ), + SyncedCatalog(relations=()), + ) + setattr(adapter, SYNCED_CATALOG_ATTRIBUTE, copies[test_case.empty_copy]) + monkeypatch.setattr( + adapter, + "read_relation_changes", + (adapter.read_relation_changes, unavailable_relation_changes)[test_case.failing_check], + ) + warehouse.queries.clear() + + planned: ListedSchema = listed_schema( + catalog=InspectionCatalog(adapter=adapter, connection=connection), + database=_DATABASE, + schema=_SCHEMA, + ) + + assert {"show_tables", "show_views"} <= {query.kind for query in warehouse.queries} + assert "tables" not in {query.kind for query in warehouse.queries} + assert planned == listed_schema( + catalog=live_inspection_catalog(warehouse), database=_DATABASE, schema=_SCHEMA + ) + assert test_case.expected_message in capsys.readouterr().err diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index ec6578fac4..32e0406766 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -47,6 +47,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--reach` | With `--changed`: `job` stops at models another scheduled job refreshes, `all` builds everything downstream | | `--defer-to` | Resolve unselected model references against another target | | `--defer-sources-to` | Read managed source data from another target | +| `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](/docs/concepts/project-configuration#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](/docs/concepts/testing-reference#time-limits) | diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 1b3ade695c..55f7910385 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -23,6 +23,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--no-python` | Exclude read-side Python tasks and assets from the plan | | `--defer-to` | Resolve unselected model references against another target | +| `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](/docs/concepts/project-configuration#planning-from-synced-state) | | `--as` | Compile for another target and show its plan, including [model migration](/docs/concepts/models/migrations) decisions, using the current connection. Inspection only. | | `--json` | Output the plan as JSON | | `--timings` | After the plan, print how long each planning step took; see [Timings](#timings) | diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 908c9fd342..402839d6f9 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -162,6 +162,7 @@ A target is a named build context - the database and schema you build into, plus | `warehouses` | Snowflake warehouse per command group: `build`, `query` and `sync` (a location's warehouse syncs). See [Command-group warehouses](/docs/concepts/adapters/snowflake#command-group-warehouses) | | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures) | +| `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | ```toml [connections.warehouse] @@ -255,6 +256,37 @@ The value must be an integer of at least `1`; anything else fails with `D001`. ` `8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and uses the active target's value. +### Planning from synced state + +Every plan reads the warehouse's catalog: which tables and views exist and their columns. On a +large project that is most of a plan's time. A target can instead plan from the copy of the +catalog that `sqb dev` keeps synced in the project's control store, after checking it against +the warehouse: + +```toml +[targets.dev] +schema = "analytics_dev" +plan_from = "synced_state" +``` + +`--plan-from live` or `--plan-from synced_state` on `sqb plan` or `sqb build` overrides the target +for one invocation. The default is `live`; SQLBuild cannot tell which target is production, so +planning from synced state is something a target opts into. + +At the start of the plan, one `INFORMATION_SCHEMA.TABLES` read per database lists the schemas the +copy holds, with each table's last DDL. The plan uses that listing, so tables created or dropped +since the last sync are seen. It reads columns from the copy, except for tables whose last DDL is +later than the copy's observation of them (allowing five minutes for clock skew), which it +describes afresh. When more than 128 tables changed, it reads the schema's columns live instead. +Cursor bounds and incremental state are still read live. + +The plan reads live, and says why on stderr (`Planning live: ...`), when the target's adapter +isn't Snowflake (the only adapter that can check the copy), the project has no control store, no +location syncs the target for this project, the copy is empty, or the check fails. Otherwise it reports `Planned from synced state: checked in 1.2s, 3 tables refreshed`. The +control store is `.sqlbuild/control.sqlite3` in the project, or the database +`SQLBUILD_CONTROL_DATABASE_URL` names. Planning never writes to it; only the location's sync +refreshes the copy. + ### Clone policies Targets can declare whether they allow cloning to or from: From a023a2cd7fb9f02a33baf3de028b00c71ae26092 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 02:40:22 +0000 Subject: [PATCH 11/17] test(perf): bound the 10K live plan by the location's publish, not the warm preview The live plan was bounded at six times the warm preview. Once the warm preview got faster, live plans that had not changed came out at 5.3-6.4 times it on a loaded machine, so the ratio failed for reasons that have nothing to do with live planning. Bound the live plan by the location's publish instead, which compiles the same project on the same machine. Measured at 1.22-1.40 across six runs, quiet and loaded; the limit is 1.8. The warm preview keeps its 10 second limit. --- .../sqlbuild/cli/commands/main/plan/_test_types.py | 4 ++-- .../plan/test_duckshop_10k_preview_performance.py | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index 68e56474f0..f11a5b6db6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -87,8 +87,8 @@ class PreviewPerformanceTestCase: #: The warm preview's share of the location's publish, a linear 10K compile on one machine. expected_max_warm_to_publish: float expected_max_check_seconds: float - #: A live plan of the same warehouse, compile included, as a multiple of the warm preview. - expected_max_live_to_warm: float + #: A live plan of the same warehouse, compile included, as a multiple of the location's publish. + expected_max_live_to_publish: float #: Selected models analysed again against inspected source columns in the live plan. expected_source_rebinding: tuple[int, int] #: Every statement the live plan sends the warehouse, cursor batches included. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py index f43f7d80ba..2c621ff0a7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py @@ -9,7 +9,8 @@ Each case also plans the project live, as `sqb plan` does, against an offline Snowflake warehouse holding the same synced catalog, a few sources carrying an undeclared column. It bounds the models analysed again against inspected source columns, the warehouse statements sent, cursor batching -and the live plan's time as a multiple of the warm preview's. +and the live plan's time as a multiple of the location's publish, which compiles the same +project on the same machine. It runs in the 10,000-model cell of the dense compile job, after the compile guards. """ @@ -60,7 +61,7 @@ expected_max_warm_seconds=10.0, expected_max_warm_to_publish=0.45, expected_max_check_seconds=0.5, - expected_max_live_to_warm=6.0, + expected_max_live_to_publish=1.8, expected_source_rebinding=(25, 150), expected_max_warehouse_statements=1_150, ), @@ -72,7 +73,7 @@ expected_max_warm_seconds=10.0, expected_max_warm_to_publish=0.45, expected_max_check_seconds=0.5, - expected_max_live_to_warm=6.0, + expected_max_live_to_publish=1.8, expected_source_rebinding=(25, 150), expected_max_warehouse_statements=1_150, ), @@ -140,7 +141,7 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then f"publish={served.published_seconds:.2f}s warm_to_publish={warm_to_publish:.2f} " f"check_median={check_seconds * 1000:.0f}ms runs={len(reasons)} unbuilt={unbuilt} " f"definition_changes={definition_changes} models={len(document['models'])} " - f"live={live.seconds:.2f}s live_to_warm={live.seconds / warm:.2f} " + f"live={live.seconds:.2f}s live_to_publish={live.seconds / served.published_seconds:.2f} " f"live_entries={live.model_entries} rebinding={live.source_rebinding_analyses} " f"cursor_relations={live.cursor_relations} cursor_statements={live.cursor_statements} " f"warehouse_statements={live.warehouse_statements} " @@ -174,6 +175,6 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then "cursor bounds are read in full batches" ) assert live.warehouse_statements <= test_case.expected_max_warehouse_statements - assert live.seconds / warm < test_case.expected_max_live_to_warm, ( - "a live plan grew faster than the warm preview of the same warehouse" + assert live.seconds / served.published_seconds < test_case.expected_max_live_to_publish, ( + "a live plan grew faster than compiling and publishing the same project" ) From b6c6fea3cb9fa98f18cd967e90c40a781922e10f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 04:10:33 +0000 Subject: [PATCH 12/17] feat(python-nodes): depend on a job or a tag Tasks, assets and checks can declare depends_on=job("orders_job") or tag("finance"), each with an optional exclude= of ordinary selectors. Compile resolves them into edges to the set's end nodes: its models, seeds, sources, loaders, tasks and assets, less the exclude, that nothing else in the set reads. Planning, selector expansion, --changed, sqb dag and job group waits then treat them like named dependencies, and a model named and reached through a set is one edge. Compile fails for an unknown job or tag, a set its exclude empties, a loader declaring one, a code location prefix (reserved), a set node reading the dependant's output (naming the nearest reader and the exclude that breaks the cycle), and a job group the new edges would make wait in a loop, as groups wait job by job today; that error names the nodes closing the loop and suggests moving them to another job. Each node's identity records the end nodes it resolved, and the plan lists what it now also waits on, or no longer waits on, since its last build, in text and as wait_changes in --json. Published Python node definitions carry set_dependencies for the UI's lineage to use. --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../_helpers/combined.rs | 142 +++++++++ .../_helpers/group_loops.rs | 176 +++++++++++ .../node_set_dependencies/_helpers/labels.rs | 58 ++++ .../src/node_set_dependencies/_helpers/mod.rs | 4 + .../node_set_dependencies/_helpers/sets.rs | 280 ++++++++++++++++++ .../src/node_set_dependencies/constants.rs | 26 ++ .../src/node_set_dependencies/main/mod.rs | 3 + .../main/resolve_node_set_dependencies.rs | 88 ++++++ .../main/set_dependency_metadata.rs | 19 ++ .../main/wait_changes.rs | 55 ++++ .../src/node_set_dependencies/mod.rs | 9 + .../src/node_set_dependencies/models.rs | 44 +++ .../src/node_set_dependencies/tests.rs | 8 + .../node_set_dependencies/tests/helpers.rs | 211 +++++++++++++ .../test_resolve_node_set_dependencies.rs | 173 +++++++++++ .../node_set_dependencies/tests/test_types.rs | 30 ++ .../tests/test_wait_changes.rs | 50 ++++ .../src/cli_text/_helpers/plan_sections.rs | 6 + crates/sqlbuild-core/src/cli_text/models.rs | 2 + .../src/entries/_helpers/user_code.rs | 5 + crates/sqlbuild-host/src/entries/models.rs | 1 + crates/sqlbuild-python-base/src/types.rs | 6 + .../compile_driver/_helpers/setup/project.rs | 7 +- .../_helpers/records/declaration_records.rs | 13 + .../_helpers/snapshots/native_classes.rs | 2 + .../discovery/main/records/keep_node_sets.rs | 22 ++ .../src/discovery/main/records/mod.rs | 2 + .../main/records/stored_node_sets.rs | 19 ++ .../src/plan_output/_helpers/plan_json.rs | 6 + .../src/plan_output/_helpers/plan_text.rs | 12 +- .../src/planner_migrations/models.rs | 4 +- .../compiled_project/_helpers/job_catalog.rs | 13 +- .../compiled_project/main/declared_jobs.rs | 14 + .../src/compiled_project/main/mod.rs | 1 + .../src/graph/_helpers/python_nodes/mod.rs | 1 + .../python_nodes/python_node_graph.rs | 184 +++++++++++- .../_helpers/python_nodes/set_dependencies.rs | 46 +++ .../src/graph/main/python_nodes/mod.rs | 1 + .../main/python_nodes/project_graph_error.rs | 10 + .../main/python_nodes/python_node_graph.rs | 54 +--- .../main/python_nodes/resolve_node_sets.rs | 58 ++++ .../src/graph/models.rs | 8 + .../_helpers/location_command/snapshot.rs | 26 ++ fensu.toml | 8 +- .../docs/concepts/python-nodes/overview.md | 2 +- .../concepts/python-nodes/sql-references.md | 58 +++- src/sqlbuild/_native_host.py | 1 + src/sqlbuild/assets.py | 6 +- src/sqlbuild/checks.py | 7 +- .../_helpers/validation/discovery.py | 3 +- .../python_nodes/_helpers/execution.py | 5 +- .../_helpers/dependency_normalization.py | 11 +- src/sqlbuild/python_nodes/main/apply_asset.py | 8 +- src/sqlbuild/python_nodes/main/apply_check.py | 9 +- src/sqlbuild/python_nodes/main/apply_task.py | 9 +- src/sqlbuild/python_nodes/models.py | 23 +- src/sqlbuild/python_nodes/types.py | 7 + src/sqlbuild/refs.py | 26 +- src/sqlbuild/tasks.py | 7 +- .../sqlbuild/compiler/pipeline/_test_types.py | 29 ++ .../src/sqlbuild/compiler/pipeline/helpers.py | 43 +++ .../pipeline/test_set_dependencies.py | 214 +++++++++++++ .../python_nodes/decorators/_test_types.py | 10 + .../decorators/test_python_decorators.py | 40 ++- .../docs/concepts/python-nodes/overview.mdx | 2 +- .../concepts/python-nodes/sql-references.mdx | 58 +++- 67 files changed, 2396 insertions(+), 90 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/combined.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/group_loops.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/main/resolve_node_set_dependencies.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/main/set_dependency_metadata.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/main/wait_changes.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/mod.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/models.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/tests.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_wait_changes.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/keep_node_sets.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/records/stored_node_sets.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs create mode 100644 crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs create mode 100644 crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs create mode 100644 tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index ca5a7c5549..5bb2c63eae 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -23,6 +23,7 @@ pub mod lineage; pub mod lineage_command; pub mod lineage_graph; pub mod manifest; +pub mod node_set_dependencies; pub mod nullability_rules; pub mod planner_changes; pub mod planner_fixtures; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/combined.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/combined.rs new file mode 100644 index 0000000000..467db9f76b --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/combined.rs @@ -0,0 +1,142 @@ +//! The combined SQL and Python dependency graph a set dependency is reduced and checked over. + +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; + +use sqlbuild_core::graph_walk::main::find_cycle::find_cycle; + +use crate::node_set_dependencies::constants::SOURCE_KIND; +use crate::node_set_dependencies::models::GraphNode; +use crate::python_selection::models::{PythonSelectionGraph, SqlSelectionGraph}; + +/// Each node's upstream and downstream nodes, a source's loader upstream of it. +pub(crate) struct Combined { + upstream: HashMap>, + downstream: HashMap>, + python: Vec, +} + +impl Combined { + pub(crate) fn new(sql: SqlSelectionGraph<'_>, python: &PythonSelectionGraph) -> Self { + let mut upstream: HashMap> = HashMap::new(); + for (key, dependencies) in &sql.graph.upstream { + upstream + .entry(GraphNode::Sql(key.clone())) + .or_default() + .extend(dependencies.iter().cloned().map(GraphNode::Sql)); + } + for (source, loader) in sql.sources { + if let Some(loader) = loader { + upstream + .entry(GraphNode::Sql((SOURCE_KIND.to_owned(), source.clone()))) + .or_default() + .push(GraphNode::Python(loader.clone())); + } + } + for (name, dependencies) in &python.upstream { + upstream + .entry(GraphNode::Python(name.clone())) + .or_default() + .extend(dependencies.iter().cloned().map(GraphNode::Python)); + } + for node in &python.nodes { + upstream + .entry(GraphNode::Python(node.name.clone())) + .or_default() + .extend(node.sql_deps.iter().cloned().map(GraphNode::Sql)); + } + let mut downstream: HashMap> = HashMap::new(); + for (node, dependencies) in &upstream { + for dependency in dependencies { + downstream + .entry(dependency.clone()) + .or_default() + .push(node.clone()); + } + } + Self { + upstream, + downstream, + python: python + .nodes + .iter() + .map(|node| GraphNode::Python(node.name.clone())) + .collect(), + } + } + + pub(crate) fn upstream(&self, node: &GraphNode) -> &[GraphNode] { + self.upstream.get(node).map_or(&[], Vec::as_slice) + } + + /// Every node one or more steps upstream of `starts`, walking only nodes `through` allows. + pub(crate) fn reach_upstream<'a>( + &self, + starts: impl IntoIterator, + through: impl Fn(&GraphNode) -> bool, + ) -> HashSet { + self.reach(starts, |node| self.upstream(node), through) + } + + /// Every node one or more steps downstream of `start`. + pub(crate) fn reach_downstream(&self, start: &GraphNode) -> HashSet { + self.reach( + [start], + |node| self.downstream.get(node).map_or(&[], Vec::as_slice), + |_| true, + ) + } + + fn reach<'a, 'n>( + &'n self, + starts: impl IntoIterator, + next: impl Fn(&GraphNode) -> &'n [GraphNode], + through: impl Fn(&GraphNode) -> bool, + ) -> HashSet { + let mut reached: HashSet = HashSet::new(); + let mut queue: VecDeque = starts.into_iter().cloned().collect(); + while let Some(node) = queue.pop_front() { + for neighbour in next(&node) { + if reached.insert(neighbour.clone()) && through(neighbour) { + queue.push_back(neighbour.clone()); + } + } + } + reached + } + + /// A dependency cycle through a Python node, as the path from it back to itself. + pub(crate) fn python_cycle(&self) -> Option> { + let path: Vec = find_cycle(self.python.iter().cloned(), |node| { + self.upstream(node).to_vec() + })?; + let last: &GraphNode = path.last()?; + let start: usize = path.iter().position(|node| node == last)?; + Some(path[start..].to_vec()) + } + + /// The shortest path from `from` up to `to`, both included. + pub(crate) fn path_upstream(&self, from: &GraphNode, to: &GraphNode) -> Vec { + let mut parents: HashMap = HashMap::new(); + let mut queue: VecDeque = VecDeque::from([from.clone()]); + let mut seen: BTreeSet = BTreeSet::from([from.clone()]); + while let Some(node) = queue.pop_front() { + if node == *to { + let mut path: Vec = vec![node.clone()]; + let mut current: GraphNode = node; + while let Some(parent) = parents.get(¤t) { + path.push(parent.clone()); + current = parent.clone(); + } + path.reverse(); + return path; + } + for neighbour in self.upstream(&node) { + if seen.insert(neighbour.clone()) { + parents.insert(neighbour.clone(), node.clone()); + queue.push_back(neighbour.clone()); + } + } + } + vec![from.clone(), to.clone()] + } +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/group_loops.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/group_loops.rs new file mode 100644 index 0000000000..deb16dd4b7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/group_loops.rs @@ -0,0 +1,176 @@ +//! Job groups wait job by job: a job dependency that makes two members wait for each other is a cycle. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use sqlbuild_core::graph_walk::main::find_cycle::find_cycle; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::{JobSelection, ProjectGraph}; +use crate::node_set_dependencies::_helpers::combined::Combined; +use crate::node_set_dependencies::_helpers::labels::{listed, node_label}; +use crate::node_set_dependencies::_helpers::sets::{SetMembers, failure}; +use crate::node_set_dependencies::constants::JOB_SET; +use crate::node_set_dependencies::models::GraphNode; + +/// Each member job and the nodes it selects. +type Members = Vec<(String, HashSet)>; +/// Which members each member waits for, each wait with one `(their node, our node)` reason. +type Waits = BTreeMap>; + +/// The error for the first job group a job dependency makes wait in a loop. +pub(crate) fn group_loop( + jobs: &ProjectGraph, + catalog: &BTreeMap, + (explicit, current): (&Combined, &Combined), + (sets, kinds): (&[SetMembers<'_>], &HashMap), +) -> Option { + for (group, selection) in catalog { + let JobSelection::Group { members } = selection else { + continue; + }; + let members: Members = members + .iter() + .filter_map(|member| Some((member.clone(), job_nodes(jobs, member, kinds)?))) + .collect(); + let touched: bool = sets.iter().any(|set| crosses_members(set, &members)); + if !touched || wait_cycle(&waits(&members, explicit)).is_some() { + continue; + } + let after: Waits = waits(&members, current); + let Some(cycle) = wait_cycle(&after) else { + continue; + }; + let before: Waits = waits(&members, explicit); + return Some(loop_error(group, &cycle, (&before, &after), kinds)); + } + None +} + +/// Whether `set` is a job of the group that a node of another of its jobs depends on. +fn crosses_members(set: &SetMembers<'_>, members: &Members) -> bool { + let dependant: GraphNode = GraphNode::Python(set.dependant.name.clone()); + let names: Vec<&String> = members.iter().map(|(name, _)| name).collect(); + let elsewhere: bool = members + .iter() + .filter(|(name, _)| *name != set.dependency.name) + .any(|(_, nodes)| nodes.contains(&dependant)); + set.dependency.kind == JOB_SET && names.contains(&&set.dependency.name) && elsewhere +} + +/// The nodes `job` selects, Python nodes by name; `None` when it did not resolve. +fn job_nodes( + jobs: &ProjectGraph, + job: &str, + kinds: &HashMap, +) -> Option> { + let Some(Ok(keys)) = jobs.jobs.as_ref()?.get(job) else { + return None; + }; + Some( + keys.iter() + .map(|key: &ObjectKey| match kinds.get(&key.1) { + Some(kind) if *kind == key.0 => GraphNode::Python(key.1.clone()), + _ => GraphNode::Sql(key.clone()), + }) + .collect(), + ) +} + +/// What each member waits for: another member's node feeding its own, maybe through unselected nodes. +fn waits(members: &Members, combined: &Combined) -> Waits { + let selected: HashSet<&GraphNode> = members.iter().flat_map(|(_, nodes)| nodes).collect(); + let mut found: Waits = BTreeMap::new(); + for (member, own) in members { + for to in own { + for up in combined.upstream(to) { + if own.contains(up) { + continue; + } + let froms: Vec = if selected.contains(up) { + vec![up.clone()] + } else { + combined + .reach_upstream([up], |node| !selected.contains(node)) + .into_iter() + .filter(|node| selected.contains(node)) + .collect() + }; + for from in froms { + if own.contains(&from) { + continue; + } + for (other, theirs) in members { + if other != member && theirs.contains(&from) { + found + .entry(member.clone()) + .or_default() + .entry(other.clone()) + .or_insert_with(|| (from.clone(), to.clone())); + } + } + } + } + } + } + found +} + +/// The members of a wait cycle, the first waiting for the second and the last for the first. +fn wait_cycle(waits: &Waits) -> Option> { + let path: Vec = find_cycle(waits.keys().cloned(), |member: &String| { + waits + .get(member) + .map(|waited| waited.keys().cloned().collect::>()) + .unwrap_or_default() + })?; + let last: &String = path.last()?; + let start: usize = path.iter().position(|member| member == last)?; + Some(path[start..path.len() - 1].to_vec()) +} + +fn loop_error( + group: &str, + cycle: &[String], + (before, after): (&Waits, &Waits), + kinds: &HashMap, +) -> SelectorError { + let mut steps: Vec = Vec::new(); + let mut closers: BTreeSet = BTreeSet::new(); + let mut dependants: BTreeSet = BTreeSet::new(); + for (index, member) in cycle.iter().enumerate() { + let waited: &String = &cycle[(index + 1) % cycle.len()]; + let Some((from, to)) = after.get(member).and_then(|waits| waits.get(waited)) else { + continue; + }; + let added: bool = before + .get(member) + .is_none_or(|waits| !waits.contains_key(waited)); + let (from, to): (String, String) = (node_label(from, kinds), node_label(to, kinds)); + if added { + steps.push(format!( + "'{member}' waits for '{waited}' ({to} depends on job '{waited}', which ends in {from})" + )); + dependants.insert(format!("{to} out of '{member}'")); + } else { + steps.push(format!( + "'{member}' waits for '{waited}' ('{member}''s {to} reads {from})" + )); + closers.insert(to); + } + } + let closers: Vec = closers.into_iter().collect(); + let dependants: Vec = dependants.into_iter().collect(); + failure( + format!( + "a job dependency makes job group '{group}' wait in a loop: {}", + steps.join("; ") + ), + Some(format!( + "a job group runs each job once its jobs upstream have finished, so these jobs would \ + wait for each other; move {} to another job, or move {}", + listed(&closers), + listed(&dependants) + )), + ) +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs new file mode 100644 index 0000000000..d5614d3884 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs @@ -0,0 +1,58 @@ +//! How job and tag dependency errors name nodes and sets. + +use std::collections::HashMap; + +use crate::node_set_dependencies::constants::NAMED_NODES; +use crate::node_set_dependencies::models::{GraphNode, NodeSetDependency}; + +/// `model 'orders'` or `task 'export_orders'`. +pub(crate) fn node_label(node: &GraphNode, kinds: &HashMap) -> String { + match node { + GraphNode::Sql((kind, name)) => format!("{kind} '{name}'"), + GraphNode::Python(name) => { + let kind: &str = kinds.get(name).map_or("node", String::as_str); + format!("{kind} '{name}'") + } + } +} + +/// The node's selector name, as an `exclude` lists it. +pub(crate) fn node_name(node: &GraphNode) -> &str { + match node { + GraphNode::Sql((_, name)) | GraphNode::Python(name) => name, + } +} + +/// `job 'nightly'` or `tag 'finance'`. +pub(crate) fn set_label(dependency: &NodeSetDependency) -> String { + format!("{} '{}'", dependency.kind, dependency.name) +} + +/// The authoring call that leaves `names` out of the dependency. +pub(crate) fn excluding(dependency: &NodeSetDependency, names: &[&str]) -> String { + let mut tokens: Vec<&str> = dependency + .exclude + .as_deref() + .map(|exclude| exclude.split_whitespace().collect()) + .unwrap_or_default(); + tokens.extend(names.iter().copied()); + format!( + "{}(\"{}\", exclude=\"{}\")", + dependency.kind, + dependency.name, + tokens.join(" ") + ) +} + +/// Up to [`NAMED_NODES`] labels joined, the rest counted. +pub(crate) fn listed(labels: &[String]) -> String { + let shown: Vec<&str> = labels + .iter() + .take(NAMED_NODES) + .map(String::as_str) + .collect(); + match labels.len().saturating_sub(NAMED_NODES) { + 0 => shown.join(", "), + more => format!("{} and {more} more", shown.join(", ")), + } +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/mod.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/mod.rs new file mode 100644 index 0000000000..b8c6beb9b3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod combined; +pub(crate) mod group_loops; +pub(crate) mod labels; +pub(crate) mod sets; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs new file mode 100644 index 0000000000..8399a32435 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs @@ -0,0 +1,280 @@ +//! One set dependency's members and end nodes, the edges they add, and the cycles those close. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use crate::graph::errors::SelectorError; +use crate::graph::models::JobSelection; +use crate::node_set_dependencies::_helpers::combined::Combined; +use crate::node_set_dependencies::_helpers::labels::{ + excluding, listed, node_label, node_name, set_label, +}; +use crate::node_set_dependencies::constants::{ + DEPENDABLE_PYTHON_KINDS, DEPENDABLE_SQL_KINDS, JOB_SET, LOADER_KIND, LOCATION_SEPARATOR, + NODE_SET_CODE, TAG_SET, +}; +use crate::node_set_dependencies::models::{ + DependantNode, GraphNode, NodeSetDependency, ResolvedNode, +}; +use crate::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; +use crate::python_selection::models::{PythonSelectionGraph, RunSelection, SqlSelectionGraph}; + +pub(crate) fn failure(message: String, help: Option) -> SelectorError { + SelectorError { + code: NODE_SET_CODE, + message, + help, + } +} + +/// Each Python node's kind, by name. +pub(crate) fn python_kinds(python: &PythonSelectionGraph) -> HashMap { + python + .nodes + .iter() + .map(|node| (node.name.clone(), node.kind.clone())) + .collect() +} + +/// The dependable nodes `dependency` names, less its `exclude`. +pub(crate) fn members( + (sql, python): (SqlSelectionGraph<'_>, &PythonSelectionGraph), + (dependant, dependency): (&DependantNode, &NodeSetDependency), + catalog: &BTreeMap, +) -> Result, SelectorError> { + let owner: String = format!("{} '{}'", dependant.kind, dependant.name); + let set: String = set_label(dependency); + if dependant.kind == LOADER_KIND { + return Err(failure( + format!("{owner} depends on {set}; loaders cannot depend on a job or a tag"), + Some("name the loaders, tasks or assets it depends on one by one".to_owned()), + )); + } + if dependency.name.contains(LOCATION_SEPARATOR) { + return Err(failure( + format!("{owner} depends on {set}, a job of another code location"), + Some("depending on another location's jobs is not supported yet".to_owned()), + )); + } + if dependency.kind == JOB_SET && !catalog.contains_key(&dependency.name) { + return Err(failure( + format!("{owner} depends on {set}, which the project does not declare"), + Some( + "name a job or job group from orchestration/jobs or orchestration/job_groups" + .to_owned(), + ), + )); + } + let tagged: bool = sql + .graph + .tags + .iter() + .any(|(tag, _)| *tag == dependency.name) + || python.tags.iter().any(|(tag, _)| *tag == dependency.name); + if dependency.kind == TAG_SET && !tagged { + return Err(failure( + format!("{owner} depends on {set}, which no node carries"), + None, + )); + } + let exclude: Vec = dependency + .exclude + .as_deref() + .map(|exclude| exclude.split_whitespace().map(str::to_owned).collect()) + .unwrap_or_default(); + let selected: RunSelection = resolve_python_sql_nodes( + sql, + python, + &[format!("{}:{}", dependency.kind, dependency.name)], + &exclude, + ) + .map_err(|error| { + failure( + format!("{owner} depends on {set}: {}", error.message), + error.help, + ) + })?; + let kinds: HashMap = python_kinds(python); + let found: BTreeSet = selected + .sql_keys + .into_iter() + .filter(|(kind, _)| DEPENDABLE_SQL_KINDS.contains(&kind.as_str())) + .map(GraphNode::Sql) + .chain( + selected + .python_node_names + .into_iter() + .filter(|name| { + kinds + .get(name) + .is_some_and(|kind| DEPENDABLE_PYTHON_KINDS.contains(&kind.as_str())) + }) + .map(GraphNode::Python), + ) + .collect(); + if found.is_empty() { + let after: &str = if exclude.is_empty() { + "" + } else { + " after its exclude" + }; + return Err(failure( + format!( + "{owner} depends on {set}, which leaves no model, seed, source, loader, task or \ + asset{after}" + ), + None, + )); + } + Ok(found) +} + +/// The members nothing else in the set reads, directly or through other nodes. +pub(crate) fn end_nodes(combined: &Combined, members: &BTreeSet) -> Vec { + let read: HashSet = combined.reach_upstream(members, |_| true); + members + .iter() + .filter(|member| !read.contains(*member)) + .cloned() + .collect() +} + +/// `python` with each resolved set's end nodes added to its node's dependencies. +pub(crate) fn with_edges( + python: &PythonSelectionGraph, + resolved: &[ResolvedNode], +) -> PythonSelectionGraph { + let mut graph: PythonSelectionGraph = python.clone(); + for node in resolved { + for end in node.sets.iter().flat_map(|set| &set.ends) { + match end { + GraphNode::Sql(key) => { + if let Some(owner) = graph.nodes.iter_mut().find(|item| item.name == node.name) + && !owner.sql_deps.contains(key) + { + owner.sql_deps.push(key.clone()); + } + } + GraphNode::Python(upstream) => { + graph.upstream = + with_edge(std::mem::take(&mut graph.upstream), &node.name, upstream); + graph.downstream = + with_edge(std::mem::take(&mut graph.downstream), upstream, &node.name); + } + } + } + } + graph +} + +/// `rows` with `other` listed against `name` once. +fn with_edge( + mut rows: Vec<(String, Vec)>, + name: &str, + other: &str, +) -> Vec<(String, Vec)> { + match rows.iter_mut().find(|(row, _)| row == name) { + Some((_, names)) if names.iter().any(|existing| existing == other) => {} + Some((_, names)) => names.push(other.to_owned()), + None => rows.push((name.to_owned(), vec![other.to_owned()])), + } + rows +} + +/// The `readers` no other reader stands between and `dependant`. +fn nearest_readers<'a>( + explicit: &Combined, + readers: &[&'a GraphNode], + dependant: &GraphNode, +) -> Vec<&'a GraphNode> { + let mut nearest: Vec<&GraphNode> = Vec::new(); + for reader in readers { + let reached: HashSet = + explicit.reach_upstream([*reader], |node| !readers.contains(&node)); + if reached.contains(dependant) { + nearest.push(reader); + } + } + nearest +} + +/// One resolved set with the members it was reduced from, for cycle errors. +pub(crate) struct SetMembers<'a> { + pub(crate) dependant: &'a DependantNode, + pub(crate) dependency: &'a NodeSetDependency, + pub(crate) members: BTreeSet, + pub(crate) ends: Vec, +} + +/// The set whose added edge is the cycle step `pair`, a dependant reading one of its end nodes. +fn closing_set<'s, 'a>( + explicit: &Combined, + sets: &'s [SetMembers<'a>], + pair: &[GraphNode], +) -> Option<&'s SetMembers<'a>> { + let GraphNode::Python(name) = &pair[0] else { + return None; + }; + let named: bool = explicit.upstream(&pair[0]).contains(&pair[1]); + sets.iter() + .find(|set| set.dependant.name == *name && set.ends.contains(&pair[1]) && !named) +} + +/// The error for a dependency cycle a set dependency closes; `None` when every cycle predates them. +pub(crate) fn set_cycle( + (explicit, current): (&Combined, &Combined), + sets: &[SetMembers<'_>], + kinds: &HashMap, +) -> Option { + let cycle: Vec = current.python_cycle()?; + let (index, set): (usize, &SetMembers<'_>) = cycle + .windows(2) + .enumerate() + .find_map(|(index, pair)| Some((index, closing_set(explicit, sets, pair)?)))?; + let dependant: GraphNode = cycle[index].clone(); + let owner: String = node_label(&dependant, kinds); + let label: String = set_label(set.dependency); + if set.members.contains(&dependant) { + return Some(failure( + format!("{owner} depends on {label}, which includes {owner} itself"), + Some(format!( + "leave it out of the dependency: {}", + excluding(set.dependency, &[node_name(&dependant)]) + )), + )); + } + let downstream: HashSet = explicit.reach_downstream(&dependant); + let readers: Vec<&GraphNode> = set + .members + .iter() + .filter(|member| downstream.contains(*member)) + .collect(); + let nearest: Vec<&GraphNode> = nearest_readers(explicit, &readers, &dependant); + let closer: &GraphNode = nearest.first().copied().or(readers.first().copied())?; + let path: Vec = explicit + .path_upstream(closer, &dependant) + .iter() + .map(|node| node_label(node, kinds)) + .collect(); + let nearest_labels: Vec = nearest.iter().map(|node| node_label(node, kinds)).collect(); + let reader_names: Vec<&str> = readers.iter().map(|node| node_name(node)).collect(); + let owner_of_set: String = match set.dependency.kind.as_str() { + JOB_SET => format!("{label}'s"), + _ => format!("the {label} node"), + }; + let remedy: &str = match set.dependency.kind.as_str() { + JOB_SET => "move it to another job", + _ => "remove the tag from it", + }; + Some(failure( + format!( + "{owner} depends on {label}, but {owner_of_set} {} reads {owner}'s output ({}), \ + a dependency cycle", + listed(&nearest_labels), + path.join(" <- ") + ), + Some(format!( + "{remedy}, or leave every node of the set that reads {owner} out of the dependency: {}", + excluding(set.dependency, &reader_names) + )), + )) +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs new file mode 100644 index 0000000000..363915abc0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs @@ -0,0 +1,26 @@ +//! Kinds, codes and messages of job and tag dependencies. + +/// A dependency on a job's (or job group's) selection. +pub(crate) const JOB_SET: &str = "job"; +/// A dependency on every node carrying a tag. +pub(crate) const TAG_SET: &str = "tag"; +/// The planner code these compile errors carry. +pub(crate) const NODE_SET_CODE: &str = "S000"; +/// SQL resource types a Python node may depend on. +pub(crate) const DEPENDABLE_SQL_KINDS: [&str; 3] = ["model", "seed", "source"]; +/// Python node kinds a Python node may depend on. +pub(crate) const DEPENDABLE_PYTHON_KINDS: [&str; 3] = ["loader", "task", "asset"]; +/// The Python node kind that may depend on Python nodes only. +pub(crate) const LOADER_KIND: &str = "loader"; +/// The SQL resource type a loader fills. +pub(crate) const SOURCE_KIND: &str = "source"; +/// What separates a code location from a job name in a cross-location dependency. +pub(crate) const LOCATION_SEPARATOR: char = ':'; +/// Nodes a cycle error names one by one before summarising the rest. +pub(crate) const NAMED_NODES: usize = 5; +/// The identity metadata key a Python node's resolved set dependencies are recorded under. +pub(crate) const SET_DEPENDENCIES_KEY: &str = "set_dependencies"; +/// Each recorded set's label key. +pub(crate) const SET_KEY: &str = "set"; +/// Each recorded set's end node labels key. +pub(crate) const ENDS_KEY: &str = "ends"; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/main/mod.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/main/mod.rs new file mode 100644 index 0000000000..aa5575f475 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/main/mod.rs @@ -0,0 +1,3 @@ +pub mod resolve_node_set_dependencies; +pub mod set_dependency_metadata; +pub mod wait_changes; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/main/resolve_node_set_dependencies.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/main/resolve_node_set_dependencies.rs new file mode 100644 index 0000000000..7122804f0b --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/main/resolve_node_set_dependencies.rs @@ -0,0 +1,88 @@ +//! `resolve_node_set_dependencies`: each `job(...)` and `tag(...)` dependency as edges to end nodes. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::graph::errors::SelectorError; +use crate::graph::models::{JobSelection, ProjectGraph}; +use crate::node_set_dependencies::_helpers::combined::Combined; +use crate::node_set_dependencies::_helpers::group_loops::group_loop; +use crate::node_set_dependencies::_helpers::sets::{ + SetMembers, end_nodes, members, python_kinds, set_cycle, with_edges, +}; +use crate::node_set_dependencies::models::{DependantNode, GraphNode, ResolvedNode, ResolvedSet}; +use crate::python_selection::main::attach_python_jobs::attach_python_jobs; +use crate::python_selection::models::{PythonSelectionGraph, SqlSelectionGraph}; + +/// Each dependant's sets reduced to end nodes, resolved again over added edges until none change. +pub fn resolve_node_set_dependencies( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + dependants: &[DependantNode], + catalog: &BTreeMap, +) -> Result, SelectorError> { + if dependants.is_empty() { + return Ok(Vec::new()); + } + let jobs: ProjectGraph = attach_python_jobs(sql, python, catalog.clone()); + let with_jobs: SqlSelectionGraph<'_> = SqlSelectionGraph { + graph: &jobs, + ..sql + }; + let kinds: HashMap = python_kinds(python); + let explicit: Combined = Combined::new(sql, python); + let mut current: PythonSelectionGraph = python.clone(); + let mut sets: Vec> = Vec::new(); + for _ in 0..=dependants.len() { + let combined: Combined = Combined::new(sql, ¤t); + sets = Vec::new(); + for dependant in dependants { + for dependency in &dependant.sets { + let found: BTreeSet = + members((with_jobs, ¤t), (dependant, dependency), catalog)?; + let ends: Vec = end_nodes(&combined, &found); + sets.push(SetMembers { + dependant, + dependency, + members: found, + ends, + }); + } + } + let next: PythonSelectionGraph = with_edges(python, &resolved(&sets)); + if let Some(error) = set_cycle((&explicit, &Combined::new(sql, &next)), &sets, &kinds) { + return Err(error); + } + if next == current { + break; + } + current = next; + } + let resolved: Vec = resolved(&sets); + let current: Combined = Combined::new(sql, ¤t); + match group_loop(&jobs, catalog, (&explicit, ¤t), (&sets, &kinds)) { + Some(error) => Err(error), + None => Ok(resolved), + } +} + +/// Each dependant's resolved sets, in declaration order. +fn resolved(sets: &[SetMembers<'_>]) -> Vec { + let mut nodes: Vec = Vec::new(); + for set in sets { + let resolved: ResolvedSet = ResolvedSet { + dependency: set.dependency.clone(), + ends: set.ends.clone(), + }; + match nodes + .iter_mut() + .find(|node| node.name == set.dependant.name) + { + Some(node) => node.sets.push(resolved), + None => nodes.push(ResolvedNode { + name: set.dependant.name.clone(), + sets: vec![resolved], + }), + } + } + nodes +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/main/set_dependency_metadata.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/main/set_dependency_metadata.rs new file mode 100644 index 0000000000..a4cc6d095b --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/main/set_dependency_metadata.rs @@ -0,0 +1,19 @@ +//! `set_dependency_metadata`: a Python node's identity metadata with its resolved sets recorded. + +use serde_json::{Value, json}; + +use crate::node_set_dependencies::constants::{ENDS_KEY, SET_DEPENDENCIES_KEY, SET_KEY}; + +/// `metadata_json` with each `(set label, end node labels)` recorded; a non-object is unchanged. +#[must_use] +pub fn set_dependency_metadata(metadata_json: &str, sets: &[(String, Vec)]) -> String { + let Ok(Value::Object(mut metadata)) = serde_json::from_str::(metadata_json) else { + return metadata_json.to_owned(); + }; + let recorded: Vec = sets + .iter() + .map(|(set, ends)| json!({ SET_KEY: set, ENDS_KEY: ends })) + .collect(); + metadata.insert(SET_DEPENDENCIES_KEY.to_owned(), Value::Array(recorded)); + Value::Object(metadata).to_string() +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/main/wait_changes.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/main/wait_changes.rs new file mode 100644 index 0000000000..5cd0f9eda1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/main/wait_changes.rs @@ -0,0 +1,55 @@ +//! `wait_changes`: what a Python node newly waits on, or no longer, through its set dependencies. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +use crate::node_set_dependencies::constants::{ENDS_KEY, SET_DEPENDENCIES_KEY, SET_KEY}; + +/// Each end node `node` gained or lost per set since its last build; nothing if that recorded none. +#[must_use] +pub fn wait_changes(node: &str, previous: Option<&str>, current: Option<&str>) -> Vec { + let (Some(previous), Some(current)) = (previous.and_then(sets), current.and_then(sets)) else { + return Vec::new(); + }; + let labels: BTreeSet<&String> = previous.keys().chain(current.keys()).collect(); + let empty: BTreeSet = BTreeSet::new(); + let mut lines: Vec = Vec::new(); + for label in labels { + let before: &BTreeSet = previous.get(label).unwrap_or(&empty); + let after: &BTreeSet = current.get(label).unwrap_or(&empty); + lines.extend( + after + .difference(before) + .map(|end| format!("{node} now also waits on {end} through {label}")), + ); + lines.extend( + before + .difference(after) + .map(|end| format!("{node} no longer waits on {end} through {label}")), + ); + } + lines +} + +/// Each recorded set's end node labels, or `None` when the metadata records no sets. +fn sets(metadata_json: &str) -> Option>> { + let Ok(metadata) = serde_json::from_str::(metadata_json) else { + return None; + }; + let recorded: &Vec = metadata.get(SET_DEPENDENCIES_KEY)?.as_array()?; + Some(recorded.iter().filter_map(recorded_set).collect()) +} + +/// One recorded set's label and end node labels. +fn recorded_set(set: &Value) -> Option<(String, BTreeSet)> { + let label: String = set.get(SET_KEY)?.as_str()?.to_owned(); + let ends: BTreeSet = set + .get(ENDS_KEY)? + .as_array()? + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(); + Some((label, ends)) +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/mod.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/mod.rs new file mode 100644 index 0000000000..888f54a0fd --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/mod.rs @@ -0,0 +1,9 @@ +//! Python-node dependencies on a job or a tag, resolved into edges to the set's end nodes. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs new file mode 100644 index 0000000000..e8b824ace1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs @@ -0,0 +1,44 @@ +//! A Python node's job and tag dependencies and the end nodes each resolves to. + +use crate::assembly::project::types::ObjectKey; + +/// One `job(...)` or `tag(...)` dependency as authored. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeSetDependency { + /// `job` or `tag`. + pub kind: String, + pub name: String, + /// Selector tokens removed from the set before it is reduced to its end nodes. + pub exclude: Option, +} + +/// A Python node and the set dependencies it declares. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DependantNode { + pub name: String, + /// The node's kind, such as `task`. + pub kind: String, + pub sets: Vec, +} + +/// A node of the combined SQL and Python graph. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum GraphNode { + Sql(ObjectKey), + Python(String), +} + +/// One set dependency and the end nodes it resolved to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResolvedSet { + pub dependency: NodeSetDependency, + /// The set's dependable nodes nothing else in the set reads, in a stable order. + pub ends: Vec, +} + +/// A Python node's set dependencies, each resolved. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResolvedNode { + pub name: String, + pub sets: Vec, +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests.rs new file mode 100644 index 0000000000..78ba424993 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_resolve_node_set_dependencies.rs"] +mod resolve_node_set_dependencies; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_wait_changes.rs"] +mod wait_changes; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs new file mode 100644 index 0000000000..716ac53e18 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs @@ -0,0 +1,211 @@ +use std::collections::{BTreeMap, HashMap}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::main::build_project_graph::build_project_graph; +use crate::graph::models::{GraphResource, JobSelection, ProjectGraph}; +use crate::node_set_dependencies::_helpers::labels::node_label; +use crate::node_set_dependencies::main::resolve_node_set_dependencies::resolve_node_set_dependencies; +use crate::node_set_dependencies::main::set_dependency_metadata::set_dependency_metadata; +use crate::node_set_dependencies::models::{DependantNode, NodeSetDependency, ResolvedNode}; +use crate::python_selection::main::python_node_graph_indexes::python_node_graph_indexes; +use crate::python_selection::models::{ + InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, SqlSelectionGraph, +}; + +/// `(name, kind, tags, upstream nodes, sql deps)`. +type NodeSpec = ( + &'static str, + &'static str, + &'static [&'static str], + &'static [&'static str], + &'static [(&'static str, &'static str)], +); +/// `(kind, name, exclude)` of one set dependency. +pub(super) type SetSpec = (&'static str, &'static str, Option<&'static str>); + +const NODES: &[NodeSpec] = &[ + ("report", "task", &["reports"], &[], &[]), + ("export_orders", "task", &[], &[], &[("model", "orders")]), + ( + "audit_export", + "task", + &[], + &["export_orders", "report"], + &[], + ), + ("self_tagged", "task", &["finance"], &[], &[]), + ("sync_events", "loader", &[], &[], &[]), +]; + +/// `(resource type, name, tags, dependencies)` of each SQL resource. +const RESOURCES: &[(&str, &str, &[&str], &[&str])] = &[ + ("source", "raw_orders", &[], &[]), + ("model", "stg_orders", &[], &["source:raw_orders"]), + ("model", "orders", &["finance"], &["model:stg_orders"]), + ("model", "order_items", &["finance"], &["model:stg_orders"]), + ( + "model", + "finance_daily", + &[], + &["model:orders", "model:order_items"], + ), + ("model", "customers", &["crm"], &[]), + ( + "model", + "customer_orders", + &[], + &["model:customers", "model:orders"], + ), +]; + +/// Each job's `select`; a group lists its member jobs. +const JOBS: &[(&str, &str)] = &[ + ("nightly", "stg_orders orders order_items"), + ("single", "customers"), + ("diamond", "stg_orders orders order_items finance_daily"), + ("with_task", "orders task:export_orders"), + ("loop", "task:export_orders task:audit_export"), + ("nightly_wide", "orders customer_orders"), + ("reporting", "task:report customers"), +]; + +fn key(spec: &str) -> ObjectKey { + let (kind, name) = spec.split_once(':').unwrap_or(("model", spec)); + (kind.to_owned(), name.to_owned()) +} + +fn owned(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +fn sql_graph() -> ProjectGraph { + build_project_graph( + &RESOURCES + .iter() + .map(|(kind, name, tags, deps)| GraphResource { + key: ((*kind).to_owned(), (*name).to_owned()), + deps: deps.iter().map(|dep| key(dep)).collect(), + tags: owned(tags), + folder: (*kind == "model").then(String::new), + }) + .collect::>(), + ) +} + +fn python_graph() -> PythonSelectionGraph { + let inventory: Vec = NODES + .iter() + .map(|(name, _, tags, upstream, _)| InventoryNode { + name: (*name).to_owned(), + tags: owned(tags), + relative_path: format!("python/{name}.py"), + upstream: owned(upstream), + }) + .collect(); + let indexes: InventoryIndexes = python_node_graph_indexes(&inventory); + PythonSelectionGraph { + nodes: NODES + .iter() + .map(|(name, kind, _, _, sql_deps)| PythonSelectionNode { + name: (*name).to_owned(), + kind: (*kind).to_owned(), + sql_deps: sql_deps + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(), + }) + .collect(), + upstream: indexes.upstream, + downstream: indexes.downstream, + tags: indexes.tags, + paths: indexes.paths, + } +} + +/// The fixture's jobs and the job groups `groups` lists. +fn catalog(groups: &[(&str, &[&str])]) -> BTreeMap { + let mut catalog: BTreeMap = JOBS + .iter() + .map(|(name, select)| { + ( + (*name).to_owned(), + JobSelection::Job { + select: select.split_whitespace().map(str::to_owned).collect(), + exclude: Vec::new(), + }, + ) + }) + .collect(); + catalog.extend(groups.iter().map(|(name, members)| { + ( + (*name).to_owned(), + JobSelection::Group { + members: owned(members), + }, + ) + })); + catalog +} + +/// The sets of `dependant` resolved over the fixture project. +pub(super) fn resolve( + (dependant, kind): (&str, &str), + sets: &[SetSpec], + groups: &[(&str, &[&str])], +) -> Result, SelectorError> { + let graph: ProjectGraph = sql_graph(); + let sources: Vec<(String, Option)> = vec![("raw_orders".to_owned(), None)]; + resolve_node_set_dependencies( + SqlSelectionGraph { + graph: &graph, + sources: &sources, + allow_warehouse_round_trips: false, + }, + &python_graph(), + &[DependantNode { + name: dependant.to_owned(), + kind: kind.to_owned(), + sets: sets + .iter() + .map(|(kind, name, exclude)| NodeSetDependency { + kind: (*kind).to_owned(), + name: (*name).to_owned(), + exclude: exclude.map(str::to_owned), + }) + .collect(), + }], + &catalog(groups), + ) +} + +/// Each resolved set's end nodes, labelled as errors name them. +pub(super) fn ends(resolved: &[ResolvedNode]) -> Vec> { + let kinds: HashMap = NODES + .iter() + .map(|spec| (spec.0.to_owned(), spec.1.to_owned())) + .collect(); + resolved + .iter() + .flat_map(|node| &node.sets) + .map(|set| set.ends.iter().map(|end| node_label(end, &kinds)).collect()) + .collect() +} + +/// The fragments `text` lacks. +pub(super) fn missing<'a>(text: &str, fragments: &[&'a str]) -> Vec<&'a str> { + fragments + .iter() + .filter(|fragment| !text.contains(**fragment)) + .copied() + .collect() +} + +/// Base identity metadata with `sets` recorded as `(set label, end node labels)`. +pub(super) fn recorded(base: &str, sets: &[(&str, &[&str])]) -> String { + let owned_sets: Vec<(String, Vec)> = sets + .iter() + .map(|(set, ends)| ((*set).to_owned(), owned(ends))) + .collect(); + set_dependency_metadata(base, &owned_sets) +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs new file mode 100644 index 0000000000..9c3fb98dec --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs @@ -0,0 +1,173 @@ +use crate::node_set_dependencies::tests::helpers::{ends, missing, resolve}; +use crate::node_set_dependencies::tests::test_types::{EndNodesTestCase, FailureTestCase}; + +/// The group running `nightly_wide` and `reporting` together. +const GROUP: &[(&str, &[&str])] = &[("daily_group", &["nightly_wide", "reporting"])]; + +#[test] +fn given_job_and_tag_dependencies_when_resolving_then_each_reduces_to_its_end_nodes() { + let test_cases = [ + EndNodesTestCase { + description: "a job whose selection branches ends in each branch", + groups: &[], + sets: &[("job", "nightly", None)], + expected_ends: &[&["model 'order_items'", "model 'orders'"]], + }, + EndNodesTestCase { + description: "a job of one node ends in it", + groups: &[], + sets: &[("job", "single", None)], + expected_ends: &[&["model 'customers'"]], + }, + EndNodesTestCase { + description: "a diamond ends in the node joining its branches", + groups: &[], + sets: &[("job", "diamond", None)], + expected_ends: &[&["model 'finance_daily'"]], + }, + EndNodesTestCase { + description: "a Python node reading the job's model is its end", + groups: &[], + sets: &[("job", "with_task", None)], + expected_ends: &[&["task 'export_orders'"]], + }, + EndNodesTestCase { + description: "exclude removes nodes before reducing to end nodes", + groups: &[], + sets: &[("job", "diamond", Some("finance_daily"))], + expected_ends: &[&["model 'order_items'", "model 'orders'"]], + }, + EndNodesTestCase { + description: "exclude takes tag selectors", + groups: &[], + sets: &[("job", "diamond", Some("tag:finance finance_daily"))], + expected_ends: &[&["model 'stg_orders'"]], + }, + EndNodesTestCase { + description: "a tag ends in its tagged nodes nothing tagged reads", + groups: &[], + sets: &[("tag", "crm", None)], + expected_ends: &[&["model 'customers'"]], + }, + EndNodesTestCase { + description: "a tag excluding a Python node keeps its SQL", + groups: &[], + sets: &[("tag", "finance", Some("self_tagged"))], + expected_ends: &[&["model 'order_items'", "model 'orders'"]], + }, + EndNodesTestCase { + description: "a job reading the dependant's job resolves when no group runs them together", + groups: &[], + sets: &[("job", "nightly_wide", None)], + expected_ends: &[&["model 'customer_orders'"]], + }, + EndNodesTestCase { + description: "each of several sets resolves on its own", + groups: &[], + sets: &[("job", "single", None), ("tag", "crm", None)], + expected_ends: &[&["model 'customers'"], &["model 'customers'"]], + }, + ]; + for test_case in test_cases { + let resolved = resolve(("report", "task"), test_case.sets, test_case.groups) + .expect(test_case.description); + assert_eq!( + ends(&resolved), + test_case.expected_ends, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_cause() { + let test_cases = [ + FailureTestCase { + description: "an unknown job", + dependant: ("report", "task"), + groups: &[], + sets: &[("job", "weekly", None)], + expected_fragments: &[ + "task 'report' depends on job 'weekly', which the project does not declare", + ], + }, + FailureTestCase { + description: "an unknown tag", + dependant: ("report", "task"), + groups: &[], + sets: &[("tag", "marketing", None)], + expected_fragments: &[ + "task 'report' depends on tag 'marketing', which no node carries", + ], + }, + FailureTestCase { + description: "a set its exclude empties", + dependant: ("report", "task"), + groups: &[], + sets: &[("job", "single", Some("customers"))], + expected_fragments: &[ + "depends on job 'single', which leaves no model", + "after its exclude", + ], + }, + FailureTestCase { + description: "another location's job", + dependant: ("report", "task"), + groups: &[], + sets: &[("job", "billing:nightly", None)], + expected_fragments: &["a job of another code location"], + }, + FailureTestCase { + description: "a loader's set", + dependant: ("sync_events", "loader"), + groups: &[], + sets: &[("job", "single", None)], + expected_fragments: &[ + "loader 'sync_events' depends on job 'single'; loaders cannot depend on a job or a tag", + ], + }, + FailureTestCase { + description: "a job node reading the dependant's output closes a cycle", + dependant: ("report", "task"), + groups: &[], + sets: &[("job", "loop", None)], + expected_fragments: &[ + "task 'report' depends on job 'loop', but job 'loop''s task 'audit_export' reads task 'report''s output", + "job(\"loop\", exclude=\"audit_export\")", + ], + }, + FailureTestCase { + description: "a tag the dependant carries includes it", + dependant: ("self_tagged", "task"), + groups: &[], + sets: &[("tag", "finance", None)], + expected_fragments: &[ + "task 'self_tagged' depends on tag 'finance', which includes task 'self_tagged' itself", + "tag(\"finance\", exclude=\"self_tagged\")", + ], + }, + FailureTestCase { + description: "a group whose jobs would wait for each other", + dependant: ("report", "task"), + groups: GROUP, + sets: &[("job", "nightly_wide", None)], + expected_fragments: &[ + "job group 'daily_group' wait in a loop", + "'nightly_wide' waits for 'reporting' ('nightly_wide''s model 'customer_orders' reads model 'customers')", + "move model 'customer_orders' to another job", + ], + }, + ]; + for test_case in test_cases { + let error = resolve(test_case.dependant, test_case.sets, test_case.groups) + .expect_err(test_case.description); + let text: String = format!("{} {}", error.message, error.help.unwrap_or_default()); + assert_eq!( + missing(&text, test_case.expected_fragments), + Vec::<&str>::new(), + "{}: {text}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs new file mode 100644 index 0000000000..8bc4c18a1c --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs @@ -0,0 +1,30 @@ +/// One set dependency of the `report` task and the end nodes it resolves to. +pub(super) struct EndNodesTestCase { + pub(super) description: &'static str, + /// `(kind, name, exclude)` of each `job(...)` or `tag(...)` dependency. + pub(super) sets: &'static [(&'static str, &'static str, Option<&'static str>)], + /// The job groups the catalog holds, by name and members. + pub(super) groups: &'static [(&'static str, &'static [&'static str])], + /// Each end node as errors name it, per set. + pub(super) expected_ends: &'static [&'static [&'static str]], +} + +/// A dependency that cannot resolve, and fragments its error must carry. +pub(super) struct FailureTestCase { + pub(super) description: &'static str, + /// The dependant's name and kind. + pub(super) dependant: (&'static str, &'static str), + pub(super) sets: &'static [(&'static str, &'static str, Option<&'static str>)], + /// The job groups the catalog holds, by name and members. + pub(super) groups: &'static [(&'static str, &'static [&'static str])], + pub(super) expected_fragments: &'static [&'static str], +} + +/// The sets a node was last built with and has now, and the wait changes a plan lists. +pub(super) struct WaitChangesTestCase { + pub(super) description: &'static str, + /// `None` when the last build recorded no sets. + pub(super) previous: Option<&'static [(&'static str, &'static [&'static str])]>, + pub(super) current: &'static [(&'static str, &'static [&'static str])], + pub(super) expected_lines: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_wait_changes.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_wait_changes.rs new file mode 100644 index 0000000000..0fd5956839 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_wait_changes.rs @@ -0,0 +1,50 @@ +use crate::node_set_dependencies::main::wait_changes::wait_changes; +use crate::node_set_dependencies::tests::helpers::recorded; +use crate::node_set_dependencies::tests::test_types::WaitChangesTestCase; + +const BASE: &str = r#"{"dependencies":[]}"#; + +#[test] +fn given_set_ends_that_changed_since_the_last_build_when_planning_then_lists_each_wait_change() { + let test_cases = [ + WaitChangesTestCase { + description: "a job gaining an end node", + previous: Some(&[("job 'nightly_job'", &["model 'orders'"])]), + current: &[("job 'nightly_job'", &["model 'orders'", "model 'items'"])], + expected_lines: &["report now also waits on model 'items' through job 'nightly_job'"], + }, + WaitChangesTestCase { + description: "a job losing an end node", + previous: Some(&[("job 'nightly_job'", &["model 'orders'", "model 'items'"])]), + current: &[("job 'nightly_job'", &["model 'orders'"])], + expected_lines: &["report no longer waits on model 'items' through job 'nightly_job'"], + }, + WaitChangesTestCase { + description: "unchanged end nodes", + previous: Some(&[("tag 'finance'", &["model 'orders'"])]), + current: &[("tag 'finance'", &["model 'orders'"])], + expected_lines: &[], + }, + WaitChangesTestCase { + description: "a build that recorded no sets", + previous: None, + current: &[("tag 'finance'", &["model 'orders'"])], + expected_lines: &[], + }, + ]; + for test_case in test_cases { + let previous: String = test_case + .previous + .map_or_else(|| BASE.to_owned(), |sets| recorded(BASE, sets)); + assert_eq!( + wait_changes( + "report", + Some(&previous), + Some(&recorded(BASE, test_case.current)) + ), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs index 5af09aa592..3a2ce7ce86 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -252,6 +252,12 @@ pub(crate) fn python_entry_lines( width, true, )]; + lines.extend( + entry + .wait_changes + .iter() + .map(|change| STYLE.muted(&format!(" {change}"))), + ); lines.extend(identity_diff_lines(entry)); lines }) diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 4e45437716..34817c0b7b 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -364,6 +364,8 @@ pub struct PythonPlanText { pub identity_status: String, pub source_diff: Vec, pub dependency_diff: Vec, + /// What the node now also waits on, or no longer, through its job and tag dependencies. + pub wait_changes: Vec, } /// One provider used by a selected Python surface. diff --git a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs index fa4ea13ddc..8e77164e4a 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs @@ -234,6 +234,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::UserCode, }, + HostEntry { + name: "NodeSetRef", + kind: HostKind::Class, + edge: HostEdge::UserCode, + }, HostEntry { name: "PythonCheckSeverity", kind: HostKind::Class, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 8fca0613e7..4c39f96c72 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -175,6 +175,7 @@ pub enum UserCodeHost { AuditCase, RelationRenderer, SqlResourceRef, + NodeSetRef, PythonCheckSeverity, SqlResourceRefKind, LoadCustomRules, diff --git a/crates/sqlbuild-python-base/src/types.rs b/crates/sqlbuild-python-base/src/types.rs index d7009884c7..97b617f0fd 100644 --- a/crates/sqlbuild-python-base/src/types.rs +++ b/crates/sqlbuild-python-base/src/types.rs @@ -17,3 +17,9 @@ pub trait FromPositional: Sized { pub trait IntoPositional { fn into_positional<'py>(self, py: Python<'py>) -> PyResult>; } + +/// Each Python node's resolved job and tag dependencies, or why they did not resolve. +pub type NodeSetOutcome = Result< + Vec, + sqlbuild_analysis::graph::errors::SelectorError, +>; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs index 8fcac2dff0..5af92903d1 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs @@ -12,6 +12,7 @@ use pyo3::pyfunction; use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; +use sqlbuild_python_project::graph::main::python_nodes::resolve_node_sets::resolve_node_sets; use crate::compile_driver::_helpers::setup::target_defaults::TargetDefaults; use crate::compile_driver::_helpers::setup::target_defaults::apply_target_defaults; @@ -125,10 +126,12 @@ pub(crate) fn compiled_project( py: Python<'_>, request: ProjectRequest, ) -> PyResult> { - produced( + let project: Bound<'_, PyAny> = produced( FrontierStage::CompiledProject, python_compiled_project(py, &request)?, - ) + )?; + let _ = resolve_node_sets(request.discovered_inputs.bind(py), &project)?; + Ok(project) } /// `_build_python_compiled_project`. diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs index e1c0c868e3..436c151826 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs @@ -36,6 +36,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; @@ -367,6 +368,17 @@ fn macro_file<'py>( context.record(RecordId::DiscoveredMacroFile, &fields) } +/// The discovery session `discovered` inputs were read in, which keeps their resolved set dependencies. +pub(crate) fn discovered_session<'py>( + discovered: &Bound<'py, PyAny>, +) -> PyResult>> { + let session: Bound<'py, PyAny> = native_field(discovered, "native_session")?; + if session.is_none() { + return Ok(None); + } + Ok(Some(session.cast_into::()?)) +} + /// The session of this pass, created and memoised by the first declaration collection. pub(crate) fn discovery_session<'py>( project_dir: &Bound<'py, PyAny>, @@ -382,6 +394,7 @@ pub(crate) fn discovery_session<'py>( py, NativeDiscoverySession { inner: DiscoverySession::new(project_root(project_dir)?, declaration_file_options()), + node_sets: std::sync::Mutex::new(None), }, )?; memo.set_item(SESSION_MEMO_KEY, &created)?; diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs index 6d3dca266f..0467afb706 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs @@ -42,6 +42,8 @@ pub(crate) struct ModelTextRequest { #[derive(Debug)] pub(crate) struct NativeDiscoverySession { pub(crate) inner: sqlbuild_discovery::declaration_files::models::DiscoverySession, + /// The Python nodes' job and tag dependencies, as the latest compile of these inputs resolved them. + pub(crate) node_sets: std::sync::Mutex>, } /// One in-memory declaration file parse request; `kind` names the collection it belongs to. diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/keep_node_sets.rs b/crates/sqlbuild-python-front/src/discovery/main/records/keep_node_sets.rs new file mode 100644 index 0000000000..c93f30f00e --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/keep_node_sets.rs @@ -0,0 +1,22 @@ +//! Keep a compile's resolved job and tag dependencies with the discovered inputs it compiled. + +use std::sync::PoisonError; + +use crate::discovery::_helpers::records::declaration_records::discovered_session; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::types::NodeSetOutcome; + +/// Keep `outcome` in the discovery session `discovered` was read in, when it has one. +pub fn keep_node_sets( + discovered: &Bound<'_, PyAny>, + outcome: Option, +) -> PyResult<()> { + if let Some(session) = discovered_session(discovered)? { + *session + .get() + .node_sets + .lock() + .unwrap_or_else(PoisonError::into_inner) = outcome; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs index 3e85c0fcd2..87ce8c6245 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs @@ -14,6 +14,7 @@ pub mod discover_sql_hook_files; pub mod discover_test_files; pub mod discovered_model_files; pub mod externally_described_models; +pub mod keep_node_sets; pub mod parse_sql_test_file; pub mod projected_text; pub mod register_declaration_records; @@ -22,3 +23,4 @@ pub mod register_model_records; pub mod register_sql_test_records; pub mod register_yaml_records; pub mod retained_discovery_session; +pub mod stored_node_sets; diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/stored_node_sets.rs b/crates/sqlbuild-python-front/src/discovery/main/records/stored_node_sets.rs new file mode 100644 index 0000000000..8c75209723 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/stored_node_sets.rs @@ -0,0 +1,19 @@ +//! The job and tag dependencies the latest compile of discovered inputs resolved. + +use std::sync::PoisonError; + +use crate::discovery::_helpers::records::declaration_records::discovered_session; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::types::NodeSetOutcome; + +/// The resolved set dependencies the latest compile of `discovered` kept, if any. +pub fn stored_node_sets(discovered: &Bound<'_, PyAny>) -> PyResult> { + Ok(discovered_session(discovered)?.and_then(|session| { + session + .get() + .node_sets + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + })) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index 6e450f2ef4..1f1979f407 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -10,6 +10,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use crate::plan_output::_helpers::cursor_details::{CursorDetails, cursor_details, owner}; +use crate::plan_output::_helpers::plan_text::entry_wait_changes; use crate::plan_output::_helpers::plan_text::{model_config_diff, model_query_diff}; use crate::plan_output::_helpers::python_identity::{ definition_source_text, dependency_source_text, @@ -468,6 +469,11 @@ pub(crate) fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = + entry_wait_changes(entry, &native_field(entry, "name")?.extract::()?)?; + if !changes.is_empty() { + item.set_item("wait_changes", changes)?; + } Ok(item.into_any()) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 95f586c794..10a1bfb6ac 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -6,6 +6,7 @@ use crate::plan_output::main::items::items; use crate::plan_output::main::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::node_set_dependencies::main::wait_changes::wait_changes; use sqlbuild_core::cli_text::models::{ CliStyle, CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, PlanWarningText, ProviderUsageText, PythonPlanText, RenameHintText, SchemaFindingText, @@ -300,6 +301,13 @@ fn source_load_text(entry: &Bound<'_, PyAny>) -> PyResult { }) } +/// What a Python entry now also waits on, or no longer, since the build its identity was recorded by. +pub(crate) fn entry_wait_changes(entry: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let previous: Option = native_field(entry, "previous_metadata_json")?.extract()?; + let current: Option = native_field(entry, "current_metadata_json")?.extract()?; + Ok(wait_changes(name, previous.as_deref(), current.as_deref())) +} + /// Both identity diffs of a changed Python entry, each as `format_query_diff` lines. fn python_text(entry: &Bound<'_, PyAny>) -> PyResult { let identity_status: String = text(&native_field( @@ -331,8 +339,10 @@ fn python_text(entry: &Bound<'_, PyAny>) -> PyResult { } } let [source_diff, dependency_diff] = diffs; + let name: String = text(&native_field(entry, "name")?)?; Ok(PythonPlanText { - name: text(&native_field(entry, "name")?)?, + wait_changes: entry_wait_changes(entry, &name)?, + name, kind: text(&native_field(&native_field(entry, "kind")?, "value")?)?, phase: text(&native_field(&native_field(entry, "phase")?, "value")?)?, identity_status, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs index e59c81e680..9dbc21f7eb 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs @@ -1,8 +1,8 @@ //! Public records of planner migrations. use crate::planner_migrations::_helpers::discovery::state_inspection::RelationKey; -use crate::planner_migrations::_helpers::discovery::state_inspection::lowered; use crate::planner_migrations::_helpers::discovery::state_inspection::fingerprint_key; +use crate::planner_migrations::_helpers::discovery::state_inspection::lowered; use crate::planner_migrations::_helpers::discovery::state_inspection::pending; use crate::planner_migrations::_helpers::discovery::state_inspection::relation_key; use crate::planner_migrations::_helpers::discovery::state_inspection::schema_or_empty; @@ -31,7 +31,6 @@ use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; -use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; @@ -40,6 +39,7 @@ use sqlbuild_python_front::state_rows::constants::MODEL_KIND; use sqlbuild_python_front::state_rows::constants::OLD_NAME_KIND; use sqlbuild_python_front::state_rows::main::migrations::layout::layout; use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; +use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; use sqlbuild_state::events::models::{EventRelation, FingerprintKey}; use std::collections::HashMap; use std::collections::HashSet; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index 53d25310cf..25cad5a3e8 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -98,9 +98,16 @@ pub(crate) fn job_catalog(py: Python<'_>) -> PyResult> { /// Each job's selector tokens and each group's members declared for `target`, and the names a schedule or sensor starts there. fn inventory_catalog(inventory: &OrchestrationInventory, target: Option<&str>) -> Catalog { - let declared_for = |targets: &[String]| { + catalog_where(inventory, |targets: &[String]| { targets.is_empty() || target.is_some_and(|target| targets.iter().any(|name| name == target)) - }; + }) +} + +/// Each job's selector tokens and each group's members `declared_for` accepts the targets of. +pub(crate) fn catalog_where( + inventory: &OrchestrationInventory, + declared_for: impl Fn(&[String]) -> bool, +) -> Catalog { let tokens = |value: &str| value.split_whitespace().map(str::to_owned).collect(); let mut catalog: BTreeMap = BTreeMap::new(); let mut triggered: BTreeSet = BTreeSet::new(); @@ -148,7 +155,7 @@ fn inventory_catalog(inventory: &OrchestrationInventory, target: Option<&str>) - ); } for sensor in &inventory.sensors { - if sensor.targets.as_deref().is_none_or(declared_for) { + if sensor.targets.as_deref().is_none_or(&declared_for) { triggered.extend( sensor .job_references diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs new file mode 100644 index 0000000000..6510eb7a92 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs @@ -0,0 +1,14 @@ +//! Every job and group a project declares, for resolving `job(...)` dependencies. + +use crate::compiled_project::_helpers::job_catalog::catalog_where; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::graph::models::JobSelection; +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use std::collections::BTreeMap; + +/// Every job and group the project declares, whatever targets they are declared for. +pub fn declared_jobs(project_dir: &Bound<'_, PyAny>) -> PyResult> { + let inventory: OrchestrationInventory = orchestration_declarations(project_dir)?; + Ok(catalog_where(&inventory, |_| true).0) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs index cbc2c7c0c7..5fa46fa6cc 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -1,5 +1,6 @@ pub mod config_values; pub mod declarations; +pub mod declared_jobs; pub mod entry_schema_fact; pub mod family_fact; pub mod graph_with_jobs; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs index 4388c92d11..58ff9d5b96 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod python_lifecycle; pub(crate) mod python_node_graph; pub(crate) mod python_selection; +pub(crate) mod set_dependencies; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs index ccd8d7dd2e..32638fb9af 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs @@ -4,14 +4,21 @@ use crate::graph::main::python_nodes::python_node_graph::python_node_graph; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use sqlbuild_analysis::node_set_dependencies::main::set_dependency_metadata::set_dependency_metadata; +use sqlbuild_analysis::node_set_dependencies::models::{GraphNode, ResolvedNode, ResolvedSet}; +use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNode}; use sqlbuild_host::entries::models::UserCodeHost; +use std::collections::HashMap; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::main::python_files::node_identity::node_identity; use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; @@ -32,6 +39,106 @@ pub(crate) struct Vocabulary<'py> { pub(crate) kinds: Bound<'py, PyAny>, pub(crate) sql_ref: Bound<'py, PyAny>, pub(crate) providers: Bound<'py, PyDict>, + /// Each node's resolved `job(...)` and `tag(...)` dependencies; empty for the explicit graph. + pub(crate) sets: Vec, + /// Each discovered node's kind, such as `task`, by name. + pub(crate) node_kinds: HashMap, +} + +impl Vocabulary<'_> { + /// Each set `name` declares, as `(set label, end node labels)` its identity records. + fn recorded_sets(&self, name: &str) -> Vec<(String, Vec)> { + self.sets + .iter() + .filter(|node| node.name == name) + .flat_map(|node| &node.sets) + .map(|set| self.recorded_set(set)) + .collect() + } + + /// One set's label and its end node labels. + fn recorded_set(&self, set: &ResolvedSet) -> (String, Vec) { + let ends: Vec = set.ends.iter().map(|end| self.end_label(end)).collect(); + ( + format!("{} '{}'", set.dependency.kind, set.dependency.name), + ends, + ) + } + + /// `model 'orders'` or `task 'export_orders'`. + fn end_label(&self, end: &GraphNode) -> String { + match end { + GraphNode::Sql((kind, name)) => format!("{kind} '{name}'"), + GraphNode::Python(name) => format!( + "{} '{name}'", + self.node_kinds.get(name).map_or("node", String::as_str) + ), + } + } + + /// The end nodes of every set dependency `name` declares. + fn set_ends<'a>(&'a self, name: &'a str) -> impl Iterator + 'a { + self.sets + .iter() + .filter(move |node| node.name == name) + .flat_map(|node| &node.sets) + .flat_map(|set| &set.ends) + } +} + +/// A `SqlResourceRef` to each SQL end node of `name`'s sets that it does not already declare. +fn set_sql_refs<'py>( + vocabulary: &Vocabulary<'py>, + name: &str, + mut declared: Vec<(String, String)>, +) -> PyResult>> { + let py: Python<'py> = vocabulary.sql_ref.py(); + let kinds: Bound<'py, PyAny> = host_entry(py, UserCodeHost::SqlResourceRefKind)?; + let mut references: Vec> = Vec::new(); + for end in vocabulary.set_ends(name) { + let GraphNode::Sql(key) = end else { + continue; + }; + if declared.contains(key) { + continue; + } + declared.push(key.clone()); + references.push(call_with( + &vocabulary.sql_ref, + &[ + ("kind", native_field(&kinds, &key.0.to_uppercase())?), + ("name", PyString::new(py, &key.1).into_any()), + ], + )?); + } + Ok(references) +} + +/// `identity` with `recorded` sets in its metadata, so a later plan can list wait changes. +fn with_recorded_sets<'py>( + identity: Bound<'py, PyAny>, + recorded: &[(String, Vec)], +) -> PyResult> { + if recorded.is_empty() || identity.is_none() { + return Ok(identity); + } + let metadata: String = native_field(&identity, "metadata_json")?.extract()?; + let updated: String = set_dependency_metadata(&metadata, recorded); + replaced( + &identity, + &[( + "metadata_json", + PyString::new(identity.py(), &updated).into_any(), + )], + ) +} + +/// `(kind, name)` of a `SqlResourceRef`. +fn sql_ref_key(reference: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + native_field(&native_field(reference, "kind")?, "value")?.extract()?, + native_field(reference, "name")?.extract()?, + )) } fn attribute_value<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { @@ -100,14 +207,19 @@ pub(crate) fn python_node<'py>( if identity.is_none() { identity = node_identity(py, (kind, documented), &get)?; } + let name: String = get(NAME_KEY)?.extract()?; + identity = with_recorded_sets(identity, &vocabulary.recorded_sets(&name))?; let depends_on: Bound<'py, PyAny> = get("depends_on")?; let mut sql_deps: Vec> = Vec::new(); + let mut declared: Vec<(String, String)> = Vec::new(); for dependency in depends_on.try_iter()? { let dependency: Bound<'py, PyAny> = dependency?; if dependency.is_instance(&vocabulary.sql_ref)? { + declared.push(sql_ref_key(&dependency)?); sql_deps.push(dependency); } } + sql_deps.extend(set_sql_refs(vocabulary, &name, declared)?); let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("kind", kind_member), ("file_path", get("file_path")?), @@ -133,8 +245,9 @@ pub(crate) fn python_node<'py>( pub(crate) fn upstream_names( py: Python<'_>, nodes: &[Bound<'_, PyAny>], - sql_ref: &Bound<'_, PyAny>, + vocabulary: &Vocabulary<'_>, ) -> PyResult>> { + let sql_ref: &Bound<'_, PyAny> = &vocabulary.sql_ref; let by_key: Bound<'_, PyDict> = PyDict::new(py); for node in nodes { by_key.set_item(native_field(node, "function")?, native_field(node, "name")?)?; @@ -161,11 +274,80 @@ pub(crate) fn upstream_names( names.push(upstream.extract()?); } } + let name: String = native_field(node, "name")?.extract()?; + for end in vocabulary.set_ends(&name) { + if let GraphNode::Python(upstream) = end + && !names.contains(upstream) + { + names.push(upstream.clone()); + } + } Ok(names) }) .collect() } +/// Each discovered node's lowercase kind, by name. +fn node_kinds(discovered_inputs: &Bound<'_, PyAny>) -> PyResult> { + let mut kinds: HashMap = HashMap::new(); + for (collection, kind, _) in KINDS { + for discovered in native_field(discovered_inputs, collection)?.try_iter()? { + kinds.insert( + native_field(&discovered?, NAME_KEY)?.extract()?, + kind.to_lowercase(), + ); + } + } + Ok(kinds) +} + +/// The Python-node graph of only the dependencies the nodes name one by one. +pub(crate) fn explicit_python_node_graph<'py>( + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + graph_with_sets(discovered_inputs, Vec::new()) +} + +/// The executable Python-node graph with each of `sets` an edge to its end nodes. +pub(crate) fn graph_with_sets<'py>( + discovered_inputs: &Bound<'py, PyAny>, + sets: Vec, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let providers: Bound<'py, PyDict> = PyDict::new(py); + for provider in native_field(discovered_inputs, "providers")?.try_iter()? { + let provider: Bound<'py, PyAny> = provider?; + providers.set_item(native_field(&provider, "name")?, provider)?; + } + let vocabulary: Vocabulary<'py> = Vocabulary { + node: record_class(py, RecordId::DiscoveredPythonNode)?, + kinds: host_entry(py, UserCodeHost::PythonNodeKind)?, + sql_ref: host_entry(py, UserCodeHost::SqlResourceRef)?, + providers, + node_kinds: if sets.is_empty() { + HashMap::new() + } else { + node_kinds(discovered_inputs)? + }, + sets, + }; + let mut nodes: Vec> = Vec::new(); + within_identity_pass(py, || { + for (collection, kind, documented) in KINDS { + for discovered in native_field(discovered_inputs, collection)?.try_iter()? { + nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); + } + } + Ok(()) + })?; + let inventory: Vec = nodes + .iter() + .zip(upstream_names(py, &nodes, &vocabulary)?) + .map(|(node, upstream)| inventory_node(node, upstream)) + .collect::>()?; + graph_record(py, &nodes, graph_indexes(&inventory)) +} + pub(crate) fn inventory_node( node: &Bound<'_, PyAny>, upstream: Vec, diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs new file mode 100644 index 0000000000..30e2fb4665 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs @@ -0,0 +1,46 @@ +//! The discovered Python nodes declaring `job(...)` or `tag(...)` dependencies. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::node_set_dependencies::models::{DependantNode, NodeSetDependency}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Discovered function collections and the node kind each holds. +const COLLECTIONS: [(&str, &str); 4] = [ + ("loader_functions", "loader"), + ("task_functions", "task"), + ("asset_functions", "asset"), + ("check_functions", "check"), +]; + +/// Each discovered Python node declaring a `job(...)` or `tag(...)` dependency. +pub(crate) fn dependants(discovered: &Bound<'_, PyAny>) -> PyResult> { + let node_set: Bound<'_, PyAny> = host_entry(discovered.py(), UserCodeHost::NodeSetRef)?; + let mut found: Vec = Vec::new(); + for (collection, kind) in COLLECTIONS { + for function in native_field(discovered, collection)?.try_iter()? { + let function: Bound<'_, PyAny> = function?; + let mut sets: Vec = Vec::new(); + for dependency in native_field(&function, "depends_on")?.try_iter()? { + let dependency: Bound<'_, PyAny> = dependency?; + if dependency.is_instance(&node_set)? { + sets.push(NodeSetDependency { + kind: native_field(&native_field(&dependency, "kind")?, "value")? + .extract()?, + name: native_field(&dependency, "name")?.extract()?, + exclude: native_field(&dependency, "exclude")?.extract()?, + }); + } + } + if !sets.is_empty() { + found.push(DependantNode { + name: native_field(&function, "name")?.extract()?, + kind: kind.to_owned(), + sets, + }); + } + } + } + Ok(found) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs index e0c9f75bcb..26ca9f0d63 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs @@ -6,5 +6,6 @@ pub mod python_node_names; pub mod register_python_lifecycle; pub mod register_python_node_graph; pub mod register_python_selection; +pub mod resolve_node_sets; pub mod resolve_python_check_excludes; pub mod run_selection_from_inputs; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs index 6fc4c5c0ea..790c948091 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs @@ -4,12 +4,15 @@ use crate::graph::_helpers::python_nodes::python_selection::{ allows_round_trips, project_sources, selection_graph, }; use crate::graph::main::python_nodes::python_node_graph::python_node_graph; +use crate::graph::main::python_nodes::resolve_node_sets::resolve_node_sets; use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph}; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::python_selection::main::python_sql_graph_error::python_sql_graph_error; use sqlbuild_analysis::python_selection::models::{PythonSelectionGraph, SqlSelectionGraph}; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::types::NodeSetOutcome; +use sqlbuild_python_front::discovery::main::records::stored_node_sets::stored_node_sets; /// The project's first whole-graph boundary error across SQL resources and Python nodes, as every selection reports it. pub fn project_graph_error( @@ -19,6 +22,13 @@ pub fn project_graph_error( let py: Python<'_> = project_graph.py(); let project_graph: &Bound<'_, CompiledProjectGraph> = project_graph.cast()?; let project: &Bound<'_, PyAny> = project_graph.get().project(py); + let sets: Option = match stored_node_sets(discovered_inputs)? { + Some(outcome) => Some(outcome), + None => resolve_node_sets(discovered_inputs, project)?, + }; + if let Some(Err(error)) = sets { + return Ok(Some(error)); + } let sources: Vec<(String, Option)> = project_sources(project)?; let nodes: PythonSelectionGraph = selection_graph(&python_node_graph(discovered_inputs)?)?; let native: &Bound<'_, NativeProjectGraph> = project_graph.get().native(py); diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs index 53c6bd7bfa..be1152468b 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs @@ -1,51 +1,19 @@ //! The internal executable Python-node graph of discovered project inputs. -use crate::graph::_helpers::python_nodes::python_node_graph::KINDS; -use crate::graph::_helpers::python_nodes::python_node_graph::Vocabulary; -use crate::graph::_helpers::python_nodes::python_node_graph::graph_record; -use crate::graph::_helpers::python_nodes::python_node_graph::inventory_node; -use crate::graph::_helpers::python_nodes::python_node_graph::python_node; -use crate::graph::_helpers::python_nodes::python_node_graph::upstream_names; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; -use sqlbuild_analysis::python_selection::models::InventoryNode; -use sqlbuild_host::entries::models::UserCodeHost; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; +use crate::graph::_helpers::python_nodes::python_node_graph::graph_with_sets; +use crate::graph::_helpers::python_nodes::python_selection::planner_error; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::node_set_dependencies::models::ResolvedNode; +use sqlbuild_python_front::discovery::main::records::stored_node_sets::stored_node_sets; -/// The internal executable Python-node graph of discovered project inputs. +/// The executable Python-node graph, each resolved `job(...)` and `tag(...)` an edge to its end nodes. pub fn python_node_graph<'py>( discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = discovered_inputs.py(); - let providers: Bound<'py, PyDict> = PyDict::new(py); - for provider in native_field(discovered_inputs, "providers")?.try_iter()? { - let provider: Bound<'py, PyAny> = provider?; - providers.set_item(native_field(&provider, "name")?, provider)?; - } - let vocabulary: Vocabulary<'py> = Vocabulary { - node: record_class(py, RecordId::DiscoveredPythonNode)?, - kinds: host_entry(py, UserCodeHost::PythonNodeKind)?, - sql_ref: host_entry(py, UserCodeHost::SqlResourceRef)?, - providers, + let sets: Vec = match stored_node_sets(discovered_inputs)? { + Some(Ok(resolved)) => resolved, + Some(Err(error)) => return Err(planner_error(discovered_inputs.py(), error)), + None => Vec::new(), }; - let mut nodes: Vec> = Vec::new(); - within_identity_pass(py, || { - for (collection, kind, documented) in KINDS { - for discovered in native_field(discovered_inputs, collection)?.try_iter()? { - nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); - } - } - Ok(()) - })?; - let inventory: Vec = nodes - .iter() - .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?) - .map(|(node, upstream)| inventory_node(node, upstream)) - .collect::>()?; - graph_record(py, &nodes, graph_indexes(&inventory)) + graph_with_sets(discovered_inputs, sets) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs new file mode 100644 index 0000000000..a494b6d3c8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs @@ -0,0 +1,58 @@ +//! Python nodes' `job(...)` and `tag(...)` dependencies, resolved once a compile has the SQL graph. + +use std::collections::BTreeMap; + +use crate::compiled_project::main::declared_jobs::declared_jobs; +use crate::graph::_helpers::python_nodes::python_node_graph::explicit_python_node_graph; +use crate::graph::_helpers::python_nodes::python_selection::{ + allows_round_trips, project_sources, selection_graph, +}; +use crate::graph::_helpers::python_nodes::set_dependencies::dependants; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::graph::models::JobSelection; +use sqlbuild_analysis::node_set_dependencies::main::resolve_node_set_dependencies::resolve_node_set_dependencies; +use sqlbuild_analysis::node_set_dependencies::models::DependantNode; +use sqlbuild_analysis::python_selection::models::{PythonSelectionGraph, SqlSelectionGraph}; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::types::NodeSetOutcome; +use sqlbuild_python_front::discovery::main::records::keep_node_sets::keep_node_sets; + +/// Resolve and keep the set dependencies of `discovered` over `project`'s graph, errors included. +pub fn resolve_node_sets( + discovered: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult> { + let found: Vec = dependants(discovered)?; + let outcome: Option = if found.is_empty() { + None + } else { + let native: Bound<'_, NativeProjectGraph> = + native_field(project, "lineage_graph")?.cast_into()?; + let sources: Vec<(String, Option)> = project_sources(project)?; + let python: PythonSelectionGraph = + selection_graph(&explicit_python_node_graph(discovered)?)?; + let project_dir: Bound<'_, PyAny> = native_field(discovered, "project_dir")?; + let catalog: BTreeMap = if project_dir.is_none() { + BTreeMap::new() + } else { + declared_jobs(&project_dir)? + }; + let allow_warehouse_round_trips: bool = allows_round_trips(project)?; + Some(compiler_guard(|| { + Ok(resolve_node_set_dependencies( + SqlSelectionGraph { + graph: native.get().graph(), + sources: &sources, + allow_warehouse_round_trips, + }, + &python, + &found, + &catalog, + )) + })?) + }; + keep_node_sets(discovered, outcome.clone())?; + Ok(outcome) +} diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index 52e20da977..e3cf9b4061 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -37,6 +37,7 @@ use sqlbuild_analysis::graph::models::BuildResources; use sqlbuild_analysis::graph::models::EdgeDirection; use sqlbuild_analysis::graph::models::GraphIndexes; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::node_set_dependencies::models::ResolvedNode; use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; @@ -48,6 +49,7 @@ use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::discovery::main::records::stored_node_sets::stored_node_sets; use std::cell::RefCell; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -398,6 +400,8 @@ pub struct ProjectSelection { pub(crate) sources: Vec<(String, Option)>, allow_warehouse_round_trips: bool, pub nodes: PythonSelectionGraph, + /// Each Python node's resolved `job(...)` and `tag(...)` dependencies. + pub sets: Vec, } impl ProjectSelection { @@ -406,6 +410,10 @@ impl ProjectSelection { sources: project_sources(project)?, allow_warehouse_round_trips: allows_round_trips(project)?, nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, + sets: match stored_node_sets(discovered_inputs)? { + Some(Ok(sets)) => sets, + _ => Vec::new(), + }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index f556259c9f..670971770a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{IntoPyObject, pyfunction}; use serde_json::{Value, json}; +use sqlbuild_analysis::node_set_dependencies::models::GraphNode; use sqlbuild_analysis::planner_scope::main::node_loads::node_loads; use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; @@ -168,6 +169,31 @@ fn python_nodes( ))); }; published.definition["deps"] = json!(deps); + if let Some(resolved) = selected + .sets + .iter() + .find(|resolved| resolved.name == node.name) + { + published.definition["set_dependencies"] = json!( + resolved + .sets + .iter() + .map(|set| json!({ + "kind": set.dependency.kind, + "name": set.dependency.name, + "exclude": set.dependency.exclude, + "ends": set + .ends + .iter() + .filter_map(|end| match end { + GraphNode::Sql((kind, name)) => Some(format!("{kind}:{name}")), + GraphNode::Python(name) => selected.node_key(name), + }) + .collect::>(), + })) + .collect::>() + ); + } published.definition["tags"] = json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); if let Some(source) = selected.intermediate_source(&node.kind, &node.name) { diff --git a/fensu.toml b/fensu.toml index 5537713324..a1be51a1d9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -674,7 +674,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl [[targets.python.dead_code.roots]] modules = ["sqlbuild.python_nodes.models"] -symbols = ["AuditCase", "SqlResourceRef"] +symbols = ["AuditCase", "NodeSetRef", "SqlResourceRef"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] @@ -1495,6 +1495,12 @@ path = "src/sqlbuild/runtime/user_code/classes/user_code_finder.py" symbols = ["UserCodeFinder.find_spec"] reason = "importlib calls meta-path finders positionally as find_spec(fullname, path, target)." +[[targets.python.rule_exceptions]] +rule = "FFS120" +path = "src/sqlbuild/refs.py" +symbols = ["job", "tag"] +reason = "Like model(name), job and tag take the job or tag name positionally, as authors write depends_on=job(\"orders_job\")." + [[targets.python.rule_exceptions]] rule = "FFS120" path = "src/sqlbuild/assets.py" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index d5991ed34e..1d108783cb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -133,7 +133,7 @@ Every decorator accepts the same organizational metadata: | Argument | Description | |----------|-------------| -| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references) | +| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references or a `job()`/`tag()`; see [SQL References](sql-references.md#depending-on-a-job-or-a-tag)) | | `tags` | Labels for selection, filtering, and catalog grouping | | `group` | A display/catalog grouping string | | `description` | Required human-readable docs; defaults to the function docstring ([Descriptions](../models/configuration.md#descriptions)) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md index 54c656c706..44a6114391 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/sql-references/ - Declaring a reference - Resolving with ctx.relation - Hard-coded relation names +- Depending on a job or a tag - model vs source - Where references are allowed @@ -119,6 +120,61 @@ This is a guard against accidental hard-coding, not a sandbox. SQL sent directly the project's sources. To turn both checks off while migrating a project, set [`[references] enforce_explicit = false`](../project-configuration.md#references). +## Depending on a job or a tag + +Naming every upstream model ties a node to model names that change with each rename or split. +`job()` and `tag()` from `sqlbuild.refs` depend on a whole set instead: + +```python +from sqlbuild.refs import job, tag +from sqlbuild.tasks import task + +@task(depends_on=(job("orders_job", exclude="tag:experimental"), tag("finance_inputs"))) +def publish_revenue(ctx): + """Publish revenue.""" + ... +``` + +- `job(name)` depends on the job's **end nodes**: the models, seeds, sources, loaders, tasks and + assets of its selection that nothing else in the selection reads. Everything else in the job is + upstream of an end node, so this waits for the whole job without listing every edge. A job group + name depends on the union of its jobs. +- `tag(name)` depends on the end nodes of every node carrying the tag. +- `exclude=` takes the usual [selectors](../selectors.md) (names, `tag:`, `path:`, + `kind:`) and removes those nodes before the set is reduced to its end nodes. + +Compile resolves each into ordinary edges to the end nodes, so planning, `--changed`, selector +expansion (`+task:publish_revenue`), `sqb dag` and job group waits treat them like named +dependencies. A model named directly and also reached through a job is one edge. Jobs resolve +whatever targets they are declared for. + +The edges only order the node. `ctx.relation(...)` still resolves only the `model()`, `source()` +and `seed()` references the node names, and column lineage stops at these edges. + +When a job's or tag's end nodes change, the plan says so under the node, against the end nodes +it was last built with: + +```text +Python read-side (1) +└── publish_revenue task (unchanged) + └── publish_revenue now also waits on model 'refund_totals' through job 'orders_job' +``` + +`--json` lists the same lines as `wait_changes` on the Python node. + +Compile fails when: + +- the job or tag does not exist, or the set is empty after `exclude`; +- a node of the set reads the dependant's output, directly or through other nodes. The error names + the node that reads back and suggests moving it to another job or excluding it; +- a [job group](../orchestration.md#group-jobs) would wait in a loop: job groups wait job + by job, so if the dependant's job waits for the job it depends on while that job reads a node of + the dependant's job, neither can start. The error names the nodes that close the loop; move + them to another job; +- a loader declares one: loaders depend on loaders, tasks and assets named one by one; +- the name has a code location prefix such as `job("billing:nightly_job")`, which is reserved for + depending on other locations' jobs. + ## model vs source | Reference | Resolves to | @@ -131,7 +187,7 @@ set [`[references] enforce_explicit = false`](../project-configuration.md#refere ## Where references are allowed -- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`. +- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`, and depend on a `job()` or `tag()`. - **Python hooks** declare the models, sources, and seeds they read with `@hook(reads=...)`; see [Python Hooks](../models/hooks/python.md#declared-reads). - **Loaders** may not reference SQL resources; they read sources with `ctx.source(...)` and upstream loaders with `ctx.loader(...)`. - A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies. diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 6e4615534d..f215c77928 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -283,6 +283,7 @@ "AuditCase": ("sqlbuild.python_nodes.models", "AuditCase"), "RELATION_RENDERER": ("sqlbuild.python_nodes.models", "RELATION_RENDERER"), "SqlResourceRef": ("sqlbuild.python_nodes.models", "SqlResourceRef"), + "NodeSetRef": ("sqlbuild.python_nodes.models", "NodeSetRef"), "PythonCheckSeverity": ("sqlbuild.python_nodes.types", "PythonCheckSeverity"), "SqlResourceRefKind": ("sqlbuild.python_nodes.types", "SqlResourceRefKind"), "load_custom_rules": ( diff --git a/src/sqlbuild/assets.py b/src/sqlbuild/assets.py index 296ae1f5d2..c242b54be1 100644 --- a/src/sqlbuild/assets.py +++ b/src/sqlbuild/assets.py @@ -12,6 +12,7 @@ from sqlbuild.python_nodes.models import ( AssetDefinition, ColumnLineageRef, + NodeSetRef, RetryPolicy, SqlResourceRef, ) @@ -26,8 +27,9 @@ def asset( name: str | None = None, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef] = (), + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), tags: Sequence[str] = (), group: str | None = None, description: str | None = None, diff --git a/src/sqlbuild/checks.py b/src/sqlbuild/checks.py index 3f5872f6b3..8c14db63b7 100644 --- a/src/sqlbuild/checks.py +++ b/src/sqlbuild/checks.py @@ -7,7 +7,7 @@ from sqlbuild.executor.python_nodes.models import CheckContext, PythonCheckResult from sqlbuild.python_nodes.main.apply_check import apply_check from sqlbuild.python_nodes.main.read_check_definition import read_check_definition -from sqlbuild.python_nodes.models import CheckDefinition, SqlResourceRef +from sqlbuild.python_nodes.models import CheckDefinition, NodeSetRef, SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity __all__ = ("CheckContext", "PythonCheckResult", "check", "get_check_definition") @@ -17,8 +17,9 @@ def check( *, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef], + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef], name: str | None = None, severity: str | PythonCheckSeverity = PythonCheckSeverity.ERROR, tags: Sequence[str] = (), diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py index 9ea32d11dd..47a53cd8b3 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py @@ -17,7 +17,7 @@ from sqlbuild.python_nodes.main.read_check_definition import read_check_definition from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition from sqlbuild.python_nodes.main.read_task_definition import read_task_definition -from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.models import NodeSetRef, SqlResourceRef type _PythonNode = ( DiscoveredLoaderFunction @@ -69,6 +69,7 @@ def _node(*, kind: str, node: _PythonNode, owners: dict[object, int]) -> dict[st if isinstance(dependency, SqlResourceRef) else (None, owners.get(dependency), _decorated_name(dependency)) for dependency in node.depends_on + if not isinstance(dependency, NodeSetRef) ], } diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 74426fec85..4501569516 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -41,6 +41,7 @@ ) from sqlbuild.python_nodes.main.calculate_retry_delay import calculate_retry_delay from sqlbuild.python_nodes.models import ( + NodeSetRef, RetryPolicy, SqlResourceRef, ) @@ -367,7 +368,9 @@ def _missing_upstream_result( """Why the node cannot run: a Python upstream outside this run that has never succeeded.""" python_dependencies: tuple[Callable[..., object], ...] = tuple( - dependency for dependency in node.depends_on if not isinstance(dependency, SqlResourceRef) + dependency + for dependency in node.depends_on + if not isinstance(dependency, (SqlResourceRef, NodeSetRef)) ) if len(upstream_results) == len(python_dependencies): return None diff --git a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py index ec6adc9837..2ada3b7eff 100644 --- a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py +++ b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py @@ -2,16 +2,17 @@ from collections.abc import Callable -from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.models import NodeSetRef, SqlResourceRef def normalize_python_node_dependencies( value: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef], -) -> tuple[Callable[..., object] | SqlResourceRef, ...]: - if callable(value) or isinstance(value, SqlResourceRef): + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef], +) -> tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...]: + if callable(value) or isinstance(value, (SqlResourceRef, NodeSetRef)): return (value,) return tuple(value) diff --git a/src/sqlbuild/python_nodes/main/apply_asset.py b/src/sqlbuild/python_nodes/main/apply_asset.py index 708ea4220c..308482d44e 100644 --- a/src/sqlbuild/python_nodes/main/apply_asset.py +++ b/src/sqlbuild/python_nodes/main/apply_asset.py @@ -16,6 +16,7 @@ from sqlbuild.python_nodes.models import ( AssetDefinition, ColumnLineageRef, + NodeSetRef, RetryPolicy, SqlResourceRef, ) @@ -28,8 +29,9 @@ def apply_asset( name: str | None = None, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef] = (), + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), tags: Sequence[str] = (), group: str | None = None, description: str | None = None, @@ -40,7 +42,7 @@ def apply_asset( ) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: """Apply SQLBuild asset metadata to a Python function.""" - normalized_deps: tuple[Callable[..., object] | SqlResourceRef, ...] = ( + normalized_deps: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = ( normalize_python_node_dependencies(depends_on) ) diff --git a/src/sqlbuild/python_nodes/main/apply_check.py b/src/sqlbuild/python_nodes/main/apply_check.py index 7c2f8efc0e..643528cdca 100644 --- a/src/sqlbuild/python_nodes/main/apply_check.py +++ b/src/sqlbuild/python_nodes/main/apply_check.py @@ -10,7 +10,7 @@ normalize_python_node_dependencies, ) from sqlbuild.python_nodes._helpers.description_resolution import resolve_description -from sqlbuild.python_nodes.models import CheckDefinition, SqlResourceRef +from sqlbuild.python_nodes.models import CheckDefinition, NodeSetRef, SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity @@ -18,8 +18,9 @@ def apply_check( *, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef], + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef], name: str | None = None, severity: str | PythonCheckSeverity = PythonCheckSeverity.ERROR, tags: Sequence[str] = (), @@ -29,7 +30,7 @@ def apply_check( ) -> Callable[[Callable[..., object]], Callable[..., object]]: """Apply SQLBuild check metadata to a Python function.""" - normalized_deps: tuple[Callable[..., object] | SqlResourceRef, ...] = ( + normalized_deps: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = ( normalize_python_node_dependencies(depends_on) ) diff --git a/src/sqlbuild/python_nodes/main/apply_task.py b/src/sqlbuild/python_nodes/main/apply_task.py index 0abe98f455..83363f2b44 100644 --- a/src/sqlbuild/python_nodes/main/apply_task.py +++ b/src/sqlbuild/python_nodes/main/apply_task.py @@ -10,7 +10,7 @@ normalize_python_node_dependencies, ) from sqlbuild.python_nodes._helpers.description_resolution import resolve_description -from sqlbuild.python_nodes.models import RetryPolicy, SqlResourceRef, TaskDefinition +from sqlbuild.python_nodes.models import NodeSetRef, RetryPolicy, SqlResourceRef, TaskDefinition def apply_task( @@ -19,8 +19,9 @@ def apply_task( name: str | None = None, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef] = (), + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), tags: Sequence[str] = (), group: str | None = None, description: str | None = None, @@ -29,7 +30,7 @@ def apply_task( ) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: """Apply SQLBuild task metadata to a Python function.""" - normalized_deps: tuple[Callable[..., object] | SqlResourceRef, ...] = ( + normalized_deps: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = ( normalize_python_node_dependencies(depends_on) ) diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py index 6ecf319de8..7d47dde0bd 100644 --- a/src/sqlbuild/python_nodes/models.py +++ b/src/sqlbuild/python_nodes/models.py @@ -14,7 +14,7 @@ ) from sqlbuild.compiler.auditing.types import AuditSeverity from sqlbuild.python_nodes.constants import RELATION_RENDERER -from sqlbuild.python_nodes.types import PythonCheckSeverity, SqlResourceRefKind +from sqlbuild.python_nodes.types import NodeSetKind, PythonCheckSeverity, SqlResourceRefKind @dataclass(frozen=True) @@ -33,6 +33,21 @@ def __str__(self) -> str: return renderer(self) +@dataclass(frozen=True) +class NodeSetRef: + """A dependency on a job's or tag's end nodes, resolved into node edges at compile.""" + + kind: NodeSetKind + name: str + exclude: str | None = None + + def __str__(self) -> str: + """Render as the authoring call.""" + + exclude: str = "" if self.exclude is None else f', exclude="{self.exclude}"' + return f'{self.kind.value}("{self.name}"{exclude})' + + @dataclass(frozen=True) class ColumnLineageRef: """Graph-based upstream column reference for declared Python-node lineage.""" @@ -126,7 +141,7 @@ class TaskDefinition: """Metadata attached to a decorated SQLBuild task function.""" name: str - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = () + depends_on: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = () tags: tuple[str, ...] = () group: str | None = None description: str | None = None @@ -139,7 +154,7 @@ class AssetDefinition: """Metadata attached to a decorated SQLBuild asset function.""" name: str - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = () + depends_on: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = () tags: tuple[str, ...] = () group: str | None = None description: str | None = None @@ -154,7 +169,7 @@ class CheckDefinition: """Metadata attached to a decorated SQLBuild check function.""" name: str - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + depends_on: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] severity: PythonCheckSeverity = PythonCheckSeverity.ERROR tags: tuple[str, ...] = () group: str | None = None diff --git a/src/sqlbuild/python_nodes/types.py b/src/sqlbuild/python_nodes/types.py index ba01ad5969..5300b68fb0 100644 --- a/src/sqlbuild/python_nodes/types.py +++ b/src/sqlbuild/python_nodes/types.py @@ -38,6 +38,13 @@ class PythonCheckSeverity(StrEnum): WARN = "warn" +class NodeSetKind(StrEnum): + """What a set dependency names: a job's selection or every node carrying a tag.""" + + JOB = "job" + TAG = "tag" + + class SqlResourceRefKind(StrEnum): """Typed reference target for SQL graph resources.""" diff --git a/src/sqlbuild/refs.py b/src/sqlbuild/refs.py index ff596e8d74..94050fb166 100644 --- a/src/sqlbuild/refs.py +++ b/src/sqlbuild/refs.py @@ -1,9 +1,21 @@ """Public typed references for SQL graph resources.""" +from sqlbuild.python_nodes.models import NodeSetRef as NodeSetRef from sqlbuild.python_nodes.models import SqlResourceRef as SqlResourceRef +from sqlbuild.python_nodes.types import NodeSetKind as NodeSetKind from sqlbuild.python_nodes.types import SqlResourceRefKind as SqlResourceRefKind -__all__ = ("SqlResourceRef", "SqlResourceRefKind", "model", "seed", "source") +__all__ = ( + "NodeSetKind", + "NodeSetRef", + "SqlResourceRef", + "SqlResourceRefKind", + "job", + "model", + "seed", + "source", + "tag", +) def model(name: str) -> SqlResourceRef: @@ -22,3 +34,15 @@ def seed(name: str) -> SqlResourceRef: """Return a typed reference to a SQLBuild seed.""" return SqlResourceRef(kind=SqlResourceRefKind.SEED, name=name) + + +def job(name: str, *, exclude: str | None = None) -> NodeSetRef: + """Depend on a job's end nodes: its selection less `exclude` that nothing else in it reads.""" + + return NodeSetRef(kind=NodeSetKind.JOB, name=name, exclude=exclude) + + +def tag(name: str, *, exclude: str | None = None) -> NodeSetRef: + """Depend on the end nodes of every node carrying the tag, less `exclude`.""" + + return NodeSetRef(kind=NodeSetKind.TAG, name=name, exclude=exclude) diff --git a/src/sqlbuild/tasks.py b/src/sqlbuild/tasks.py index 007b573892..5ce20863f4 100644 --- a/src/sqlbuild/tasks.py +++ b/src/sqlbuild/tasks.py @@ -8,7 +8,7 @@ from sqlbuild.executor.python_nodes.models import TaskContext from sqlbuild.python_nodes.main.apply_task import apply_task from sqlbuild.python_nodes.main.read_task_definition import read_task_definition -from sqlbuild.python_nodes.models import RetryPolicy, SqlResourceRef, TaskDefinition +from sqlbuild.python_nodes.models import NodeSetRef, RetryPolicy, SqlResourceRef, TaskDefinition __all__ = ("SkipMode", "TaskContext", "get_task_definition", "task") @@ -19,8 +19,9 @@ def task( name: str | None = None, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef] = (), + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), tags: Sequence[str] = (), group: str | None = None, description: str | None = None, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index be3b05377d..2abc0dd5ca 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -421,3 +421,32 @@ class AdapterRecoveryCase: ("B002", "downstream"), ("B212", "upstream"), ) + + +@dataclass(frozen=True) +class SetDependencyPlanTestCase: + """A task's `job(...)` or `tag(...)` dependency and the models planning its upstream selects.""" + + description: str + depends_on: str + expected_models: tuple[str, ...] + expected_sql_deps: tuple[str, ...] + + +@dataclass(frozen=True) +class SetDependencyErrorTestCase: + """A `job(...)` or `tag(...)` dependency compile refuses, and what its error says.""" + + description: str + depends_on: str + job_files: tuple[tuple[str, str], ...] + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class WaitChangeTestCase: + """A job selection edited after a build, and the wait changes the next plan lists.""" + + description: str + edited_select: str + expected_changes: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 4dde272237..e51e7ecab9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -244,3 +244,46 @@ def postgres_semantic_shapes( resolved_connection=_OFFLINE_POSTGRES_CONNECTION, ) return semantic_shapes(project=project) + + +def set_dependency_project( + *, depends_on: str, job_files: tuple[tuple[str, str], ...] = () +) -> dict[str, str]: + """Four models, a `nightly_job` over three of them, and a `report` task on `depends_on`.""" + + model: str = "MODEL (description 'Test model {name}.', materialized table{tags});\n{sql}\n" + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "set_dependencies"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = ":memory:"\n\n[targets.dev]\nschema = "dev"\n' + ), + "models/stg_orders.sql": model.format(name="stg_orders", tags="", sql="SELECT 1 AS id"), + "models/orders.sql": model.format( + name="orders", tags=", tags [finance]", sql='SELECT id FROM __ref("stg_orders")' + ), + "models/order_items.sql": model.format( + name="order_items", tags=", tags [finance]", sql='SELECT id FROM __ref("stg_orders")' + ), + "models/customers.sql": model.format(name="customers", tags="", sql="SELECT 2 AS id"), + "orchestration/jobs/nightly_job.toml": ( + 'name = "nightly_job"\ndescription = "Nightly."\n' + 'select = "stg_orders orders order_items"\n' + ), + "python/tasks/report.py": ( + "from sqlbuild.refs import job, model, tag\nfrom sqlbuild.tasks import task\n\n\n" + f"@task(depends_on={depends_on})\n" + 'def report(ctx):\n """Test task report."""\n return None\n' + ), + } + files.update(dict(job_files)) + return files + + +def python_sql_deps(discovered_inputs: DiscoveredProjectInputs, name: str) -> tuple[str, ...]: + """`kind:name` of each SQL resource a Python node's graph edges read, in order.""" + + graph: Any = native_module.build_python_node_graph(discovered_inputs=discovered_inputs) + return tuple( + f"{reference.kind.value}:{reference.name}" + for reference in graph.nodes_by_name[name].sql_deps + ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py new file mode 100644 index 0000000000..ef20d12692 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py @@ -0,0 +1,214 @@ +"""Python nodes depending on a job or a tag get edges to its end nodes through the real pipeline.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, + PlannerInputError, + discover_project_inputs, + run_cli, + run_compile_pipeline, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.runtime.contracts.models import ConnectionHooks +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + SetDependencyErrorTestCase, + SetDependencyPlanTestCase, + WaitChangeTestCase, +) +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( + python_sql_deps, + set_dependency_project, +) + +_GROUP_FILES: tuple[tuple[str, str], ...] = ( + ( + "models/customer_orders.sql", + "MODEL (description 'Test model customer_orders.', materialized table);\n" + 'SELECT o.id FROM __ref("orders") AS o JOIN __ref("customers") AS c ON o.id = c.id\n', + ), + ( + "orchestration/jobs/wide_job.toml", + 'name = "wide_job"\ndescription = "Wide."\nselect = "orders customer_orders"\n', + ), + ( + "orchestration/jobs/reporting_job.toml", + 'name = "reporting_job"\ndescription = "Reporting."\nselect = "task:report customers"\n', + ), + ( + "orchestration/job_groups/daily_job_group.toml", + 'name = "daily_job_group"\ndescription = "Daily."\njobs = ["wide_job", "reporting_job"]\n', + ), +) + + +@pytest.mark.parametrize( + "test_case", + [ + SetDependencyPlanTestCase( + description="a job dependency reads the job's end nodes", + depends_on='job("nightly_job")', + expected_models=("order_items", "orders", "stg_orders"), + expected_sql_deps=("model:order_items", "model:orders"), + ), + SetDependencyPlanTestCase( + description="exclude removes nodes before reducing to end nodes", + depends_on='job("nightly_job", exclude="order_items")', + expected_models=("orders", "stg_orders"), + expected_sql_deps=("model:orders",), + ), + SetDependencyPlanTestCase( + description="a tag dependency reads its tagged end nodes", + depends_on='tag("finance", exclude="orders")', + expected_models=("order_items", "stg_orders"), + expected_sql_deps=("model:order_items",), + ), + SetDependencyPlanTestCase( + description="a model named and reached through a job is read once", + depends_on='(model("orders"), job("nightly_job"))', + expected_models=("order_items", "orders", "stg_orders"), + expected_sql_deps=("model:orders", "model:order_items"), + ), + ], + ids=lambda case: case.description, +) +def test_given_set_dependency_when_planning_the_tasks_upstream_then_selects_the_set( + test_case: SetDependencyPlanTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files(tmp_path, set_dependency_project(depends_on=test_case.depends_on)) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + + result: CompilePipelineResult = run_compile_pipeline( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + options=CompilePipelineOptions( + no_sql_validation=True, select=("+task:report",), resolve_python_run_selectors=True + ), + hooks=ConnectionHooks(), + ) + + assert ( + tuple(sorted(entry.name for entry in result.plan_output.model_entries)) + == test_case.expected_models + ) + assert python_sql_deps(discovered, "report") == test_case.expected_sql_deps + + +@pytest.mark.parametrize( + "test_case", + [ + SetDependencyErrorTestCase( + description="an unknown job", + depends_on='job("weekly_job")', + job_files=(), + expected_fragments=( + "task 'report' depends on job 'weekly_job', which the project does not declare", + ), + ), + SetDependencyErrorTestCase( + description="an unknown tag", + depends_on='tag("marketing")', + job_files=(), + expected_fragments=("task 'report' depends on tag 'marketing', which no node carries",), + ), + SetDependencyErrorTestCase( + description="a job its exclude empties", + depends_on='job("nightly_job", exclude="stg_orders orders order_items")', + job_files=(), + expected_fragments=("which leaves no model, seed, source, loader, task or asset",), + ), + SetDependencyErrorTestCase( + description="job group members that would wait for each other", + depends_on='job("wide_job")', + job_files=_GROUP_FILES, + expected_fragments=( + "a job dependency makes job group 'daily_job_group' wait in a loop", + "'wide_job''s model 'customer_orders' reads model 'customers'", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unresolvable_set_dependency_when_planning_then_refuses_naming_the_cause( + test_case: SetDependencyErrorTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files( + tmp_path, + set_dependency_project(depends_on=test_case.depends_on, job_files=test_case.job_files), + ) + + with pytest.raises(PlannerInputError) as raised: + _ = run_compile_pipeline( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + options=CompilePipelineOptions( + no_sql_validation=True, resolve_python_run_selectors=True + ), + hooks=ConnectionHooks(), + ) + + text: str = f"{raised.value} {getattr(raised.value, 'help', '')}" + assert all(fragment in text for fragment in test_case.expected_fragments), text + + +@pytest.mark.parametrize( + "test_case", + [ + WaitChangeTestCase( + description="a job gaining an end node", + edited_select="stg_orders orders order_items customers", + expected_changes=( + "report now also waits on model 'customers' through job 'nightly_job'", + ), + ), + WaitChangeTestCase( + description="a job losing an end node", + edited_select="stg_orders orders", + expected_changes=( + "report no longer waits on model 'order_items' through job 'nightly_job'", + ), + ), + WaitChangeTestCase( + description="an unchanged job", + edited_select="stg_orders orders order_items", + expected_changes=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_job_dependency_when_the_job_changes_then_the_plan_lists_wait_changes( + test_case: WaitChangeTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], + capsys: pytest.CaptureFixture[str], +) -> None: + files: dict[str, str] = set_dependency_project(depends_on='job("nightly_job")') + files["sqlbuild_project.toml"] = files["sqlbuild_project.toml"].replace( + ":memory:", str(tmp_path / "warehouse.duckdb") + ) + write_repo_files(tmp_path, files) + assert run_cli(["--project-dir", str(tmp_path), "--no-color", "build"]) == 0 + job: Path = tmp_path / "orchestration/jobs/nightly_job.toml" + _ = job.write_text( + job.read_text().replace("stg_orders orders order_items", test_case.edited_select) + ) + _ = capsys.readouterr() + + assert run_cli(["--project-dir", str(tmp_path), "--no-color", "plan", "--json"]) == 0 + + document: dict[str, Any] = json.loads(capsys.readouterr().out) + nodes: dict[str, dict[str, Any]] = {node["name"]: node for node in document["python_nodes"]} + assert tuple(nodes["report"].get("wait_changes", ())) == test_case.expected_changes diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py index 7eb3747406..2099112251 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py @@ -62,3 +62,13 @@ class HookReadsTestCase: reads: object expected_reads: tuple[str, ...] = () expected_error: str = "" + + +@dataclass(frozen=True) +class NodeSetDependencyTestCase: + """A task's dependency on a job or tag, as stored and as it renders.""" + + description: str + dependency: object + expected_dependency: object + expected_text: str diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py index 2b40ee7ef0..725c615b33 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py @@ -16,11 +16,12 @@ CheckDefinition, ColumnLineageRef, FactoryDefinition, + NodeSetRef, RetryPolicy, TaskDefinition, ) -from sqlbuild.python_nodes.types import PythonCheckSeverity -from sqlbuild.refs import model, seed +from sqlbuild.python_nodes.types import NodeSetKind, PythonCheckSeverity +from sqlbuild.refs import job, model, seed, tag from sqlbuild.tasks import get_task_definition, task from tests.unit.src.sqlbuild.python_nodes.decorators._test_types import ( AssetDecoratorMetadataTestCase, @@ -28,6 +29,7 @@ FactoryDecoratorErrorTestCase, FactoryDecoratorMetadataTestCase, HookReadsTestCase, + NodeSetDependencyTestCase, TaskDecoratorMetadataTestCase, ) from tests.unit.src.sqlbuild.python_nodes.decorators.helpers import upstream_asset, upstream_task @@ -308,3 +310,37 @@ def refresh_orders() -> None: with pytest.raises(SharedInputError, match=test_case.expected_error): cast(Any, hook(reads=cast(Any, test_case.reads)))(refresh_orders) + + +@pytest.mark.parametrize( + "test_case", + [ + NodeSetDependencyTestCase( + description="a job dependency", + dependency=job("nightly_job"), + expected_dependency=NodeSetRef(kind=NodeSetKind.JOB, name="nightly_job"), + expected_text='job("nightly_job")', + ), + NodeSetDependencyTestCase( + description="a tag dependency with an exclude", + dependency=tag("finance", exclude="tag:draft orders"), + expected_dependency=NodeSetRef( + kind=NodeSetKind.TAG, name="finance", exclude="tag:draft orders" + ), + expected_text='tag("finance", exclude="tag:draft orders")', + ), + ], + ids=lambda case: case.description, +) +def test_given_job_or_tag_dependency_when_decorating_a_task_then_keeps_it_as_declared( + test_case: NodeSetDependencyTestCase, +) -> None: + @task(depends_on=cast(Any, test_case.dependency)) + def report() -> None: + """Test task report.""" + + definition: TaskDefinition | None = get_task_definition(report) + + assert definition is not None + assert definition.depends_on == (test_case.expected_dependency,) + assert str(definition.depends_on[0]) == test_case.expected_text diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 190d19d562..48b1be1078 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -117,7 +117,7 @@ Every decorator accepts the same organizational metadata: | Argument | Description | |----------|-------------| -| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references) | +| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references or a `job()`/`tag()`; see [SQL References](/docs/concepts/python-nodes/sql-references#depending-on-a-job-or-a-tag)) | | `tags` | Labels for selection, filtering, and catalog grouping | | `group` | A display/catalog grouping string | | `description` | Required human-readable docs; defaults to the function docstring ([Descriptions](/docs/concepts/models/configuration#descriptions)) | diff --git a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx index e0101e9095..b485d4eb8e 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx @@ -115,6 +115,62 @@ This is a guard against accidental hard-coding, not a sandbox. SQL sent directly the project's sources. To turn both checks off while migrating a project, set [`[references] enforce_explicit = false`](/docs/concepts/project-configuration#references). +## Depending on a job or a tag + +Naming every upstream model ties a node to model names that change with each rename or split. +`job()` and `tag()` from `sqlbuild.refs` depend on a whole set instead: + +```python +from sqlbuild.refs import job, tag +from sqlbuild.tasks import task + + +@task(depends_on=(job("orders_job", exclude="tag:experimental"), tag("finance_inputs"))) +def publish_revenue(ctx): + """Publish revenue.""" + ... +``` + +- `job(name)` depends on the job's **end nodes**: the models, seeds, sources, loaders, tasks and + assets of its selection that nothing else in the selection reads. Everything else in the job is + upstream of an end node, so this waits for the whole job without listing every edge. A job group + name depends on the union of its jobs. +- `tag(name)` depends on the end nodes of every node carrying the tag. +- `exclude=` takes the usual [selectors](/docs/concepts/selectors) (names, `tag:`, `path:`, + `kind:`) and removes those nodes before the set is reduced to its end nodes. + +Compile resolves each into ordinary edges to the end nodes, so planning, `--changed`, selector +expansion (`+task:publish_revenue`), `sqb dag` and job group waits treat them like named +dependencies. A model named directly and also reached through a job is one edge. Jobs resolve +whatever targets they are declared for. + +The edges only order the node. `ctx.relation(...)` still resolves only the `model()`, `source()` +and `seed()` references the node names, and column lineage stops at these edges. + +When a job's or tag's end nodes change, the plan says so under the node, against the end nodes +it was last built with: + +```text +Python read-side (1) +└── publish_revenue task (unchanged) + └── publish_revenue now also waits on model 'refund_totals' through job 'orders_job' +``` + +`--json` lists the same lines as `wait_changes` on the Python node. + +Compile fails when: + +- the job or tag does not exist, or the set is empty after `exclude`; +- a node of the set reads the dependant's output, directly or through other nodes. The error names + the node that reads back and suggests moving it to another job or excluding it; +- a [job group](/docs/concepts/orchestration#group-jobs) would wait in a loop: job groups wait job + by job, so if the dependant's job waits for the job it depends on while that job reads a node of + the dependant's job, neither can start. The error names the nodes that close the loop; move + them to another job; +- a loader declares one: loaders depend on loaders, tasks and assets named one by one; +- the name has a code location prefix such as `job("billing:nightly_job")`, which is reserved for + depending on other locations' jobs. + ## model vs source | Reference | Resolves to | @@ -127,7 +183,7 @@ set [`[references] enforce_explicit = false`](/docs/concepts/project-configurati ## Where references are allowed -- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`. +- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`, and depend on a `job()` or `tag()`. - **Python hooks** declare the models, sources, and seeds they read with `@hook(reads=...)`; see [Python Hooks](/docs/concepts/models/hooks/python#declared-reads). - **Loaders** may not reference SQL resources; they read sources with `ctx.source(...)` and upstream loaders with `ctx.loader(...)`. - A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies. From fe5add4272111e27e75527ea60dd320b2ddfb32e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 08:11:23 +0000 Subject: [PATCH 13/17] perf(compile): verify a model's typed SQL macro markers once per process A plan of an unchanged project still compiled in 6-7 s with every model's analysis answered by the compile cache: the cache covers analysis, not the per-model work around it. The largest piece was building each model's analysis text, which wraps typed SQL macro calls in markers and parses the whole model to check each marker lands where it parses, up to once per marked call. Every compile did this twice per model, once for the analysis request and once for the compiled model. Keep the verified marker set per (dialect, SQL, calls) for the process, a bounded memo the second read and later compiles in long-lived processes reuse. On a 2,280-model project, compiled-model assembly drops from 0.49 s to 0.05 s per compile. --- .../sql_macros/_helpers/typing/typed_calls.rs | 39 +++++++++++++++++-- .../src/sql_macros/constants.rs | 2 + .../src/sql_macros/models.rs | 2 +- .../src/sql_macros/tests/test_typed_calls.rs | 6 +++ fensu.toml | 6 +++ 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs index 34ddbb32df..01f77b06dc 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs @@ -1,6 +1,8 @@ //! Expression macro calls followed through the edits that move them, and their typed markers. use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex, MutexGuard, PoisonError}; use sqlquack_syntax::expressions::tree::constants::{ TYPED_EXPRESSION_FUNCTION, TYPED_LIKE_EXPRESSION_FUNCTION, @@ -12,9 +14,16 @@ use crate::analysis_route::models::AnalysisRoute; use crate::analysis_route::types::NativeDialect; use crate::compiled_project::models::SplicedSpan; use crate::sql_macros::_helpers::parsing::body::with_line_comment_closed; -use crate::sql_macros::constants::NULL_KEYWORD; +use crate::sql_macros::constants::{MARKER_MEMO_ENTRIES, NULL_KEYWORD}; use crate::sql_macros::models::{TextSplice, TypedCall}; +/// The dialect, SQL and calls a parse-verified marker set was chosen for. +type MarkerKey = (String, String, Vec); + +/// Parse-verified marker sets, as a compile reads each analysis text twice and processes recompile. +static MARKER_MEMO: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + /// `calls` moved `offset` bytes later. pub(crate) fn shifted(calls: Vec, offset: usize) -> impl Iterator { calls.into_iter().map(move |call| TypedCall { @@ -176,8 +185,32 @@ pub(crate) fn markable_calls<'c>( let AnalysisRoute::Native(parse_dialect) = analysis_route(Some(dialect)) else { return Cow::Owned(Vec::new()); }; + let key: MarkerKey = (dialect.to_owned(), sql.to_owned(), calls.to_vec()); + if let Some(kept) = marker_memo().get(&key) { + return Cow::Owned(kept.clone()); + } + let kept: Vec = parsed_markable_calls(sql, calls, (parse_dialect, marked)); + let mut memo = marker_memo(); + if memo.len() >= MARKER_MEMO_ENTRIES { + memo.clear(); + } + memo.insert(key, kept.clone()); + Cow::Owned(kept) +} + +/// The process's parse-verified marker sets, locked. +fn marker_memo() -> MutexGuard<'static, HashMap>> { + MARKER_MEMO.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// The calls whose markers parse as typed expressions, checked by parsing the marked SQL. +fn parsed_markable_calls( + sql: &str, + calls: &[TypedCall], + (parse_dialect, marked): (&'static dyn NativeDialect, usize), +) -> Vec { if lowered_markers(&typed_sql(sql, calls).0, parse_dialect) == Some(marked) { - return Cow::Borrowed(calls); + return calls.to_vec(); } let mut kept: Vec = Vec::with_capacity(calls.len()); let mut kept_marked: usize = 0; @@ -193,7 +226,7 @@ pub(crate) fn markable_calls<'c>( kept.pop(); } } - Cow::Owned(kept) + kept } /// Whether `typed_sql` wraps `call` in a marker. diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index 5d51555fc9..29e6b8dd67 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -145,3 +145,5 @@ pub(crate) const BIGQUERY_BIGNUMERIC_LIMITS: (u32, u32) = (38, 38); pub(crate) const BIGQUERY_BIG_DECIMAL_NAMES: [&str; 2] = ["BIGNUMERIC", "BIGDECIMAL"]; /// How a relation function such as `__ref("orders")` starts. pub(crate) const RELATION_FUNCTION_PREFIX: &str = "__"; +/// Parse-verified marker sets a process keeps before starting over; each holds one model's SQL. +pub(crate) const MARKER_MEMO_ENTRIES: usize = 8192; diff --git a/crates/sqlbuild-analysis/src/sql_macros/models.rs b/crates/sqlbuild-analysis/src/sql_macros/models.rs index 5657995e5c..5eaafa0a41 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/models.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/models.rs @@ -303,7 +303,7 @@ pub struct SlotCapture { } /// One expression macro call's expansion in a text: its byte range and the macro's `returns`. -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct TypedCall { pub start: usize, pub end: usize, diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs index 5f86bfd1f5..7ac928966d 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -196,6 +196,12 @@ fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns .collect(); let (typed, insertions) = typed_analysis_sql(test_case.sql, &calls, "duckdb"); assert_eq!(typed, test_case.expected_sql, "{}", test_case.description); + let (remembered, _) = typed_analysis_sql(test_case.sql, &calls, "duckdb"); + assert_eq!( + remembered, test_case.expected_sql, + "{}", + test_case.description + ); let removed: String = { let mut text: String = String::new(); let mut cursor: usize = 0; diff --git a/fensu.toml b/fensu.toml index a1be51a1d9..383c573c4d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1971,6 +1971,12 @@ path = "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection. symbols = ["connection_config"] reason = "A once-per-process warning is process state by definition: a command resolves its connection at startup for provider setup and again for its own work, and threading a flag through every caller would not guarantee once. Approved 17 Nov for this module only." +[[targets.rust.rule_exceptions]] +rule = "FPRSS130" +path = "crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs" +symbols = ["marker_memo"] +reason = "Verifying typed markers parses the model once per marked call; the result depends only on the SQL, the calls and the dialect, so a bounded process-wide memo serves a compile's repeated reads and recompiles in long-lived processes." + [[targets.rust.rule_exceptions]] rule = "FPRSS130" path = "crates/sqlbuild-python-base/src/native_threads/_helpers/exit_signal.rs" From 02e5b53fff2ff91dcd5d34b13cf05eb981d3cda9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 08:11:46 +0000 Subject: [PATCH 14/17] perf(plan): read target columns for the selection and its upstream only A plan read the warehouse columns of every built model in the project, whatever it selected: on a 2,400-model Snowflake project that is one SHOW COLUMNS IN SCHEMA of 131K rows, measured at 3-29 s, even for a one-model plan. Read columns only for the relations of the selection and what it reads, the models whose changes the plan and its stale warnings detect; an unnarrowed plan still reads them all. Snowflake reads up to 200 relations with concurrent SHOW COLUMNS IN TABLE (about 8 ms a relation at the inspection concurrency, 200 in 1.7 s) instead of from 33, so a small selection no longer lists the schema; the sequential direct path keeps 32. INFORMATION_SCHEMA.COLUMNS with an IN list is not an option there: 7.8 s for 30 tables. Parsing the listing's type JSON is memoised by text: a 131K-row schema holds 135 distinct texts, 0.28 s down to 0.02 s. --- .../_helpers/inspection/snapshot.rs | 14 ++++-- .../snowflake/_helpers/metadata_types.py | 11 +++- .../snowflake/classes/snowflake_adapter.py | 3 +- .../_helpers/test_warehouse_snapshot.py | 24 +++++++++ .../compiler/planner/main/_test_types.py | 1 + .../sqlbuild/compiler/planner/main/helpers.py | 3 +- .../main/test_snowflake_inspection_budget.py | 20 ++++++-- .../snowflake/inspection/_test_types.py | 9 ++++ .../adapters/snowflake/inspection/helpers.py | 24 ++++++++- .../inspection/test_inspection_catalog.py | 50 +++++++++++++++++-- 10 files changed, 144 insertions(+), 15 deletions(-) diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index 56000c8401..a580b7064f 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -386,17 +386,21 @@ fn gather_relations<'py>( Ok((result, states)) } +/// Listed columns of `relevant` nodes, the selection and the upstream its warnings check; else all. fn gather_columns<'py>( request: &SnapshotRequest<'py>, relations: &Bound<'py, PyDict>, + relevant: &Bound<'py, PyAny>, ) -> PyResult> { let py = request.project.py(); let mut node_names: HashSet = HashSet::new(); for node in destination_nodes(&request.project)? { - node_names.insert( - sqlbuild_python_project::resource_views::main::resource_reads::name::name(&node)? - .extract()?, - ); + if selected_node(relevant, &node)? { + node_names.insert( + sqlbuild_python_project::resource_views::main::resource_reads::name::name(&node)? + .extract()?, + ); + } } let mut needed: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); for (logical, relation) in relations.iter() { @@ -693,7 +697,7 @@ pub(crate) fn warehouse_snapshot<'py>( optional_names(py, query_schemas)?, optional_names(py, metadata_names.as_ref())?, )?; - let columns = gather_columns(request, &relations)?; + let columns = gather_columns(request, &relations, &relevant)?; let fingerprints = gather_fingerprints( request, &database, diff --git a/src/sqlbuild/adapters/snowflake/_helpers/metadata_types.py b/src/sqlbuild/adapters/snowflake/_helpers/metadata_types.py index 2d1846bc3c..bec56a4c72 100644 --- a/src/sqlbuild/adapters/snowflake/_helpers/metadata_types.py +++ b/src/sqlbuild/adapters/snowflake/_helpers/metadata_types.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from functools import lru_cache from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.snowflake.constants import NUMBER_TYPE_NAME, TEXT_TYPE_NAMES @@ -11,6 +12,7 @@ "FIXED": NUMBER_TYPE_NAME, "REAL": "FLOAT", } +_SHOW_TYPE_CACHE_SIZE: int = 4096 _INVALID_SHOW_TYPE_MESSAGE: str = "Snowflake SHOW COLUMNS returned invalid type metadata" @@ -38,8 +40,15 @@ def information_schema_type( def show_columns_type(raw_data_type: object) -> str: """Render the same type string as INFORMATION_SCHEMA for one SHOW COLUMNS data_type JSON.""" + return _show_type(str(raw_data_type)) + + +@lru_cache(maxsize=_SHOW_TYPE_CACHE_SIZE) +def _show_type(raw_data_type: str) -> str: + """`show_columns_type` of one JSON text; a schema repeats a few hundred texts in 100K+ rows.""" + try: - decoded: object = json.loads(str(raw_data_type)) + decoded: object = json.loads(raw_data_type) except (json.JSONDecodeError, TypeError) as error: raise AdapterUserError(message=_INVALID_SHOW_TYPE_MESSAGE) from error if not isinstance(decoded, dict) or not isinstance(decoded.get("type"), str): diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index f0fb30be60..ad77213d88 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -188,6 +188,7 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS _EXACT_COLUMN_INSPECTION_LIMIT: int = 32 +_CONCURRENT_EXACT_COLUMN_INSPECTION_LIMIT: int = 200 _STATE_TABLE_RETENTION_ATTRIBUTE: str = "_state_table_retention_fallbacks_state" _METADATA_INSPECTION_CONCURRENCY: int = 8 _SHOW_RESULT_LIMIT: int = 10_000 @@ -229,7 +230,7 @@ class SnowflakeAdapter( statement_limit_excludes_queueing: ClassVar[bool] = True metadata_inspection_concurrency: ClassVar[int] = _METADATA_INSPECTION_CONCURRENCY concurrent_ddl_limit: ClassVar[int] = 16 - exact_column_inspection_limit: ClassVar[int] = _EXACT_COLUMN_INSPECTION_LIMIT + exact_column_inspection_limit: ClassVar[int] = _CONCURRENT_EXACT_COLUMN_INSPECTION_LIMIT max_identifier_length: ClassVar[int] = 255 supports_session_warehouse: ClassVar[bool] = True diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index 4d3edc35c2..1a69dee445 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -224,6 +224,30 @@ expected_column_table_names=frozenset(f"model_{index}" for index in range(251)), expected_fingerprint_names=frozenset(), ), + GatherWarehouseSnapshotTestCase( + description=( + "lists the whole schema beyond the name threshold but reads columns only for " + "the selection and what it reads" + ), + setup_sql=( + *(f"CREATE TABLE staging.model_{index} (id INTEGER)" for index in range(251)), + "CREATE TABLE staging.unselected (value VARCHAR)", + ), + model_locations={ + **{f"model_{index}": "staging" for index in range(METADATA_NAME_FILTER_LIMIT + 1)}, + "unselected": "staging", + }, + seed_locations={}, + selected_keys=frozenset( + CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=f"model_{index}") + for index in range(METADATA_NAME_FILTER_LIMIT + 1) + ), + expected_relation_names=frozenset( + {*(f"model_{index}" for index in range(251)), "unselected"} + ), + expected_column_table_names=frozenset(f"model_{index}" for index in range(251)), + expected_fingerprint_names=frozenset(), + ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index c44c4047ec..c98ad03871 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -135,6 +135,7 @@ class SnowflakeManySchemasTestCase: description: str schema_count: int models_per_schema: int + schema_listing_beyond_32: bool expected_per_relation_column_reads: int expected_schema_column_reads: int diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 8c0c472404..eeb122945f 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -502,13 +502,14 @@ def plan_offline_snowflake_project( warehouse: RecordingSnowflakeWarehouse, no_cache: bool = True, plan_from: str | None = None, + adapter_type: type[OfflineSnowflakeAdapter] = OfflineSnowflakeAdapter, ) -> CompilePipelineResult: """Compile and plan a synthetic Snowflake project against the recording warehouse.""" inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project.project_dir) return run_compile_pipeline( discovered_inputs=inputs, - adapter=OfflineSnowflakeAdapter(warehouse=warehouse), + adapter=adapter_type(warehouse=warehouse), options=CompilePipelineOptions( no_sql_validation=True, no_cache=no_cache, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py index 7d00289314..5934ad93d4 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py @@ -34,6 +34,7 @@ OfflineSnowflakeAdapter, RecordedQuery, RecordingSnowflakeWarehouse, + SchemaListingSnowflakeAdapter, SyntheticSnowflakeProject, write_synthetic_snowflake_project, ) @@ -82,7 +83,7 @@ def test_given_large_multi_schema_project_when_planning_then_metadata_reads_stay ) result: CompilePipelineResult = plan_offline_snowflake_project( - project=project, warehouse=warehouse + project=project, warehouse=warehouse, adapter_type=SchemaListingSnowflakeAdapter ) metadata: tuple[RecordedQuery, ...] = offline_metadata_queries(warehouse) @@ -179,9 +180,18 @@ def test_given_planned_project_when_planning_again_then_cursor_bounds_are_read_a description="many small schemas read model columns once per schema", schema_count=24, models_per_schema=2, + schema_listing_beyond_32=True, expected_per_relation_column_reads=30, expected_schema_column_reads=24, - ) + ), + SnowflakeManySchemasTestCase( + description="many small schemas read each relation's columns once in production", + schema_count=24, + models_per_schema=2, + schema_listing_beyond_32=False, + expected_per_relation_column_reads=78, + expected_schema_column_reads=0, + ), ], ids=lambda case: case.description, ) @@ -200,7 +210,11 @@ def test_given_many_small_schemas_when_planning_then_columns_are_read_per_schema ) result: CompilePipelineResult = plan_offline_snowflake_project( - project=project, warehouse=warehouse + project=project, + warehouse=warehouse, + adapter_type=(OfflineSnowflakeAdapter, SchemaListingSnowflakeAdapter)[ + test_case.schema_listing_beyond_32 + ], ) assert len(result.plan_output.model_entries) == len(project.model_schemas) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py index d0814cd101..c95c119fb6 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py @@ -250,3 +250,12 @@ class SyncedFallbackTestCase: empty_copy: bool failing_check: bool expected_message: str + + +@dataclass(frozen=True) +class ExactColumnThresholdTestCase: + """A column request of one schema's relations and how many reads of each kind serve it.""" + + description: str + relation_count: int + expected_query_kinds: dict[str, int] diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py index 4fd598c1f5..70ab1ce838 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py @@ -16,7 +16,7 @@ from datetime import UTC, datetime from itertools import chain, compress from pathlib import Path -from typing import Any +from typing import Any, ClassVar import duckdb import pyarrow @@ -96,6 +96,8 @@ _BOUND_MARKERS: tuple[str, str] = ("MIN(", "MAX(") _BOUND_VALUE_INDEX: dict[bool, int] = {True: 0, False: 1} SHOW_RESULT_CAP: int = 10_000 +# The relation count beyond which schema listing tests read a schema's columns in one listing. +SCHEMA_LISTING_EXACT_COLUMN_LIMIT: int = 32 _SHOW_TABLE_FIELDS: tuple[str, ...] = ( "created_on", "name", @@ -811,6 +813,16 @@ def connect(self, config: dict[str, Any]) -> _SnowflakeConnection: return _SnowflakeConnection(self._recording_warehouse.raw_connection()) +class SchemaListingSnowflakeAdapter(OfflineSnowflakeAdapter): + """An offline adapter reading a schema's columns in one listing beyond a few relations. + + Keeps the fixtures of schema listing tests small: production reads up to a few hundred + relations one by one. + """ + + exact_column_inspection_limit: ClassVar[int] = SCHEMA_LISTING_EXACT_COLUMN_LIMIT + + @dataclass(frozen=True) class SyntheticSnowflakeProject: """Synthetic multi-schema project files and the warehouse relations they inspect.""" @@ -989,6 +1001,16 @@ def build_offline_snowflake( return adapter, adapter.connect({}), warehouse +def build_schema_listing_snowflake( + *, relations: tuple[FakeRelation, ...] +) -> tuple[OfflineSnowflakeAdapter, _SnowflakeConnection, RecordingSnowflakeWarehouse]: + """Build an offline adapter that lists a schema's columns beyond a few relations.""" + + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse(relations=relations) + adapter: OfflineSnowflakeAdapter = SchemaListingSnowflakeAdapter(warehouse=warehouse) + return adapter, adapter.connect({}), warehouse + + def sorted_relation_reprs(relations: tuple[RelationInfo, ...]) -> list[str]: """Return an order-independent form of listed relations without SHOW-absent LAST_ALTERED.""" diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_inspection_catalog.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_inspection_catalog.py index e63d00f698..092fadcfbb 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_inspection_catalog.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/test_inspection_catalog.py @@ -24,6 +24,7 @@ CappedColumnFallbackTestCase, ColumnRequestEquivalenceTestCase, DroppedRelationTestCase, + ExactColumnThresholdTestCase, FreshnessErrorTestCase, FreshnessReuseTestCase, MissingSchemaTestCase, @@ -41,6 +42,7 @@ RecordingSnowflakeWarehouse, build_inspection_catalog_relations, build_offline_snowflake, + build_schema_listing_snowflake, build_wide_schema_relations, build_wide_schemas_relations, sorted_relation_reprs, @@ -152,7 +154,7 @@ def test_given_metadata_request_when_served_by_catalog_then_matches_direct_adapt def test_given_relations_when_reading_columns_through_catalog_then_matches_direct_adapter( test_case: ColumnRequestEquivalenceTestCase, ) -> None: - adapter, connection, warehouse = build_offline_snowflake( + adapter, connection, warehouse = build_schema_listing_snowflake( relations=build_inspection_catalog_relations() ) relations: tuple[RelationInfo, ...] = adapter.list_relations( @@ -375,7 +377,7 @@ def test_given_unreadable_schema_when_prefetching_best_effort_then_only_real_rea def test_given_schema_beyond_show_cap_when_listing_then_matches_direct_adapter( test_case: ShowResultCapTestCase, ) -> None: - adapter, connection, warehouse = build_offline_snowflake( + adapter, connection, warehouse = build_schema_listing_snowflake( relations=build_wide_schema_relations( relation_count=test_case.relation_count, columns_per_relation=test_case.columns_per_relation, @@ -530,7 +532,7 @@ def test_given_dropped_relation_when_reading_exact_columns_then_it_is_missing( def test_given_cap_sized_column_listing_when_reading_then_only_needed_relations_are_reread( test_case: CappedColumnFallbackTestCase, ) -> None: - adapter, connection, warehouse = build_offline_snowflake( + adapter, connection, warehouse = build_schema_listing_snowflake( relations=build_wide_schema_relations( relation_count=test_case.relation_count, columns_per_relation=test_case.columns_per_relation, @@ -723,3 +725,45 @@ def test_given_schemas_when_prefetching_listings_then_tables_and_views_run_toget if __name__ == "__main__": pytest.main([__file__, "-vv"]) + + +@pytest.mark.parametrize( + "test_case", + [ + ExactColumnThresholdTestCase( + description="a few hundred relations are read one by one", + relation_count=150, + expected_query_kinds={"show_columns": 150}, + ), + ExactColumnThresholdTestCase( + description="more relations read the schema's columns in one listing", + relation_count=250, + expected_query_kinds={"show_schema_columns": 1}, + ), + ], + ids=lambda case: case.description, +) +def test_given_column_request_size_when_reading_through_catalog_then_picks_cheaper_read( + test_case: ExactColumnThresholdTestCase, +) -> None: + adapter, connection, warehouse = build_offline_snowflake( + relations=build_wide_schema_relations( + relation_count=test_case.relation_count, columns_per_relation=3 + ) + ) + listed: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, database="analytics", schemas=("inventory",) + ) + direct: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( + adapter.get_columns_for_relations(connection=connection, relations=listed) + ) + warehouse.reset() + + served: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = InspectionCatalog( + adapter=adapter, connection=connection + ).get_columns_for_relations(relations=listed) + + assert served == direct + assert Counter(query.kind for query in warehouse.queries) == Counter( + test_case.expected_query_kinds + ) From a700dab40cc384f00a67740f824752d088428c8d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:06:03 +0000 Subject: [PATCH 15/17] chore(fensu): record the intentional mirrors the plan work touches The Python node decorators, two-field key readers and the development location read each touch an existing near-miss cluster; each keeps its own record type or SQL, with the shared work already in one owner. --- fensu.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fensu.toml b/fensu.toml index 383c573c4d..bdf4213ae1 100644 --- a/fensu.toml +++ b/fensu.toml @@ -4010,3 +4010,15 @@ reason = "Each registers its own module's Python functions; only the add-functio [[dupes.allowlist]] paths = ["crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs"] reason = "source_start and source_end apply opposite boundary rules: a start inside a rewrite maps to the start of the replaced text, while an exclusive end inside or closing one maps to its end, and an end at a removal maps to where the removed text began. Only the partition-and-shift shape is alike." + +[[dupes.allowlist]] +paths = ["src/sqlbuild/python_nodes/main/_apply_loader.py", "src/sqlbuild/python_nodes/main/apply_asset.py", "src/sqlbuild/python_nodes/main/apply_check.py", "src/sqlbuild/python_nodes/main/apply_hook.py", "src/sqlbuild/python_nodes/main/apply_task.py", "src/sqlbuild/assets.py", "src/sqlbuild/python_nodes/_helpers/loader_authoring.py"] +reason = "Each is one public Python node decorator with its own keyword arguments and definition record. Dependency normalisation and description resolution go through the shared normalize_python_node_dependencies and resolve_description; only the decorate-and-attach shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/rules/_helpers/configuration/catalogue.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs"] +reason = "Each reads a two-field key from a different record type (a SqlResourceRef's kind value, a CompiledObjectKey's resource type, a rule catalogue or graph row); only the read-two-fields shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-orchestration/src/credentials/main/locations/location_named.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs"] +reason = "One reads the active location with a name, the other the development location of a target whose project path matches; each is one control-store read with its own SQL and filter, and only the read-then-decode-the-identity shape is alike." From 382755bcebd10b0a1595e3a4d1f8f3d81ef699e5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:47:15 +0000 Subject: [PATCH 16/17] refactor(graph): publish set dependencies from the project selection Moves the set_dependencies publication into ProjectSelection so the Python extension crate stays within its code-line cap after merging main. --- .../src/graph/models.rs | 27 +++++++++++++++++++ .../_helpers/compile_driver/plan_pipeline.rs | 4 +-- .../_helpers/location_command/snapshot.rs | 27 ++----------------- .../bindings/_helpers/plan_command/plan.rs | 4 +-- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index e3cf9b4061..76de4d0736 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -37,7 +37,9 @@ use sqlbuild_analysis::graph::models::BuildResources; use sqlbuild_analysis::graph::models::EdgeDirection; use sqlbuild_analysis::graph::models::GraphIndexes; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::node_set_dependencies::models::GraphNode; use sqlbuild_analysis::node_set_dependencies::models::ResolvedNode; +use sqlbuild_analysis::node_set_dependencies::models::ResolvedSet; use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; @@ -452,6 +454,31 @@ impl ProjectSelection { .map(|node| format!("{}:{name}", node.kind)) } + /// A Python node's published `set_dependencies`: each job or tag it depends on and its end node keys. + pub fn set_dependencies(&self, name: &str) -> Option { + let resolved = self.sets.iter().find(|resolved| resolved.name == name)?; + let sets: Vec = + resolved.sets.iter().map(|set| self.set_json(set)).collect(); + Some(serde_json::Value::Array(sets)) + } + + /// One published job or tag dependency with the keys of its end nodes. + fn set_json(&self, set: &ResolvedSet) -> serde_json::Value { + let mut ends: Vec = Vec::new(); + for end in &set.ends { + match end { + GraphNode::Sql((kind, name)) => ends.push(format!("{kind}:{name}")), + GraphNode::Python(name) => ends.extend(self.node_key(name)), + } + } + serde_json::json!({ + "kind": set.dependency.kind, + "name": set.dependency.name, + "exclude": set.dependency.exclude, + "ends": ends, + }) + } + /// Each Python node's published key and the keys it reads: its upstream Python nodes and its SQL deps. pub fn node_deps(&self) -> Vec<(String, BTreeSet)> { let upstream: BTreeMap<&str, &[String]> = self diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index 16d820b0aa..9b5c72c112 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -5,9 +5,7 @@ use pyo3::types::{PyDict, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_planner::live_reads::models::LiveReads; -use sqlbuild_python_planner::planner_phases::constants::COMPILING; -use sqlbuild_python_planner::planner_phases::constants::CONNECTING; -use sqlbuild_python_planner::planner_phases::constants::EVALUATING_RULES; +use sqlbuild_python_planner::planner_phases::constants::{COMPILING, CONNECTING, EVALUATING_RULES}; use sqlbuild_python_planner::planner_phases::main::timing::timed::timed; use sqlbuild_python_planner::planner_warehouse::models::PlannerReads; use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 670971770a..f2e7efbe18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -7,7 +7,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{IntoPyObject, pyfunction}; use serde_json::{Value, json}; -use sqlbuild_analysis::node_set_dependencies::models::GraphNode; use sqlbuild_analysis::planner_scope::main::node_loads::node_loads; use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; @@ -169,30 +168,8 @@ fn python_nodes( ))); }; published.definition["deps"] = json!(deps); - if let Some(resolved) = selected - .sets - .iter() - .find(|resolved| resolved.name == node.name) - { - published.definition["set_dependencies"] = json!( - resolved - .sets - .iter() - .map(|set| json!({ - "kind": set.dependency.kind, - "name": set.dependency.name, - "exclude": set.dependency.exclude, - "ends": set - .ends - .iter() - .filter_map(|end| match end { - GraphNode::Sql((kind, name)) => Some(format!("{kind}:{name}")), - GraphNode::Python(name) => selected.node_key(name), - }) - .collect::>(), - })) - .collect::>() - ); + if let Some(sets) = selected.set_dependencies(&node.name) { + published.definition["set_dependencies"] = sets; } published.definition["tags"] = json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 6f4644d36d..d9fc02c4d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -1,9 +1,7 @@ //! `sqb plan`: compile the plan pipeline and print the plan as text or JSON. use crate::bindings::_helpers::command_runtime::audit_failures::resolve_ignore_audit_failures; -use sqlbuild_python_planner::planner_phases::constants::BUILD_HISTORY; -use sqlbuild_python_planner::planner_phases::constants::DISCOVERING; -use sqlbuild_python_planner::planner_phases::constants::WRITING; +use sqlbuild_python_planner::planner_phases::constants::{BUILD_HISTORY, DISCOVERING, WRITING}; use sqlbuild_python_planner::planner_phases::main::timing::timed::timed; use sqlbuild_python_planner::planner_phases::models::PlanTimingScope; use sqlbuild_python_planner::planner_phases::models::PlanTimings; From f13893c3320a16defcda96fa639f7d69faf069c0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:32:43 +0000 Subject: [PATCH 17/17] test(goldens): list plan_from among a target's allowed keys --- tests/goldens/compiler/failure/target-unknown-key.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 34c7b96fbf..18a234ac3f 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ]