diff --git a/benchmarks/duckshop-10k/README.md b/benchmarks/duckshop-10k/README.md index d9d990e1c2..a16c6ff1ce 100644 --- a/benchmarks/duckshop-10k/README.md +++ b/benchmarks/duckshop-10k/README.md @@ -35,11 +35,13 @@ run `python benchmarks/duckshop-10k/generate.py --update-checksum`. | | Count | |---|---| -| Models | 10,282 | -| Staging, intermediate, marts and snapshots | 6,149 (59.8%), 1,961 (19.1%), 2,162 (21.0%), 10 | +| Models | 10,840 | +| Staging, intermediate, marts and snapshots | 6,149 (56.7%), 2,519 (23.2%), 2,162 (19.9%), 10 | +| Incremental models | 583, 112 of them with a capped automatic start and 112 renamed | +| Python export assets | 433 in 241 modules sharing one helper module | | Sources | 2,466 table sources and one expression source | | Unit tests and scenarios | 288 and 2 | -| Dependency depth | 11 at most; 168 models at depth 7 or more | +| Dependency depth | 11 at most; 170 models at depth 7 or more | Most models read one to four upstreams; a few dozen wide marts, unions and scorecards read ten or more. Staging deduplicates change logs with a change-log macro, types three platform layouts @@ -51,6 +53,13 @@ clickstream events in microbatches and builds customer identity, value and segme budget variance chain. Marts give every instance, team and region its reports, and publish facts and dimensions with enforced contracts. +A large project also carries planning workload the layers above barely touch, so the generator adds +it in the proportions such projects have: an incrementally kept history of every third timestamped +intermediate table, one in five of them capping how far ahead its automatic start may sit and one in +five renamed from an older name with `migrate_from`, and a Python export asset for every fifth +published mart, a few per module, all sharing one helper module. A live plan reads a cursor bound for +each incremental model, plans each rename and discovers each Python node. + The models that use `HASH`, `TIMESTAMP_LTZ` or `CURRENT_TIMESTAMP()` are tagged `known_divergence`, because their results depend on the engine or the session. diff --git a/benchmarks/duckshop-10k/checksum.txt b/benchmarks/duckshop-10k/checksum.txt index a49cc2c0f7..968cf304d0 100644 --- a/benchmarks/duckshop-10k/checksum.txt +++ b/benchmarks/duckshop-10k/checksum.txt @@ -1 +1 @@ -generator 3 files 10874 sha256 5f3d30b82ebf52f2eb869494f52f77d1699f9a08743febd6ee9ca0cb22ed0611 +generator 4 files 11674 sha256 d0012261f902e87a546a04bd8c6552c8a050f9cec6753b5acccc53584bce9d37 diff --git a/benchmarks/duckshop-10k/duckshop10k/__init__.py b/benchmarks/duckshop-10k/duckshop10k/__init__.py index 504bea12ab..6b01c6c3dd 100644 --- a/benchmarks/duckshop-10k/duckshop10k/__init__.py +++ b/benchmarks/duckshop-10k/duckshop10k/__init__.py @@ -4,6 +4,6 @@ regenerate checksum.txt, whenever a change alters any generated file. """ -GENERATOR_VERSION = "3" +GENERATOR_VERSION = "4" SEED = 20261024 PROJECT_NAME = "duckshop_10k" diff --git a/benchmarks/duckshop-10k/duckshop10k/project.py b/benchmarks/duckshop-10k/duckshop10k/project.py index a399a80e44..4475b2f5f1 100644 --- a/benchmarks/duckshop-10k/duckshop10k/project.py +++ b/benchmarks/duckshop-10k/duckshop10k/project.py @@ -13,6 +13,7 @@ from duckshop10k.rules_gen import build_rules from duckshop10k.staging import StagedTable, stage_collector, stage_instance from duckshop10k.tests_gen import build_scenarios, build_tests +from duckshop10k.workload import build_workload from duckshop10k.world import Instance, instances, storefront_instances @@ -89,6 +90,7 @@ def build(profile_name: str = "full") -> dict[str, str]: build_extras(project, intermediate) build_tests(project, staging.tables, intermediate) build_scenarios(project, staging.tables) + build_workload(project) build_rules(project) _static_files(project) project.write("seeds/source_instances.csv", instances_seed(systems)) diff --git a/benchmarks/duckshop-10k/duckshop10k/workload.py b/benchmarks/duckshop-10k/duckshop10k/workload.py new file mode 100644 index 0000000000..f39e47bd14 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/workload.py @@ -0,0 +1,153 @@ +"""Planning workload a large project carries: incremental history, renamed models and exports. + +A live plan of a real project reads a cursor bound for every incremental model, plans a +migration for every renamed model and discovers every Python node. The layers above build +few of each, so this module adds them in proportions such projects have: about one model in +twenty kept incrementally, a hundred of those renamed from an older name, and a few hundred +Python exports of published marts, spread over many modules that share helper code. +""" + +from __future__ import annotations + +from duckshop10k.core import Column, Model, Project + +#: Every this-many timestamped intermediate table gets an incrementally kept copy. +INCREMENT_STRIDE = 3 +#: One incremental model in this many caps how far ahead its automatic start may sit. +MAX_AHEAD_EVERY = 5 +#: One incremental model in this many was renamed from an older name. +RENAMED_EVERY = 5 +#: Every this-many published mart gets a Python export asset. +EXPORT_STRIDE = 5 +#: Export assets per Python module, so discovery walks many modules. +EXPORTS_PER_MODULE = 6 +_TIMESTAMP = "TIMESTAMP_NTZ" +_INCREMENT_FOLDER = "models/intermediate/increments" +_EXPORT_PACKAGE = "python/exports" +_SHARED_MODULE = "python/shared/export_feed.py" + +SHARED_EXPORT_HELPERS = '''"""What every export asset records about the relation it sent.""" + +from sqlbuild.assets import AssetContext +from sqlbuild.refs import SqlResourceRef + + +def feed_metadata(ctx: AssetContext, reference: SqlResourceRef, feed: str) -> dict[str, object]: + """The relation an export read, its row count and the partner feed it went to.""" + relation = ctx.relation(reference) + rows = ctx.query(f"SELECT COUNT(*) FROM {relation}").fetchone()[0] + return {"feed": feed, "relation": str(relation), "rows": rows, "format": feed_format(feed)} + + +def feed_format(feed: str) -> str: + """Partner feeds named after a region take CSV; every other feed takes Parquet.""" + return "csv" if feed.endswith(("_eu", "_us", "_apac", "_mea", "_latam")) else "parquet" +''' + + +def build_workload(project: Project) -> None: + """Add the incremental, renamed and Python export nodes to the assembled project.""" + + increments: list[Model] = _increments(project) + _rename(increments) + _exports(project) + + +def _timestamp_column(model: Model) -> Column | None: + return next((column for column in model.columns if column.type == _TIMESTAMP), None) + + +def _is_table(model: Model) -> bool: + return not any(line.startswith("materialized") for line in model.config) + + +def _increments(project: Project) -> list[Model]: + candidates: list[Model] = sorted( + ( + model + for model in project.models.values() + if model.layer == "intermediate" + and _is_table(model) + and _timestamp_column(model) is not None + ), + key=lambda model: model.name, + ) + added: list[Model] = [] + for position, upstream in enumerate(candidates[::INCREMENT_STRIDE]): + cursor: Column = next(column for column in upstream.columns if column.type == _TIMESTAMP) + config: list[str] = [ + "materialized incremental", + "incremental_strategy append", + f"cursor {cursor.name}", + "cursor_type timestamp", + "cursor_grain day", + f"cursor_inputs (\n {upstream.name} {cursor.name},\n )", + ] + if position % MAX_AHEAD_EVERY == 0: + config.append("cursor_start_max_ahead 1d") + columns: list[Column] = [ + Column(column.name, column.type, audits=("not_null",) * (column == cursor)) + for column in upstream.columns + ] + select: str = ",\n ".join(f"s.{column.name}" for column in columns) + added.append( + project.add( + Model( + name=f"{upstream.name}_history", + folder=f"{_INCREMENT_FOLDER}/{upstream.folder.rsplit('/', 1)[-1]}", + layer="intermediate", + description=( + f"Every row of {upstream.name} kept incrementally by {cursor.name}, so " + "earlier days stay queryable after the source rewrites them" + ), + columns=columns, + body=( + "WITH source_rows AS (\n SELECT *\n" + f' FROM __ref("{upstream.name}")\n),\n\n' + f"final AS (\n SELECT\n {select}\n FROM source_rows AS s\n)\n\n" + "SELECT * FROM final" + ), + config=config, + refs=[upstream.name], + ) + ) + ) + return added + + +def _rename(increments: list[Model]) -> None: + for model in increments[::RENAMED_EVERY]: + model.config.append(f"migrate_from {model.name.removesuffix('_history')}_archive") + + +def _exports(project: Project) -> None: + marts: list[Model] = sorted( + (model for model in project.models.values() if model.layer == "marts" and _is_table(model)), + key=lambda model: (model.folder, model.name), + ) + by_folder: dict[str, list[Model]] = {} + for model in marts[::EXPORT_STRIDE]: + by_folder.setdefault(model.folder.rsplit("/", 1)[-1], []).append(model) + project.write(_SHARED_MODULE, SHARED_EXPORT_HELPERS) + for folder, models in sorted(by_folder.items()): + for start in range(0, len(models), EXPORTS_PER_MODULE): + chunk: list[Model] = models[start : start + EXPORTS_PER_MODULE] + module: str = f"{folder}_{start // EXPORTS_PER_MODULE + 1:02d}" + project.write(f"{_EXPORT_PACKAGE}/{module}.py", _export_module(folder, chunk)) + + +def _export_module(folder: str, models: list[Model]) -> str: + assets: list[str] = [ + f'@asset(depends_on=model("{model.name}"))\n' + f"def export_{model.name}(ctx: AssetContext):\n" + f' """Send {model.name} to the {folder} partner feed."""\n' + f' metadata = feed_metadata(ctx, model("{model.name}"), "{folder}")\n' + " return ctx.result(metadata=metadata, materialized=True)\n" + for model in models + ] + return ( + f'"""Exports of the {folder} marts to partner feeds."""\n\n' + "from sqlbuild.assets import AssetContext, asset\n" + "from sqlbuild.refs import model\n\n" + "from python.shared.export_feed import feed_metadata\n\n\n" + "\n\n".join(assets) + ) diff --git a/benchmarks/duckshop-10k/tests/test_generator.py b/benchmarks/duckshop-10k/tests/test_generator.py index 5aa1fa478d..6d616b795f 100644 --- a/benchmarks/duckshop-10k/tests/test_generator.py +++ b/benchmarks/duckshop-10k/tests/test_generator.py @@ -39,13 +39,15 @@ def test_given_full_profile_when_building_then_matches_recorded_checksum() -> No assert checksum_line(full()) == (HERE / "checksum.txt").read_text() -def test_given_full_profile_when_measuring_then_has_about_ten_thousand_models_in_real_mix() -> None: +def test_given_full_profile_when_measuring_then_has_about_eleven_thousand_models_in_real_mix() -> ( + None +): measured = shape(full()) - assert 9_900 <= measured.models <= 10_300 - assert 58 <= measured.layer_percent("staging") <= 62 - assert 18 <= measured.layer_percent("intermediate") <= 21 - assert 19 <= measured.layer_percent("marts") + measured.layer_percent("snapshots") <= 23 + assert 10_500 <= measured.models <= 11_000 + assert 55 <= measured.layer_percent("staging") <= 59 + assert 21 <= measured.layer_percent("intermediate") <= 25 + assert 18 <= measured.layer_percent("marts") + measured.layer_percent("snapshots") <= 22 def test_given_full_profile_when_measuring_then_fan_in_and_depth_match_the_profile() -> None: @@ -78,6 +80,9 @@ def test_given_full_profile_when_measuring_then_samples_a_few_hundred_unit_tests (r"TRY_PARSE_JSON", 100), (r"\bFILTER\(", 10), (r"incremental_mode microbatch", 20), + (r"materialized incremental", 500), + (r"cursor_start_max_ahead", 100), + (r"migrate_from", 100), (r"materialized snapshot", 5), (r"__udf\(", 2), (r"__table_fn\(", 1), @@ -93,6 +98,18 @@ def test_given_full_profile_when_scanning_then_uses_each_construct( assert sum(1 for text in models(full()).values() if pattern.search(text)) >= minimum_models +def test_given_full_profile_when_scanning_then_writes_a_few_hundred_python_assets() -> None: + files = full() + modules = [path for path in files if path.startswith("python/exports/")] + assets = sum( + files[path].count("\n@asset(") + files[path].startswith("@asset(") for path in modules + ) + + assert 350 <= assets <= 500 + assert len(modules) >= 40 + assert all("from python.shared.export_feed import" in files[path] for path in modules) + + def test_given_known_divergence_constructs_when_scanning_then_models_are_tagged() -> None: for path, text in models(full()).items(): if re.search(r"\bHASH\(|TIMESTAMP_LTZ|CURRENT_TIMESTAMP\(\)", text.split(");", 1)[1]): diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index d6331e5dd2..2baa56243d 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -490,3 +490,5 @@ pub(crate) const CLICKHOUSE_DECIMAL_WIDTHS: [(&str, u32); 4] = [ ("Decimal128", 38), ("Decimal256", 76), ]; +/// Snowflake's precision of a fixed-point type written without one. +pub(crate) const SNOWFLAKE_DEFAULT_PRECISION: i64 = 38; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 7824a37df3..7bf841082f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -5,6 +5,7 @@ pub mod native_gap; pub(crate) mod native_outcomes; pub mod native_revalidation; pub mod query_expression_ranges; +pub mod same_column_type; pub mod schema_column; pub mod skipped_analysis; pub mod unapplied_nullability_rules; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/same_column_type.rs b/crates/sqlbuild-analysis/src/analysis_route/main/same_column_type.rs new file mode 100644 index 0000000000..9796803da1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/same_column_type.rs @@ -0,0 +1,48 @@ +//! Whether a declared column type and a warehouse-listed one name the type the engine stores. + +use crate::analysis_route::constants::{SNOWFLAKE_DEFAULT_PRECISION, SNOWFLAKE_DIALECT}; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::type_system::main::same_type::same_type; +use crate::type_system::models::{NormalizedType, PythonInteger, TypeFamily}; + +/// Whether `declared` and `listed` are one stored type in `dialect`, its defaults and synonyms met. +#[must_use] +pub fn same_column_type(dialect: Option<&str>, declared: &str, listed: &str) -> bool { + if declared.trim().eq_ignore_ascii_case(listed.trim()) { + return true; + } + let AnalysisRoute::Native(native) = analysis_route(dialect) else { + return false; + }; + let (Some(declared), Some(listed)) = ( + native.normalize_type(declared), + native.normalize_type(listed), + ) else { + return false; + }; + let (declared, listed): (NormalizedType, NormalizedType) = match native.name() { + SNOWFLAKE_DIALECT => (snowflake_defaults(declared), snowflake_defaults(listed)), + _ => (declared, listed), + }; + native.same_type(&declared, &listed) || same_type(&declared, &listed, native.name()) +} + +/// A Snowflake fixed-point type written without precision as the `NUMBER(38,0)` it stores. +fn snowflake_defaults(normalized: NormalizedType) -> NormalizedType { + if normalized.family != TypeFamily::Decimal || normalized.precision.is_some() { + return normalized; + } + let precision: PythonInteger = PythonInteger::from(SNOWFLAKE_DEFAULT_PRECISION); + let scale: PythonInteger = normalized + .scale + .clone() + .unwrap_or_else(|| PythonInteger::from(0)); + NormalizedType { + normalized_name: format!("DECIMAL({precision},{scale})"), + family: TypeFamily::Decimal, + precision: Some(precision), + scale: Some(scale), + length: None, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index 660e64b4ed..4c488bd8ed 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -24,6 +24,8 @@ mod test_native_outcome; mod test_native_revalidation; #[path = "tests/test_postgres_dialect.rs"] mod test_postgres_dialect; +#[path = "tests/test_same_column_type.rs"] +mod test_same_column_type; #[path = "tests/test_type_spelling.rs"] mod test_type_spelling; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_same_column_type.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_same_column_type.rs new file mode 100644 index 0000000000..50908556d4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_same_column_type.rs @@ -0,0 +1,132 @@ +use crate::analysis_route::main::same_column_type::same_column_type; +use crate::analysis_route::tests::test_types::EngineTypeMatchTestCase; + +#[test] +fn given_declared_and_snowflake_listed_types_when_comparing_then_default_spellings_match() { + let test_cases = [ + EngineTypeMatchTestCase { + description: "an unsized VARCHAR is VARCHAR(16777216)", + declared: "VARCHAR", + inferred: "VARCHAR(16777216)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "TEXT and STRING are VARCHAR(16777216)", + declared: "TEXT", + inferred: "VARCHAR(16777216)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "STRING is VARCHAR(16777216)", + declared: "STRING", + inferred: "VARCHAR(16777216)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a bare NUMBER is NUMBER(38,0)", + declared: "NUMBER", + inferred: "NUMBER(38,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a bare DECIMAL is NUMBER(38,0)", + declared: "DECIMAL", + inferred: "NUMBER(38,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "INT is NUMBER(38,0)", + declared: "INT", + inferred: "NUMBER(38,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "BIGINT is NUMBER(38,0)", + declared: "BIGINT", + inferred: "NUMBER(38,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "NUMBER(18) is NUMBER(18,0)", + declared: "NUMBER(18)", + inferred: "NUMBER(18,0)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "DOUBLE is FLOAT", + declared: "DOUBLE", + inferred: "FLOAT", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a stated timestamp precision meets the listed unstated one", + declared: "TIMESTAMP_NTZ(9)", + inferred: "TIMESTAMP_NTZ", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "spelling case does not matter", + declared: "varchar", + inferred: "VARCHAR(16777216)", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a shorter VARCHAR is a different type", + declared: "VARCHAR(10)", + inferred: "VARCHAR(16777216)", + expected_same_type: false, + }, + EngineTypeMatchTestCase { + description: "a scaled NUMBER is not an integer", + declared: "NUMBER(18,2)", + inferred: "NUMBER(38,0)", + expected_same_type: false, + }, + EngineTypeMatchTestCase { + description: "a string is not a number", + declared: "VARCHAR", + inferred: "NUMBER(38,0)", + expected_same_type: false, + }, + EngineTypeMatchTestCase { + description: "TIMESTAMP_TZ is not TIMESTAMP_NTZ", + declared: "TIMESTAMP_TZ", + inferred: "TIMESTAMP_NTZ", + expected_same_type: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + same_column_type(Some("snowflake"), test_case.declared, test_case.inferred), + test_case.expected_same_type, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_dialect_without_a_native_module_when_comparing_then_only_equal_text_matches() { + let test_cases = [ + EngineTypeMatchTestCase { + description: "equal text matches", + declared: "VARCHAR", + inferred: "varchar", + expected_same_type: true, + }, + EngineTypeMatchTestCase { + description: "a default length is not assumed", + declared: "VARCHAR", + inferred: "VARCHAR(16777216)", + expected_same_type: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + same_column_type(None, test_case.declared, test_case.inferred), + test_case.expected_same_type, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 848c4de883..5bb2c63eae 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -23,6 +23,7 @@ pub mod lineage; pub mod lineage_command; pub mod lineage_graph; pub mod manifest; +pub mod node_set_dependencies; pub mod nullability_rules; pub mod planner_changes; pub mod planner_fixtures; @@ -46,6 +47,7 @@ pub mod semantic_checks; pub mod semantic_usage; pub mod semantic_validation; pub mod snowflake_cost; +pub mod source_drift; pub mod sql_macros; pub mod sql_test_attachment; pub mod sql_tokens; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/combined.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/combined.rs new file mode 100644 index 0000000000..467db9f76b --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/combined.rs @@ -0,0 +1,142 @@ +//! The combined SQL and Python dependency graph a set dependency is reduced and checked over. + +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; + +use sqlbuild_core::graph_walk::main::find_cycle::find_cycle; + +use crate::node_set_dependencies::constants::SOURCE_KIND; +use crate::node_set_dependencies::models::GraphNode; +use crate::python_selection::models::{PythonSelectionGraph, SqlSelectionGraph}; + +/// Each node's upstream and downstream nodes, a source's loader upstream of it. +pub(crate) struct Combined { + upstream: HashMap>, + downstream: HashMap>, + python: Vec, +} + +impl Combined { + pub(crate) fn new(sql: SqlSelectionGraph<'_>, python: &PythonSelectionGraph) -> Self { + let mut upstream: HashMap> = HashMap::new(); + for (key, dependencies) in &sql.graph.upstream { + upstream + .entry(GraphNode::Sql(key.clone())) + .or_default() + .extend(dependencies.iter().cloned().map(GraphNode::Sql)); + } + for (source, loader) in sql.sources { + if let Some(loader) = loader { + upstream + .entry(GraphNode::Sql((SOURCE_KIND.to_owned(), source.clone()))) + .or_default() + .push(GraphNode::Python(loader.clone())); + } + } + for (name, dependencies) in &python.upstream { + upstream + .entry(GraphNode::Python(name.clone())) + .or_default() + .extend(dependencies.iter().cloned().map(GraphNode::Python)); + } + for node in &python.nodes { + upstream + .entry(GraphNode::Python(node.name.clone())) + .or_default() + .extend(node.sql_deps.iter().cloned().map(GraphNode::Sql)); + } + let mut downstream: HashMap> = HashMap::new(); + for (node, dependencies) in &upstream { + for dependency in dependencies { + downstream + .entry(dependency.clone()) + .or_default() + .push(node.clone()); + } + } + Self { + upstream, + downstream, + python: python + .nodes + .iter() + .map(|node| GraphNode::Python(node.name.clone())) + .collect(), + } + } + + pub(crate) fn upstream(&self, node: &GraphNode) -> &[GraphNode] { + self.upstream.get(node).map_or(&[], Vec::as_slice) + } + + /// Every node one or more steps upstream of `starts`, walking only nodes `through` allows. + pub(crate) fn reach_upstream<'a>( + &self, + starts: impl IntoIterator, + through: impl Fn(&GraphNode) -> bool, + ) -> HashSet { + self.reach(starts, |node| self.upstream(node), through) + } + + /// Every node one or more steps downstream of `start`. + pub(crate) fn reach_downstream(&self, start: &GraphNode) -> HashSet { + self.reach( + [start], + |node| self.downstream.get(node).map_or(&[], Vec::as_slice), + |_| true, + ) + } + + fn reach<'a, 'n>( + &'n self, + starts: impl IntoIterator, + next: impl Fn(&GraphNode) -> &'n [GraphNode], + through: impl Fn(&GraphNode) -> bool, + ) -> HashSet { + let mut reached: HashSet = HashSet::new(); + let mut queue: VecDeque = starts.into_iter().cloned().collect(); + while let Some(node) = queue.pop_front() { + for neighbour in next(&node) { + if reached.insert(neighbour.clone()) && through(neighbour) { + queue.push_back(neighbour.clone()); + } + } + } + reached + } + + /// A dependency cycle through a Python node, as the path from it back to itself. + pub(crate) fn python_cycle(&self) -> Option> { + let path: Vec = find_cycle(self.python.iter().cloned(), |node| { + self.upstream(node).to_vec() + })?; + let last: &GraphNode = path.last()?; + let start: usize = path.iter().position(|node| node == last)?; + Some(path[start..].to_vec()) + } + + /// The shortest path from `from` up to `to`, both included. + pub(crate) fn path_upstream(&self, from: &GraphNode, to: &GraphNode) -> Vec { + let mut parents: HashMap = HashMap::new(); + let mut queue: VecDeque = VecDeque::from([from.clone()]); + let mut seen: BTreeSet = BTreeSet::from([from.clone()]); + while let Some(node) = queue.pop_front() { + if node == *to { + let mut path: Vec = vec![node.clone()]; + let mut current: GraphNode = node; + while let Some(parent) = parents.get(¤t) { + path.push(parent.clone()); + current = parent.clone(); + } + path.reverse(); + return path; + } + for neighbour in self.upstream(&node) { + if seen.insert(neighbour.clone()) { + parents.insert(neighbour.clone(), node.clone()); + queue.push_back(neighbour.clone()); + } + } + } + vec![from.clone(), to.clone()] + } +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/group_loops.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/group_loops.rs new file mode 100644 index 0000000000..deb16dd4b7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/group_loops.rs @@ -0,0 +1,176 @@ +//! Job groups wait job by job: a job dependency that makes two members wait for each other is a cycle. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use sqlbuild_core::graph_walk::main::find_cycle::find_cycle; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::{JobSelection, ProjectGraph}; +use crate::node_set_dependencies::_helpers::combined::Combined; +use crate::node_set_dependencies::_helpers::labels::{listed, node_label}; +use crate::node_set_dependencies::_helpers::sets::{SetMembers, failure}; +use crate::node_set_dependencies::constants::JOB_SET; +use crate::node_set_dependencies::models::GraphNode; + +/// Each member job and the nodes it selects. +type Members = Vec<(String, HashSet)>; +/// Which members each member waits for, each wait with one `(their node, our node)` reason. +type Waits = BTreeMap>; + +/// The error for the first job group a job dependency makes wait in a loop. +pub(crate) fn group_loop( + jobs: &ProjectGraph, + catalog: &BTreeMap, + (explicit, current): (&Combined, &Combined), + (sets, kinds): (&[SetMembers<'_>], &HashMap), +) -> Option { + for (group, selection) in catalog { + let JobSelection::Group { members } = selection else { + continue; + }; + let members: Members = members + .iter() + .filter_map(|member| Some((member.clone(), job_nodes(jobs, member, kinds)?))) + .collect(); + let touched: bool = sets.iter().any(|set| crosses_members(set, &members)); + if !touched || wait_cycle(&waits(&members, explicit)).is_some() { + continue; + } + let after: Waits = waits(&members, current); + let Some(cycle) = wait_cycle(&after) else { + continue; + }; + let before: Waits = waits(&members, explicit); + return Some(loop_error(group, &cycle, (&before, &after), kinds)); + } + None +} + +/// Whether `set` is a job of the group that a node of another of its jobs depends on. +fn crosses_members(set: &SetMembers<'_>, members: &Members) -> bool { + let dependant: GraphNode = GraphNode::Python(set.dependant.name.clone()); + let names: Vec<&String> = members.iter().map(|(name, _)| name).collect(); + let elsewhere: bool = members + .iter() + .filter(|(name, _)| *name != set.dependency.name) + .any(|(_, nodes)| nodes.contains(&dependant)); + set.dependency.kind == JOB_SET && names.contains(&&set.dependency.name) && elsewhere +} + +/// The nodes `job` selects, Python nodes by name; `None` when it did not resolve. +fn job_nodes( + jobs: &ProjectGraph, + job: &str, + kinds: &HashMap, +) -> Option> { + let Some(Ok(keys)) = jobs.jobs.as_ref()?.get(job) else { + return None; + }; + Some( + keys.iter() + .map(|key: &ObjectKey| match kinds.get(&key.1) { + Some(kind) if *kind == key.0 => GraphNode::Python(key.1.clone()), + _ => GraphNode::Sql(key.clone()), + }) + .collect(), + ) +} + +/// What each member waits for: another member's node feeding its own, maybe through unselected nodes. +fn waits(members: &Members, combined: &Combined) -> Waits { + let selected: HashSet<&GraphNode> = members.iter().flat_map(|(_, nodes)| nodes).collect(); + let mut found: Waits = BTreeMap::new(); + for (member, own) in members { + for to in own { + for up in combined.upstream(to) { + if own.contains(up) { + continue; + } + let froms: Vec = if selected.contains(up) { + vec![up.clone()] + } else { + combined + .reach_upstream([up], |node| !selected.contains(node)) + .into_iter() + .filter(|node| selected.contains(node)) + .collect() + }; + for from in froms { + if own.contains(&from) { + continue; + } + for (other, theirs) in members { + if other != member && theirs.contains(&from) { + found + .entry(member.clone()) + .or_default() + .entry(other.clone()) + .or_insert_with(|| (from.clone(), to.clone())); + } + } + } + } + } + } + found +} + +/// The members of a wait cycle, the first waiting for the second and the last for the first. +fn wait_cycle(waits: &Waits) -> Option> { + let path: Vec = find_cycle(waits.keys().cloned(), |member: &String| { + waits + .get(member) + .map(|waited| waited.keys().cloned().collect::>()) + .unwrap_or_default() + })?; + let last: &String = path.last()?; + let start: usize = path.iter().position(|member| member == last)?; + Some(path[start..path.len() - 1].to_vec()) +} + +fn loop_error( + group: &str, + cycle: &[String], + (before, after): (&Waits, &Waits), + kinds: &HashMap, +) -> SelectorError { + let mut steps: Vec = Vec::new(); + let mut closers: BTreeSet = BTreeSet::new(); + let mut dependants: BTreeSet = BTreeSet::new(); + for (index, member) in cycle.iter().enumerate() { + let waited: &String = &cycle[(index + 1) % cycle.len()]; + let Some((from, to)) = after.get(member).and_then(|waits| waits.get(waited)) else { + continue; + }; + let added: bool = before + .get(member) + .is_none_or(|waits| !waits.contains_key(waited)); + let (from, to): (String, String) = (node_label(from, kinds), node_label(to, kinds)); + if added { + steps.push(format!( + "'{member}' waits for '{waited}' ({to} depends on job '{waited}', which ends in {from})" + )); + dependants.insert(format!("{to} out of '{member}'")); + } else { + steps.push(format!( + "'{member}' waits for '{waited}' ('{member}''s {to} reads {from})" + )); + closers.insert(to); + } + } + let closers: Vec = closers.into_iter().collect(); + let dependants: Vec = dependants.into_iter().collect(); + failure( + format!( + "a job dependency makes job group '{group}' wait in a loop: {}", + steps.join("; ") + ), + Some(format!( + "a job group runs each job once its jobs upstream have finished, so these jobs would \ + wait for each other; move {} to another job, or move {}", + listed(&closers), + listed(&dependants) + )), + ) +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs new file mode 100644 index 0000000000..d5614d3884 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/labels.rs @@ -0,0 +1,58 @@ +//! How job and tag dependency errors name nodes and sets. + +use std::collections::HashMap; + +use crate::node_set_dependencies::constants::NAMED_NODES; +use crate::node_set_dependencies::models::{GraphNode, NodeSetDependency}; + +/// `model 'orders'` or `task 'export_orders'`. +pub(crate) fn node_label(node: &GraphNode, kinds: &HashMap) -> String { + match node { + GraphNode::Sql((kind, name)) => format!("{kind} '{name}'"), + GraphNode::Python(name) => { + let kind: &str = kinds.get(name).map_or("node", String::as_str); + format!("{kind} '{name}'") + } + } +} + +/// The node's selector name, as an `exclude` lists it. +pub(crate) fn node_name(node: &GraphNode) -> &str { + match node { + GraphNode::Sql((_, name)) | GraphNode::Python(name) => name, + } +} + +/// `job 'nightly'` or `tag 'finance'`. +pub(crate) fn set_label(dependency: &NodeSetDependency) -> String { + format!("{} '{}'", dependency.kind, dependency.name) +} + +/// The authoring call that leaves `names` out of the dependency. +pub(crate) fn excluding(dependency: &NodeSetDependency, names: &[&str]) -> String { + let mut tokens: Vec<&str> = dependency + .exclude + .as_deref() + .map(|exclude| exclude.split_whitespace().collect()) + .unwrap_or_default(); + tokens.extend(names.iter().copied()); + format!( + "{}(\"{}\", exclude=\"{}\")", + dependency.kind, + dependency.name, + tokens.join(" ") + ) +} + +/// Up to [`NAMED_NODES`] labels joined, the rest counted. +pub(crate) fn listed(labels: &[String]) -> String { + let shown: Vec<&str> = labels + .iter() + .take(NAMED_NODES) + .map(String::as_str) + .collect(); + match labels.len().saturating_sub(NAMED_NODES) { + 0 => shown.join(", "), + more => format!("{} and {more} more", shown.join(", ")), + } +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/mod.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/mod.rs new file mode 100644 index 0000000000..b8c6beb9b3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod combined; +pub(crate) mod group_loops; +pub(crate) mod labels; +pub(crate) mod sets; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs new file mode 100644 index 0000000000..8399a32435 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/_helpers/sets.rs @@ -0,0 +1,280 @@ +//! One set dependency's members and end nodes, the edges they add, and the cycles those close. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use crate::graph::errors::SelectorError; +use crate::graph::models::JobSelection; +use crate::node_set_dependencies::_helpers::combined::Combined; +use crate::node_set_dependencies::_helpers::labels::{ + excluding, listed, node_label, node_name, set_label, +}; +use crate::node_set_dependencies::constants::{ + DEPENDABLE_PYTHON_KINDS, DEPENDABLE_SQL_KINDS, JOB_SET, LOADER_KIND, LOCATION_SEPARATOR, + NODE_SET_CODE, TAG_SET, +}; +use crate::node_set_dependencies::models::{ + DependantNode, GraphNode, NodeSetDependency, ResolvedNode, +}; +use crate::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; +use crate::python_selection::models::{PythonSelectionGraph, RunSelection, SqlSelectionGraph}; + +pub(crate) fn failure(message: String, help: Option) -> SelectorError { + SelectorError { + code: NODE_SET_CODE, + message, + help, + } +} + +/// Each Python node's kind, by name. +pub(crate) fn python_kinds(python: &PythonSelectionGraph) -> HashMap { + python + .nodes + .iter() + .map(|node| (node.name.clone(), node.kind.clone())) + .collect() +} + +/// The dependable nodes `dependency` names, less its `exclude`. +pub(crate) fn members( + (sql, python): (SqlSelectionGraph<'_>, &PythonSelectionGraph), + (dependant, dependency): (&DependantNode, &NodeSetDependency), + catalog: &BTreeMap, +) -> Result, SelectorError> { + let owner: String = format!("{} '{}'", dependant.kind, dependant.name); + let set: String = set_label(dependency); + if dependant.kind == LOADER_KIND { + return Err(failure( + format!("{owner} depends on {set}; loaders cannot depend on a job or a tag"), + Some("name the loaders, tasks or assets it depends on one by one".to_owned()), + )); + } + if dependency.name.contains(LOCATION_SEPARATOR) { + return Err(failure( + format!("{owner} depends on {set}, a job of another code location"), + Some("depending on another location's jobs is not supported yet".to_owned()), + )); + } + if dependency.kind == JOB_SET && !catalog.contains_key(&dependency.name) { + return Err(failure( + format!("{owner} depends on {set}, which the project does not declare"), + Some( + "name a job or job group from orchestration/jobs or orchestration/job_groups" + .to_owned(), + ), + )); + } + let tagged: bool = sql + .graph + .tags + .iter() + .any(|(tag, _)| *tag == dependency.name) + || python.tags.iter().any(|(tag, _)| *tag == dependency.name); + if dependency.kind == TAG_SET && !tagged { + return Err(failure( + format!("{owner} depends on {set}, which no node carries"), + None, + )); + } + let exclude: Vec = dependency + .exclude + .as_deref() + .map(|exclude| exclude.split_whitespace().map(str::to_owned).collect()) + .unwrap_or_default(); + let selected: RunSelection = resolve_python_sql_nodes( + sql, + python, + &[format!("{}:{}", dependency.kind, dependency.name)], + &exclude, + ) + .map_err(|error| { + failure( + format!("{owner} depends on {set}: {}", error.message), + error.help, + ) + })?; + let kinds: HashMap = python_kinds(python); + let found: BTreeSet = selected + .sql_keys + .into_iter() + .filter(|(kind, _)| DEPENDABLE_SQL_KINDS.contains(&kind.as_str())) + .map(GraphNode::Sql) + .chain( + selected + .python_node_names + .into_iter() + .filter(|name| { + kinds + .get(name) + .is_some_and(|kind| DEPENDABLE_PYTHON_KINDS.contains(&kind.as_str())) + }) + .map(GraphNode::Python), + ) + .collect(); + if found.is_empty() { + let after: &str = if exclude.is_empty() { + "" + } else { + " after its exclude" + }; + return Err(failure( + format!( + "{owner} depends on {set}, which leaves no model, seed, source, loader, task or \ + asset{after}" + ), + None, + )); + } + Ok(found) +} + +/// The members nothing else in the set reads, directly or through other nodes. +pub(crate) fn end_nodes(combined: &Combined, members: &BTreeSet) -> Vec { + let read: HashSet = combined.reach_upstream(members, |_| true); + members + .iter() + .filter(|member| !read.contains(*member)) + .cloned() + .collect() +} + +/// `python` with each resolved set's end nodes added to its node's dependencies. +pub(crate) fn with_edges( + python: &PythonSelectionGraph, + resolved: &[ResolvedNode], +) -> PythonSelectionGraph { + let mut graph: PythonSelectionGraph = python.clone(); + for node in resolved { + for end in node.sets.iter().flat_map(|set| &set.ends) { + match end { + GraphNode::Sql(key) => { + if let Some(owner) = graph.nodes.iter_mut().find(|item| item.name == node.name) + && !owner.sql_deps.contains(key) + { + owner.sql_deps.push(key.clone()); + } + } + GraphNode::Python(upstream) => { + graph.upstream = + with_edge(std::mem::take(&mut graph.upstream), &node.name, upstream); + graph.downstream = + with_edge(std::mem::take(&mut graph.downstream), upstream, &node.name); + } + } + } + } + graph +} + +/// `rows` with `other` listed against `name` once. +fn with_edge( + mut rows: Vec<(String, Vec)>, + name: &str, + other: &str, +) -> Vec<(String, Vec)> { + match rows.iter_mut().find(|(row, _)| row == name) { + Some((_, names)) if names.iter().any(|existing| existing == other) => {} + Some((_, names)) => names.push(other.to_owned()), + None => rows.push((name.to_owned(), vec![other.to_owned()])), + } + rows +} + +/// The `readers` no other reader stands between and `dependant`. +fn nearest_readers<'a>( + explicit: &Combined, + readers: &[&'a GraphNode], + dependant: &GraphNode, +) -> Vec<&'a GraphNode> { + let mut nearest: Vec<&GraphNode> = Vec::new(); + for reader in readers { + let reached: HashSet = + explicit.reach_upstream([*reader], |node| !readers.contains(&node)); + if reached.contains(dependant) { + nearest.push(reader); + } + } + nearest +} + +/// One resolved set with the members it was reduced from, for cycle errors. +pub(crate) struct SetMembers<'a> { + pub(crate) dependant: &'a DependantNode, + pub(crate) dependency: &'a NodeSetDependency, + pub(crate) members: BTreeSet, + pub(crate) ends: Vec, +} + +/// The set whose added edge is the cycle step `pair`, a dependant reading one of its end nodes. +fn closing_set<'s, 'a>( + explicit: &Combined, + sets: &'s [SetMembers<'a>], + pair: &[GraphNode], +) -> Option<&'s SetMembers<'a>> { + let GraphNode::Python(name) = &pair[0] else { + return None; + }; + let named: bool = explicit.upstream(&pair[0]).contains(&pair[1]); + sets.iter() + .find(|set| set.dependant.name == *name && set.ends.contains(&pair[1]) && !named) +} + +/// The error for a dependency cycle a set dependency closes; `None` when every cycle predates them. +pub(crate) fn set_cycle( + (explicit, current): (&Combined, &Combined), + sets: &[SetMembers<'_>], + kinds: &HashMap, +) -> Option { + let cycle: Vec = current.python_cycle()?; + let (index, set): (usize, &SetMembers<'_>) = cycle + .windows(2) + .enumerate() + .find_map(|(index, pair)| Some((index, closing_set(explicit, sets, pair)?)))?; + let dependant: GraphNode = cycle[index].clone(); + let owner: String = node_label(&dependant, kinds); + let label: String = set_label(set.dependency); + if set.members.contains(&dependant) { + return Some(failure( + format!("{owner} depends on {label}, which includes {owner} itself"), + Some(format!( + "leave it out of the dependency: {}", + excluding(set.dependency, &[node_name(&dependant)]) + )), + )); + } + let downstream: HashSet = explicit.reach_downstream(&dependant); + let readers: Vec<&GraphNode> = set + .members + .iter() + .filter(|member| downstream.contains(*member)) + .collect(); + let nearest: Vec<&GraphNode> = nearest_readers(explicit, &readers, &dependant); + let closer: &GraphNode = nearest.first().copied().or(readers.first().copied())?; + let path: Vec = explicit + .path_upstream(closer, &dependant) + .iter() + .map(|node| node_label(node, kinds)) + .collect(); + let nearest_labels: Vec = nearest.iter().map(|node| node_label(node, kinds)).collect(); + let reader_names: Vec<&str> = readers.iter().map(|node| node_name(node)).collect(); + let owner_of_set: String = match set.dependency.kind.as_str() { + JOB_SET => format!("{label}'s"), + _ => format!("the {label} node"), + }; + let remedy: &str = match set.dependency.kind.as_str() { + JOB_SET => "move it to another job", + _ => "remove the tag from it", + }; + Some(failure( + format!( + "{owner} depends on {label}, but {owner_of_set} {} reads {owner}'s output ({}), \ + a dependency cycle", + listed(&nearest_labels), + path.join(" <- ") + ), + Some(format!( + "{remedy}, or leave every node of the set that reads {owner} out of the dependency: {}", + excluding(set.dependency, &reader_names) + )), + )) +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs new file mode 100644 index 0000000000..363915abc0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/constants.rs @@ -0,0 +1,26 @@ +//! Kinds, codes and messages of job and tag dependencies. + +/// A dependency on a job's (or job group's) selection. +pub(crate) const JOB_SET: &str = "job"; +/// A dependency on every node carrying a tag. +pub(crate) const TAG_SET: &str = "tag"; +/// The planner code these compile errors carry. +pub(crate) const NODE_SET_CODE: &str = "S000"; +/// SQL resource types a Python node may depend on. +pub(crate) const DEPENDABLE_SQL_KINDS: [&str; 3] = ["model", "seed", "source"]; +/// Python node kinds a Python node may depend on. +pub(crate) const DEPENDABLE_PYTHON_KINDS: [&str; 3] = ["loader", "task", "asset"]; +/// The Python node kind that may depend on Python nodes only. +pub(crate) const LOADER_KIND: &str = "loader"; +/// The SQL resource type a loader fills. +pub(crate) const SOURCE_KIND: &str = "source"; +/// What separates a code location from a job name in a cross-location dependency. +pub(crate) const LOCATION_SEPARATOR: char = ':'; +/// Nodes a cycle error names one by one before summarising the rest. +pub(crate) const NAMED_NODES: usize = 5; +/// The identity metadata key a Python node's resolved set dependencies are recorded under. +pub(crate) const SET_DEPENDENCIES_KEY: &str = "set_dependencies"; +/// Each recorded set's label key. +pub(crate) const SET_KEY: &str = "set"; +/// Each recorded set's end node labels key. +pub(crate) const ENDS_KEY: &str = "ends"; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/main/mod.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/main/mod.rs new file mode 100644 index 0000000000..aa5575f475 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/main/mod.rs @@ -0,0 +1,3 @@ +pub mod resolve_node_set_dependencies; +pub mod set_dependency_metadata; +pub mod wait_changes; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/main/resolve_node_set_dependencies.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/main/resolve_node_set_dependencies.rs new file mode 100644 index 0000000000..7122804f0b --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/main/resolve_node_set_dependencies.rs @@ -0,0 +1,88 @@ +//! `resolve_node_set_dependencies`: each `job(...)` and `tag(...)` dependency as edges to end nodes. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::graph::errors::SelectorError; +use crate::graph::models::{JobSelection, ProjectGraph}; +use crate::node_set_dependencies::_helpers::combined::Combined; +use crate::node_set_dependencies::_helpers::group_loops::group_loop; +use crate::node_set_dependencies::_helpers::sets::{ + SetMembers, end_nodes, members, python_kinds, set_cycle, with_edges, +}; +use crate::node_set_dependencies::models::{DependantNode, GraphNode, ResolvedNode, ResolvedSet}; +use crate::python_selection::main::attach_python_jobs::attach_python_jobs; +use crate::python_selection::models::{PythonSelectionGraph, SqlSelectionGraph}; + +/// Each dependant's sets reduced to end nodes, resolved again over added edges until none change. +pub fn resolve_node_set_dependencies( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + dependants: &[DependantNode], + catalog: &BTreeMap, +) -> Result, SelectorError> { + if dependants.is_empty() { + return Ok(Vec::new()); + } + let jobs: ProjectGraph = attach_python_jobs(sql, python, catalog.clone()); + let with_jobs: SqlSelectionGraph<'_> = SqlSelectionGraph { + graph: &jobs, + ..sql + }; + let kinds: HashMap = python_kinds(python); + let explicit: Combined = Combined::new(sql, python); + let mut current: PythonSelectionGraph = python.clone(); + let mut sets: Vec> = Vec::new(); + for _ in 0..=dependants.len() { + let combined: Combined = Combined::new(sql, ¤t); + sets = Vec::new(); + for dependant in dependants { + for dependency in &dependant.sets { + let found: BTreeSet = + members((with_jobs, ¤t), (dependant, dependency), catalog)?; + let ends: Vec = end_nodes(&combined, &found); + sets.push(SetMembers { + dependant, + dependency, + members: found, + ends, + }); + } + } + let next: PythonSelectionGraph = with_edges(python, &resolved(&sets)); + if let Some(error) = set_cycle((&explicit, &Combined::new(sql, &next)), &sets, &kinds) { + return Err(error); + } + if next == current { + break; + } + current = next; + } + let resolved: Vec = resolved(&sets); + let current: Combined = Combined::new(sql, ¤t); + match group_loop(&jobs, catalog, (&explicit, ¤t), (&sets, &kinds)) { + Some(error) => Err(error), + None => Ok(resolved), + } +} + +/// Each dependant's resolved sets, in declaration order. +fn resolved(sets: &[SetMembers<'_>]) -> Vec { + let mut nodes: Vec = Vec::new(); + for set in sets { + let resolved: ResolvedSet = ResolvedSet { + dependency: set.dependency.clone(), + ends: set.ends.clone(), + }; + match nodes + .iter_mut() + .find(|node| node.name == set.dependant.name) + { + Some(node) => node.sets.push(resolved), + None => nodes.push(ResolvedNode { + name: set.dependant.name.clone(), + sets: vec![resolved], + }), + } + } + nodes +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/main/set_dependency_metadata.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/main/set_dependency_metadata.rs new file mode 100644 index 0000000000..a4cc6d095b --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/main/set_dependency_metadata.rs @@ -0,0 +1,19 @@ +//! `set_dependency_metadata`: a Python node's identity metadata with its resolved sets recorded. + +use serde_json::{Value, json}; + +use crate::node_set_dependencies::constants::{ENDS_KEY, SET_DEPENDENCIES_KEY, SET_KEY}; + +/// `metadata_json` with each `(set label, end node labels)` recorded; a non-object is unchanged. +#[must_use] +pub fn set_dependency_metadata(metadata_json: &str, sets: &[(String, Vec)]) -> String { + let Ok(Value::Object(mut metadata)) = serde_json::from_str::(metadata_json) else { + return metadata_json.to_owned(); + }; + let recorded: Vec = sets + .iter() + .map(|(set, ends)| json!({ SET_KEY: set, ENDS_KEY: ends })) + .collect(); + metadata.insert(SET_DEPENDENCIES_KEY.to_owned(), Value::Array(recorded)); + Value::Object(metadata).to_string() +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/main/wait_changes.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/main/wait_changes.rs new file mode 100644 index 0000000000..5cd0f9eda1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/main/wait_changes.rs @@ -0,0 +1,55 @@ +//! `wait_changes`: what a Python node newly waits on, or no longer, through its set dependencies. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +use crate::node_set_dependencies::constants::{ENDS_KEY, SET_DEPENDENCIES_KEY, SET_KEY}; + +/// Each end node `node` gained or lost per set since its last build; nothing if that recorded none. +#[must_use] +pub fn wait_changes(node: &str, previous: Option<&str>, current: Option<&str>) -> Vec { + let (Some(previous), Some(current)) = (previous.and_then(sets), current.and_then(sets)) else { + return Vec::new(); + }; + let labels: BTreeSet<&String> = previous.keys().chain(current.keys()).collect(); + let empty: BTreeSet = BTreeSet::new(); + let mut lines: Vec = Vec::new(); + for label in labels { + let before: &BTreeSet = previous.get(label).unwrap_or(&empty); + let after: &BTreeSet = current.get(label).unwrap_or(&empty); + lines.extend( + after + .difference(before) + .map(|end| format!("{node} now also waits on {end} through {label}")), + ); + lines.extend( + before + .difference(after) + .map(|end| format!("{node} no longer waits on {end} through {label}")), + ); + } + lines +} + +/// Each recorded set's end node labels, or `None` when the metadata records no sets. +fn sets(metadata_json: &str) -> Option>> { + let Ok(metadata) = serde_json::from_str::(metadata_json) else { + return None; + }; + let recorded: &Vec = metadata.get(SET_DEPENDENCIES_KEY)?.as_array()?; + Some(recorded.iter().filter_map(recorded_set).collect()) +} + +/// One recorded set's label and end node labels. +fn recorded_set(set: &Value) -> Option<(String, BTreeSet)> { + let label: String = set.get(SET_KEY)?.as_str()?.to_owned(); + let ends: BTreeSet = set + .get(ENDS_KEY)? + .as_array()? + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(); + Some((label, ends)) +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/mod.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/mod.rs new file mode 100644 index 0000000000..888f54a0fd --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/mod.rs @@ -0,0 +1,9 @@ +//! Python-node dependencies on a job or a tag, resolved into edges to the set's end nodes. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs new file mode 100644 index 0000000000..e8b824ace1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/models.rs @@ -0,0 +1,44 @@ +//! A Python node's job and tag dependencies and the end nodes each resolves to. + +use crate::assembly::project::types::ObjectKey; + +/// One `job(...)` or `tag(...)` dependency as authored. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeSetDependency { + /// `job` or `tag`. + pub kind: String, + pub name: String, + /// Selector tokens removed from the set before it is reduced to its end nodes. + pub exclude: Option, +} + +/// A Python node and the set dependencies it declares. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DependantNode { + pub name: String, + /// The node's kind, such as `task`. + pub kind: String, + pub sets: Vec, +} + +/// A node of the combined SQL and Python graph. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum GraphNode { + Sql(ObjectKey), + Python(String), +} + +/// One set dependency and the end nodes it resolved to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResolvedSet { + pub dependency: NodeSetDependency, + /// The set's dependable nodes nothing else in the set reads, in a stable order. + pub ends: Vec, +} + +/// A Python node's set dependencies, each resolved. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResolvedNode { + pub name: String, + pub sets: Vec, +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests.rs new file mode 100644 index 0000000000..78ba424993 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_resolve_node_set_dependencies.rs"] +mod resolve_node_set_dependencies; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_wait_changes.rs"] +mod wait_changes; diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs new file mode 100644 index 0000000000..716ac53e18 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/helpers.rs @@ -0,0 +1,211 @@ +use std::collections::{BTreeMap, HashMap}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::main::build_project_graph::build_project_graph; +use crate::graph::models::{GraphResource, JobSelection, ProjectGraph}; +use crate::node_set_dependencies::_helpers::labels::node_label; +use crate::node_set_dependencies::main::resolve_node_set_dependencies::resolve_node_set_dependencies; +use crate::node_set_dependencies::main::set_dependency_metadata::set_dependency_metadata; +use crate::node_set_dependencies::models::{DependantNode, NodeSetDependency, ResolvedNode}; +use crate::python_selection::main::python_node_graph_indexes::python_node_graph_indexes; +use crate::python_selection::models::{ + InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, SqlSelectionGraph, +}; + +/// `(name, kind, tags, upstream nodes, sql deps)`. +type NodeSpec = ( + &'static str, + &'static str, + &'static [&'static str], + &'static [&'static str], + &'static [(&'static str, &'static str)], +); +/// `(kind, name, exclude)` of one set dependency. +pub(super) type SetSpec = (&'static str, &'static str, Option<&'static str>); + +const NODES: &[NodeSpec] = &[ + ("report", "task", &["reports"], &[], &[]), + ("export_orders", "task", &[], &[], &[("model", "orders")]), + ( + "audit_export", + "task", + &[], + &["export_orders", "report"], + &[], + ), + ("self_tagged", "task", &["finance"], &[], &[]), + ("sync_events", "loader", &[], &[], &[]), +]; + +/// `(resource type, name, tags, dependencies)` of each SQL resource. +const RESOURCES: &[(&str, &str, &[&str], &[&str])] = &[ + ("source", "raw_orders", &[], &[]), + ("model", "stg_orders", &[], &["source:raw_orders"]), + ("model", "orders", &["finance"], &["model:stg_orders"]), + ("model", "order_items", &["finance"], &["model:stg_orders"]), + ( + "model", + "finance_daily", + &[], + &["model:orders", "model:order_items"], + ), + ("model", "customers", &["crm"], &[]), + ( + "model", + "customer_orders", + &[], + &["model:customers", "model:orders"], + ), +]; + +/// Each job's `select`; a group lists its member jobs. +const JOBS: &[(&str, &str)] = &[ + ("nightly", "stg_orders orders order_items"), + ("single", "customers"), + ("diamond", "stg_orders orders order_items finance_daily"), + ("with_task", "orders task:export_orders"), + ("loop", "task:export_orders task:audit_export"), + ("nightly_wide", "orders customer_orders"), + ("reporting", "task:report customers"), +]; + +fn key(spec: &str) -> ObjectKey { + let (kind, name) = spec.split_once(':').unwrap_or(("model", spec)); + (kind.to_owned(), name.to_owned()) +} + +fn owned(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +fn sql_graph() -> ProjectGraph { + build_project_graph( + &RESOURCES + .iter() + .map(|(kind, name, tags, deps)| GraphResource { + key: ((*kind).to_owned(), (*name).to_owned()), + deps: deps.iter().map(|dep| key(dep)).collect(), + tags: owned(tags), + folder: (*kind == "model").then(String::new), + }) + .collect::>(), + ) +} + +fn python_graph() -> PythonSelectionGraph { + let inventory: Vec = NODES + .iter() + .map(|(name, _, tags, upstream, _)| InventoryNode { + name: (*name).to_owned(), + tags: owned(tags), + relative_path: format!("python/{name}.py"), + upstream: owned(upstream), + }) + .collect(); + let indexes: InventoryIndexes = python_node_graph_indexes(&inventory); + PythonSelectionGraph { + nodes: NODES + .iter() + .map(|(name, kind, _, _, sql_deps)| PythonSelectionNode { + name: (*name).to_owned(), + kind: (*kind).to_owned(), + sql_deps: sql_deps + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(), + }) + .collect(), + upstream: indexes.upstream, + downstream: indexes.downstream, + tags: indexes.tags, + paths: indexes.paths, + } +} + +/// The fixture's jobs and the job groups `groups` lists. +fn catalog(groups: &[(&str, &[&str])]) -> BTreeMap { + let mut catalog: BTreeMap = JOBS + .iter() + .map(|(name, select)| { + ( + (*name).to_owned(), + JobSelection::Job { + select: select.split_whitespace().map(str::to_owned).collect(), + exclude: Vec::new(), + }, + ) + }) + .collect(); + catalog.extend(groups.iter().map(|(name, members)| { + ( + (*name).to_owned(), + JobSelection::Group { + members: owned(members), + }, + ) + })); + catalog +} + +/// The sets of `dependant` resolved over the fixture project. +pub(super) fn resolve( + (dependant, kind): (&str, &str), + sets: &[SetSpec], + groups: &[(&str, &[&str])], +) -> Result, SelectorError> { + let graph: ProjectGraph = sql_graph(); + let sources: Vec<(String, Option)> = vec![("raw_orders".to_owned(), None)]; + resolve_node_set_dependencies( + SqlSelectionGraph { + graph: &graph, + sources: &sources, + allow_warehouse_round_trips: false, + }, + &python_graph(), + &[DependantNode { + name: dependant.to_owned(), + kind: kind.to_owned(), + sets: sets + .iter() + .map(|(kind, name, exclude)| NodeSetDependency { + kind: (*kind).to_owned(), + name: (*name).to_owned(), + exclude: exclude.map(str::to_owned), + }) + .collect(), + }], + &catalog(groups), + ) +} + +/// Each resolved set's end nodes, labelled as errors name them. +pub(super) fn ends(resolved: &[ResolvedNode]) -> Vec> { + let kinds: HashMap = NODES + .iter() + .map(|spec| (spec.0.to_owned(), spec.1.to_owned())) + .collect(); + resolved + .iter() + .flat_map(|node| &node.sets) + .map(|set| set.ends.iter().map(|end| node_label(end, &kinds)).collect()) + .collect() +} + +/// The fragments `text` lacks. +pub(super) fn missing<'a>(text: &str, fragments: &[&'a str]) -> Vec<&'a str> { + fragments + .iter() + .filter(|fragment| !text.contains(**fragment)) + .copied() + .collect() +} + +/// Base identity metadata with `sets` recorded as `(set label, end node labels)`. +pub(super) fn recorded(base: &str, sets: &[(&str, &[&str])]) -> String { + let owned_sets: Vec<(String, Vec)> = sets + .iter() + .map(|(set, ends)| ((*set).to_owned(), owned(ends))) + .collect(); + set_dependency_metadata(base, &owned_sets) +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs new file mode 100644 index 0000000000..9c3fb98dec --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_resolve_node_set_dependencies.rs @@ -0,0 +1,173 @@ +use crate::node_set_dependencies::tests::helpers::{ends, missing, resolve}; +use crate::node_set_dependencies::tests::test_types::{EndNodesTestCase, FailureTestCase}; + +/// The group running `nightly_wide` and `reporting` together. +const GROUP: &[(&str, &[&str])] = &[("daily_group", &["nightly_wide", "reporting"])]; + +#[test] +fn given_job_and_tag_dependencies_when_resolving_then_each_reduces_to_its_end_nodes() { + let test_cases = [ + EndNodesTestCase { + description: "a job whose selection branches ends in each branch", + groups: &[], + sets: &[("job", "nightly", None)], + expected_ends: &[&["model 'order_items'", "model 'orders'"]], + }, + EndNodesTestCase { + description: "a job of one node ends in it", + groups: &[], + sets: &[("job", "single", None)], + expected_ends: &[&["model 'customers'"]], + }, + EndNodesTestCase { + description: "a diamond ends in the node joining its branches", + groups: &[], + sets: &[("job", "diamond", None)], + expected_ends: &[&["model 'finance_daily'"]], + }, + EndNodesTestCase { + description: "a Python node reading the job's model is its end", + groups: &[], + sets: &[("job", "with_task", None)], + expected_ends: &[&["task 'export_orders'"]], + }, + EndNodesTestCase { + description: "exclude removes nodes before reducing to end nodes", + groups: &[], + sets: &[("job", "diamond", Some("finance_daily"))], + expected_ends: &[&["model 'order_items'", "model 'orders'"]], + }, + EndNodesTestCase { + description: "exclude takes tag selectors", + groups: &[], + sets: &[("job", "diamond", Some("tag:finance finance_daily"))], + expected_ends: &[&["model 'stg_orders'"]], + }, + EndNodesTestCase { + description: "a tag ends in its tagged nodes nothing tagged reads", + groups: &[], + sets: &[("tag", "crm", None)], + expected_ends: &[&["model 'customers'"]], + }, + EndNodesTestCase { + description: "a tag excluding a Python node keeps its SQL", + groups: &[], + sets: &[("tag", "finance", Some("self_tagged"))], + expected_ends: &[&["model 'order_items'", "model 'orders'"]], + }, + EndNodesTestCase { + description: "a job reading the dependant's job resolves when no group runs them together", + groups: &[], + sets: &[("job", "nightly_wide", None)], + expected_ends: &[&["model 'customer_orders'"]], + }, + EndNodesTestCase { + description: "each of several sets resolves on its own", + groups: &[], + sets: &[("job", "single", None), ("tag", "crm", None)], + expected_ends: &[&["model 'customers'"], &["model 'customers'"]], + }, + ]; + for test_case in test_cases { + let resolved = resolve(("report", "task"), test_case.sets, test_case.groups) + .expect(test_case.description); + assert_eq!( + ends(&resolved), + test_case.expected_ends, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unresolvable_dependencies_when_resolving_then_compile_fails_naming_the_cause() { + let test_cases = [ + FailureTestCase { + description: "an unknown job", + dependant: ("report", "task"), + groups: &[], + sets: &[("job", "weekly", None)], + expected_fragments: &[ + "task 'report' depends on job 'weekly', which the project does not declare", + ], + }, + FailureTestCase { + description: "an unknown tag", + dependant: ("report", "task"), + groups: &[], + sets: &[("tag", "marketing", None)], + expected_fragments: &[ + "task 'report' depends on tag 'marketing', which no node carries", + ], + }, + FailureTestCase { + description: "a set its exclude empties", + dependant: ("report", "task"), + groups: &[], + sets: &[("job", "single", Some("customers"))], + expected_fragments: &[ + "depends on job 'single', which leaves no model", + "after its exclude", + ], + }, + FailureTestCase { + description: "another location's job", + dependant: ("report", "task"), + groups: &[], + sets: &[("job", "billing:nightly", None)], + expected_fragments: &["a job of another code location"], + }, + FailureTestCase { + description: "a loader's set", + dependant: ("sync_events", "loader"), + groups: &[], + sets: &[("job", "single", None)], + expected_fragments: &[ + "loader 'sync_events' depends on job 'single'; loaders cannot depend on a job or a tag", + ], + }, + FailureTestCase { + description: "a job node reading the dependant's output closes a cycle", + dependant: ("report", "task"), + groups: &[], + sets: &[("job", "loop", None)], + expected_fragments: &[ + "task 'report' depends on job 'loop', but job 'loop''s task 'audit_export' reads task 'report''s output", + "job(\"loop\", exclude=\"audit_export\")", + ], + }, + FailureTestCase { + description: "a tag the dependant carries includes it", + dependant: ("self_tagged", "task"), + groups: &[], + sets: &[("tag", "finance", None)], + expected_fragments: &[ + "task 'self_tagged' depends on tag 'finance', which includes task 'self_tagged' itself", + "tag(\"finance\", exclude=\"self_tagged\")", + ], + }, + FailureTestCase { + description: "a group whose jobs would wait for each other", + dependant: ("report", "task"), + groups: GROUP, + sets: &[("job", "nightly_wide", None)], + expected_fragments: &[ + "job group 'daily_group' wait in a loop", + "'nightly_wide' waits for 'reporting' ('nightly_wide''s model 'customer_orders' reads model 'customers')", + "move model 'customer_orders' to another job", + ], + }, + ]; + for test_case in test_cases { + let error = resolve(test_case.dependant, test_case.sets, test_case.groups) + .expect_err(test_case.description); + let text: String = format!("{} {}", error.message, error.help.unwrap_or_default()); + assert_eq!( + missing(&text, test_case.expected_fragments), + Vec::<&str>::new(), + "{}: {text}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs new file mode 100644 index 0000000000..8bc4c18a1c --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_types.rs @@ -0,0 +1,30 @@ +/// One set dependency of the `report` task and the end nodes it resolves to. +pub(super) struct EndNodesTestCase { + pub(super) description: &'static str, + /// `(kind, name, exclude)` of each `job(...)` or `tag(...)` dependency. + pub(super) sets: &'static [(&'static str, &'static str, Option<&'static str>)], + /// The job groups the catalog holds, by name and members. + pub(super) groups: &'static [(&'static str, &'static [&'static str])], + /// Each end node as errors name it, per set. + pub(super) expected_ends: &'static [&'static [&'static str]], +} + +/// A dependency that cannot resolve, and fragments its error must carry. +pub(super) struct FailureTestCase { + pub(super) description: &'static str, + /// The dependant's name and kind. + pub(super) dependant: (&'static str, &'static str), + pub(super) sets: &'static [(&'static str, &'static str, Option<&'static str>)], + /// The job groups the catalog holds, by name and members. + pub(super) groups: &'static [(&'static str, &'static [&'static str])], + pub(super) expected_fragments: &'static [&'static str], +} + +/// The sets a node was last built with and has now, and the wait changes a plan lists. +pub(super) struct WaitChangesTestCase { + pub(super) description: &'static str, + /// `None` when the last build recorded no sets. + pub(super) previous: Option<&'static [(&'static str, &'static [&'static str])]>, + pub(super) current: &'static [(&'static str, &'static [&'static str])], + pub(super) expected_lines: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_wait_changes.rs b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_wait_changes.rs new file mode 100644 index 0000000000..0fd5956839 --- /dev/null +++ b/crates/sqlbuild-analysis/src/node_set_dependencies/tests/test_wait_changes.rs @@ -0,0 +1,50 @@ +use crate::node_set_dependencies::main::wait_changes::wait_changes; +use crate::node_set_dependencies::tests::helpers::recorded; +use crate::node_set_dependencies::tests::test_types::WaitChangesTestCase; + +const BASE: &str = r#"{"dependencies":[]}"#; + +#[test] +fn given_set_ends_that_changed_since_the_last_build_when_planning_then_lists_each_wait_change() { + let test_cases = [ + WaitChangesTestCase { + description: "a job gaining an end node", + previous: Some(&[("job 'nightly_job'", &["model 'orders'"])]), + current: &[("job 'nightly_job'", &["model 'orders'", "model 'items'"])], + expected_lines: &["report now also waits on model 'items' through job 'nightly_job'"], + }, + WaitChangesTestCase { + description: "a job losing an end node", + previous: Some(&[("job 'nightly_job'", &["model 'orders'", "model 'items'"])]), + current: &[("job 'nightly_job'", &["model 'orders'"])], + expected_lines: &["report no longer waits on model 'items' through job 'nightly_job'"], + }, + WaitChangesTestCase { + description: "unchanged end nodes", + previous: Some(&[("tag 'finance'", &["model 'orders'"])]), + current: &[("tag 'finance'", &["model 'orders'"])], + expected_lines: &[], + }, + WaitChangesTestCase { + description: "a build that recorded no sets", + previous: None, + current: &[("tag 'finance'", &["model 'orders'"])], + expected_lines: &[], + }, + ]; + for test_case in test_cases { + let previous: String = test_case + .previous + .map_or_else(|| BASE.to_owned(), |sets| recorded(BASE, sets)); + assert_eq!( + wait_changes( + "report", + Some(&previous), + Some(&recorded(BASE, test_case.current)) + ), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_batch_sql.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_batch_sql.rs new file mode 100644 index 0000000000..97b31e537f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_batch_sql.rs @@ -0,0 +1,27 @@ +//! One `UNION ALL` statement reading the bounds of several physical cursor reads. + +use crate::planner_warehouse::models::PhysicalCursorQuery; + +/// One `(_read, _min, _max)` row per query; a missing minimum repeats the maximum, not `NULL`. +pub fn cursor_batch_sql( + queries: &[&PhysicalCursorQuery], + as_text: &dyn Fn(&PhysicalCursorQuery, &str) -> String, +) -> String { + queries + .iter() + .enumerate() + .map(|(read, query)| { + let maximum: String = as_text(query, &format!("MAX({})", query.cursor_column)); + let minimum: String = if query.min_tags.is_empty() { + maximum.clone() + } else { + as_text(query, &format!("MIN({})", query.cursor_column)) + }; + format!( + "SELECT {read} AS _read, {minimum} AS _min, {maximum} AS _max FROM {}", + query.relation + ) + }) + .collect::>() + .join("\nUNION ALL\n") +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs index 4cd8559b89..22e9b87248 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs @@ -1,5 +1,6 @@ //! Entry points of warehouse-inspection planning. +pub mod cursor_batch_sql; pub mod cursor_bounds; pub mod cursor_query_groups; pub mod cursor_query_identity; diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs index df78d914d7..a614f5b3bf 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs @@ -1,8 +1,10 @@ //! Logical cursor bounds share one physical read per relation column and type. +use crate::planner_warehouse::main::cursor_batch_sql::cursor_batch_sql; use crate::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use crate::planner_warehouse::main::cursor_query_identity::cursor_query_identity; use crate::planner_warehouse::main::cursor_query_sql::cursor_query_sql; +use crate::planner_warehouse::models::PhysicalCursorQuery; use crate::planner_warehouse::tests::helpers::request; use crate::planner_warehouse::tests::test_types::CursorGroupTestCase; @@ -37,16 +39,26 @@ fn given_logical_bounds_when_grouping_then_reads_each_relation_once_in_first_ord "(1/2): analytics.orders.updated_at [max]", "(2/2): raw.events.updated_at [min,max]", ], + expected_batch_sql: "SELECT 0 AS _read, CAST(MAX(updated_at) AS VARCHAR) AS _min, CAST(MAX(updated_at) AS VARCHAR) AS _max FROM analytics.orders\nUNION ALL\nSELECT 1 AS _read, CAST(MIN(updated_at) AS VARCHAR) AS _min, CAST(MAX(updated_at) AS VARCHAR) AS _max FROM raw.events", }, CursorGroupTestCase { description: "no requests read nothing", requests: Vec::new(), expected_sql: Vec::new(), expected_identities: Vec::new(), + expected_batch_sql: "", }, ]; for test_case in test_cases { let groups = cursor_query_groups(&test_case.requests); + let batch: Vec<&PhysicalCursorQuery> = groups.iter().collect(); + let batch_sql: String = + cursor_batch_sql(&batch, &|_query, bound| format!("CAST({bound} AS VARCHAR)")); + assert_eq!( + batch_sql, test_case.expected_batch_sql, + "{}", + test_case.description + ); let sql: Vec = groups .iter() .map(|group| cursor_query_sql(group, &|bound| format!("CAST({bound} AS VARCHAR)"))) diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs index f4ae048082..332339cdc2 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs @@ -5,6 +5,7 @@ pub(super) struct CursorGroupTestCase { pub requests: Vec, pub expected_sql: Vec<&'static str>, pub expected_identities: Vec<&'static str>, + pub expected_batch_sql: &'static str, } pub(super) struct RetentionDirectionTestCase { diff --git a/crates/sqlbuild-analysis/src/source_drift/_helpers/mod.rs b/crates/sqlbuild-analysis/src/source_drift/_helpers/mod.rs new file mode 100644 index 0000000000..208de6bb67 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod walk; diff --git a/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs b/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs new file mode 100644 index 0000000000..04078968dd --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs @@ -0,0 +1,340 @@ +//! The walk deciding whether new columns on some relations could change a query's result. + +use std::collections::HashSet; + +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlquack_syntax::expressions::queries::models::{JoinKind, Select, With}; +use sqlquack_syntax::{Expression, ExpressionWalk}; + +use crate::source_drift::constants::{COUNT_FUNCTION, OPAQUE_KINDS, SHARED_SCOPE_RELATIONS}; + +/// The relations that gained columns and the names they gained, casefolded. +pub(crate) struct Drift<'a> { + pub(crate) relations: &'a HashSet, + pub(crate) added: &'a HashSet, +} + +/// One query scope's visible relations: whether each exposes a drifted relation's columns. +type Visible = Vec; + +/// CTE names in scope and whether each exposes a drifted relation's columns, innermost last. +type Ctes = Vec<(String, bool)>; + +pub(crate) struct Walk<'a> { + drift: Drift<'a>, + /// Set once anything shows the result could change. + pub(crate) affected: bool, +} + +impl<'a> Walk<'a> { + pub(crate) fn new(drift: Drift<'a>) -> Self { + Self { + drift, + affected: false, + } + } + + /// Mark `statement` affected when new columns could change its result. + pub(crate) fn statement(&mut self, statement: &Expression) { + self.names(statement); + if self.query(statement, &Vec::new(), &Vec::new()) { + self.affected = true; + } + } + + /// Any construct whose meaning depends on every column, and any name a new column could take. + fn names(&mut self, statement: &Expression) { + for node in statement.dfs() { + let name: Option<&str> = match node { + Expression::Column(column) => Some(column.name.name.as_str()), + Expression::Identifier(identifier) => Some(identifier.name.as_str()), + Expression::Pivot(_) + | Expression::Unpivot(_) + | Expression::PipeOperator(_) + | Expression::With(_) + | Expression::Cte(_) => { + self.affected = true; + None + } + _ if OPAQUE_KINDS.contains(&node.variant_name()) => { + self.affected = true; + None + } + _ => None, + }; + if name.is_some_and(|name| self.drift.added.contains(&python_casefold(name))) { + self.affected = true; + } + } + } + + /// Whether `query`'s output columns include a drifted relation's columns, walking its scopes. + fn query(&mut self, query: &Expression, ctes: &Ctes, outer: &Visible) -> bool { + match query { + Expression::Select(select) => self.select((query, select), ctes, outer), + Expression::Union(union) => self.set_operation( + (&union.left, &union.right), + union.with.as_ref(), + ctes, + outer, + ), + Expression::Intersect(intersect) => self.set_operation( + (&intersect.left, &intersect.right), + intersect.with.as_ref(), + ctes, + outer, + ), + Expression::Except(except) => self.set_operation( + (&except.left, &except.right), + except.with.as_ref(), + ctes, + outer, + ), + Expression::Subquery(subquery) => self.query(&subquery.this, ctes, outer), + Expression::Paren(paren) => self.query(&paren.this, ctes, outer), + other => { + let mut exposes: bool = false; + for nested in nested_queries(other) { + exposes |= self.query(nested, ctes, outer); + } + exposes + } + } + } + + /// A set operation pairs columns by position, so a branch exposing new columns changes it. + fn set_operation( + &mut self, + (left, right): (&Expression, &Expression), + with: Option<&With>, + ctes: &Ctes, + outer: &Visible, + ) -> bool { + let ctes: Ctes = self.with(with, ctes, outer); + let exposes: bool = self.query(left, &ctes, outer) | self.query(right, &ctes, outer); + if exposes { + self.affected = true; + } + exposes + } + + fn with(&mut self, with: Option<&With>, ctes: &Ctes, outer: &Visible) -> Ctes { + let mut scoped: Ctes = ctes.clone(); + for cte in with.map(|with| with.ctes.as_slice()).unwrap_or_default() { + let exposes: bool = self.query(&cte.this, &scoped, outer); + scoped.push((python_casefold(&cte.alias.name), exposes)); + } + scoped + } + + fn select( + &mut self, + (node, select): (&Expression, &Select), + ctes: &Ctes, + outer: &Visible, + ) -> bool { + let ctes: Ctes = self.with(select.with.as_ref(), ctes, outer); + let mut walked: HashSet<*const Expression> = select + .with + .iter() + .flat_map(|with| with.ctes.iter()) + .map(|cte| std::ptr::from_ref(&cte.this)) + .collect(); + let mut local: Vec<(Option, bool)> = Vec::new(); + let relations = select + .from + .iter() + .flat_map(|from| from.expressions.iter()) + .chain(select.joins.iter().map(|join| &join.this)); + for relation in relations { + walked.insert(std::ptr::from_ref(relation)); + local.extend(self.relation(relation, &ctes, outer)); + } + let drifted_here: bool = local.iter().any(|(_, exposes)| *exposes); + for join in &select.joins { + let natural: bool = matches!( + join.kind, + JoinKind::Natural + | JoinKind::NaturalLeft + | JoinKind::NaturalRight + | JoinKind::NaturalFull + ); + let using_added: bool = join + .using + .iter() + .any(|name| self.drift.added.contains(&python_casefold(&name.name))); + if (natural && drifted_here) || using_added { + self.affected = true; + } + } + let visible: Visible = outer + .iter() + .copied() + .chain(local.iter().map(|(_, exposes)| *exposes)) + .collect(); + let shared: bool = + visible.len() >= SHARED_SCOPE_RELATIONS && visible.iter().any(|exposes| *exposes); + let mut star: bool = false; + let mut unqualified: bool = false; + let mut nested: Vec<&Expression> = Vec::new(); + let mut pending: Vec<(&Expression, bool)> = Vec::new(); + for item in &select.expressions { + pending.push((item, true)); + } + let projected: HashSet<*const Expression> = + select.expressions.iter().map(std::ptr::from_ref).collect(); + for child in node.children() { + if !walked.contains(&std::ptr::from_ref(child)) + && !projected.contains(&std::ptr::from_ref(child)) + { + pending.push((child, false)); + } + } + while let Some((node, in_projection)) = pending.pop() { + match node { + Expression::Select(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) + | Expression::Subquery(_) => nested.push(node), + Expression::Star(expanded) if in_projection => { + star |= match &expanded.table { + None => drifted_here, + Some(qualifier) => { + let qualifier: String = python_casefold(&qualifier.name); + local + .iter() + .find(|(alias, _)| alias.as_deref() == Some(qualifier.as_str())) + .is_none_or(|(_, exposes)| *exposes) + } + }; + } + Expression::BracedWildcard(_) if in_projection => star |= drifted_here, + Expression::Count(_) + | Expression::AggregateFunction(_) + | Expression::Function(_) + if counts_rows(node) => + { + pending.extend( + node.children() + .into_iter() + .filter(|child| !matches!(child, Expression::Star(_))) + .map(|child| (child, in_projection)), + ); + } + Expression::Column(column) if column.table.is_none() => unqualified = true, + _ => pending.extend( + node.children() + .into_iter() + .map(|child| (child, in_projection)), + ), + } + } + if shared && unqualified { + self.affected = true; + } + for query in nested { + if self.query(query, &ctes, &visible) { + self.affected = true; + } + } + star + } + + /// Whether each relation a `FROM` item brings into scope exposes a drifted relation's columns. + fn relation( + &mut self, + relation: &Expression, + ctes: &Ctes, + outer: &Visible, + ) -> Vec<(Option, bool)> { + match relation { + Expression::Table(table) => { + let alias: String = + python_casefold(&table.alias.as_ref().unwrap_or(&table.name).name); + let name: String = python_casefold(&table.name.name); + let qualified: String = python_casefold( + &[&table.catalog, &table.schema] + .into_iter() + .flatten() + .map(|part| part.name.as_str()) + .chain([table.name.name.as_str()]) + .collect::>() + .join("."), + ); + let unqualified: bool = table.schema.is_none() && table.catalog.is_none(); + let cte: Option = unqualified + .then(|| { + ctes.iter() + .rev() + .find(|(cte, _)| *cte == name) + .map(|(_, exposes)| *exposes) + }) + .flatten(); + vec![( + Some(alias), + cte.unwrap_or_else(|| { + self.drift.relations.contains(&name) + || self.drift.relations.contains(&qualified) + }), + )] + } + Expression::Subquery(subquery) => vec![( + subquery + .alias + .as_ref() + .map(|alias| python_casefold(&alias.name)), + self.query(&subquery.this, ctes, outer), + )], + other => { + let mut found: Vec<(Option, bool)> = Vec::new(); + let mut pending: Vec<&Expression> = other.children(); + while let Some(node) = pending.pop() { + match node { + Expression::Table(_) | Expression::Subquery(_) => { + found.extend(self.relation(node, ctes, outer)); + } + Expression::Select(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) => { + found.push((None, self.query(node, ctes, outer))) + } + _ => pending.extend(node.children()), + } + } + if found.is_empty() { + found.push((None, false)); + } + found + } + } + } +} + +/// Whether `node` is `COUNT`, whose `*` counts rows rather than listing columns. +fn counts_rows(node: &Expression) -> bool { + match node { + Expression::Count(_) => true, + Expression::AggregateFunction(call) => call.name.eq_ignore_ascii_case(COUNT_FUNCTION), + Expression::Function(call) => call.name.eq_ignore_ascii_case(COUNT_FUNCTION), + _ => false, + } +} + +/// The outermost queries nested anywhere inside `expression`. +fn nested_queries(expression: &Expression) -> Vec<&Expression> { + let mut found: Vec<&Expression> = Vec::new(); + let mut pending: Vec<&Expression> = expression.children(); + while let Some(node) = pending.pop() { + match node { + Expression::Select(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) + | Expression::Subquery(_) => found.push(node), + _ => pending.extend(node.children()), + } + } + found +} diff --git a/crates/sqlbuild-analysis/src/source_drift/constants.rs b/crates/sqlbuild-analysis/src/source_drift/constants.rs new file mode 100644 index 0000000000..7ca9ea23c5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/constants.rs @@ -0,0 +1,8 @@ +//! Constants of the added-column walk. + +/// Node kinds the parser keeps as unparsed text, whose columns the walk cannot see. +pub(crate) const OPAQUE_KINDS: [&str; 2] = ["command", "raw"]; +/// The aggregate whose `*` argument counts rows. +pub(crate) const COUNT_FUNCTION: &str = "COUNT"; +/// Relations in one scope from which an unqualified name could bind to either of two. +pub(crate) const SHARED_SCOPE_RELATIONS: usize = 2; diff --git a/crates/sqlbuild-analysis/src/source_drift/main/mod.rs b/crates/sqlbuild-analysis/src/source_drift/main/mod.rs new file mode 100644 index 0000000000..5daf588a3c --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/main/mod.rs @@ -0,0 +1 @@ +pub mod reads_added_columns; diff --git a/crates/sqlbuild-analysis/src/source_drift/main/reads_added_columns.rs b/crates/sqlbuild-analysis/src/source_drift/main/reads_added_columns.rs new file mode 100644 index 0000000000..6fdbf7c388 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/main/reads_added_columns.rs @@ -0,0 +1,44 @@ +//! `reads_added_columns`: whether a model must be analysed again when its inputs gain columns. + +use std::collections::HashSet; + +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlquack_syntax::Parsed; + +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; +use crate::source_drift::_helpers::walk::{Drift, Walk}; +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; + +/// Whether `query_sql` could return something else once the `drifted` relations gain `added`. +#[must_use] +pub fn reads_added_columns( + dialect: &str, + (query_sql, placeholders): (&str, &[(String, String)]), + drifted: &[String], + added: &[String], +) -> bool { + let AnalysisRoute::Native(native) = analysis_route(Some(dialect)) else { + return true; + }; + let Ok(normalized) = normalize_native_sql(query_sql, native.name(), native.stub_prefix()) + else { + return true; + }; + let sql: String = placeholder_defaults(&normalized, placeholders); + let Ok(Parsed { statements, .. }) = native.parse(&sql) else { + return true; + }; + let [statement] = statements.as_slice() else { + return true; + }; + let relations: HashSet = drifted.iter().map(|name| python_casefold(name)).collect(); + let added: HashSet = added.iter().map(|name| python_casefold(name)).collect(); + let mut walk: Walk<'_> = Walk::new(Drift { + relations: &relations, + added: &added, + }); + walk.statement(statement); + walk.affected +} diff --git a/crates/sqlbuild-analysis/src/source_drift/mod.rs b/crates/sqlbuild-analysis/src/source_drift/mod.rs new file mode 100644 index 0000000000..f484bfc4da --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/mod.rs @@ -0,0 +1,8 @@ +//! Whether columns a warehouse added to a model's inputs could change what the model returns. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/source_drift/tests.rs b/crates/sqlbuild-analysis/src/source_drift/tests.rs new file mode 100644 index 0000000000..3958450839 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_reads_added_columns.rs"] +mod test_reads_added_columns; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs b/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs new file mode 100644 index 0000000000..df91ab1e75 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs @@ -0,0 +1,104 @@ +use crate::source_drift::main::reads_added_columns::reads_added_columns; +use crate::source_drift::tests::test_types::AddedColumnsTestCase; + +#[test] +fn given_a_source_gaining_columns_when_reading_models_then_only_column_list_readers_are_affected() { + let test_cases = [ + AddedColumnsTestCase { + description: "a star over the source", + sql: r#"SELECT * FROM __source("orders")"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "explicit columns from the source alone", + sql: r#"SELECT order_id, status FROM __source("orders")"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "an unqualified name in a join with the source", + sql: r#"SELECT order_id FROM __source("orders") AS o + JOIN __ref("customers") AS c ON o.customer_id = c.customer_id"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "qualified names in a join with the source", + sql: r#"SELECT o.order_id, c.customer_name FROM __source("orders") AS o + JOIN __ref("customers") AS c ON o.customer_id = c.customer_id"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "a star reaching the output through a CTE", + sql: r#"WITH s AS (SELECT * FROM __source("orders")) SELECT * FROM s"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a star through a subquery", + sql: r#"SELECT * FROM (SELECT * FROM __source("orders")) AS s"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a starred CTE read by explicit columns", + sql: r#"WITH source_rows AS (SELECT * FROM __source("orders")), + final AS (SELECT s.order_id, s.status FROM source_rows AS s) + SELECT * FROM final"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "a starred CTE joined with an unqualified name", + sql: r#"WITH s AS (SELECT * FROM __source("orders")) + SELECT customer_name FROM s JOIN __ref("customers") AS c ON s.customer_id = c.customer_id"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a set operation with a starred branch", + sql: r#"SELECT * FROM __source("orders") UNION ALL SELECT * FROM __source("orders")"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a name the source gains, qualified", + sql: r#"SELECT o.order_id FROM __source("orders") AS o WHERE o.region = 'EU'"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "an unqualified name a correlated subquery may now bind to the source", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c + WHERE EXISTS (SELECT 1 FROM __source("orders") AS o WHERE o.customer_id = customer_id)"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "a count of every row", + sql: r#"SELECT COUNT(*) AS orders FROM __source("orders")"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "a star over another relation in a join with the source", + sql: r#"SELECT c.*, o.order_id FROM __ref("customers") AS c + JOIN __source("orders") AS o ON o.customer_id = c.customer_id"#, + expected_affected: false, + }, + AddedColumnsTestCase { + description: "a star over the source in a join", + sql: r#"SELECT o.*, c.customer_name FROM __ref("customers") AS c + JOIN __source("orders") AS o ON o.customer_id = c.customer_id"#, + expected_affected: true, + }, + AddedColumnsTestCase { + description: "SQL the parser cannot read", + sql: r#"SELECT FROM WHERE __source("orders")"#, + expected_affected: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + reads_added_columns( + "snowflake", + (test_case.sql, &[]), + &["orders".to_owned()], + &["region".to_owned()], + ), + test_case.expected_affected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs b/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs new file mode 100644 index 0000000000..daf66cf1a3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs @@ -0,0 +1,6 @@ +/// A model query, and whether `orders` gaining a `region` column could change its result. +pub(super) struct AddedColumnsTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_affected: bool, +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs index 34ddbb32df..01f77b06dc 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs @@ -1,6 +1,8 @@ //! Expression macro calls followed through the edits that move them, and their typed markers. use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex, MutexGuard, PoisonError}; use sqlquack_syntax::expressions::tree::constants::{ TYPED_EXPRESSION_FUNCTION, TYPED_LIKE_EXPRESSION_FUNCTION, @@ -12,9 +14,16 @@ use crate::analysis_route::models::AnalysisRoute; use crate::analysis_route::types::NativeDialect; use crate::compiled_project::models::SplicedSpan; use crate::sql_macros::_helpers::parsing::body::with_line_comment_closed; -use crate::sql_macros::constants::NULL_KEYWORD; +use crate::sql_macros::constants::{MARKER_MEMO_ENTRIES, NULL_KEYWORD}; use crate::sql_macros::models::{TextSplice, TypedCall}; +/// The dialect, SQL and calls a parse-verified marker set was chosen for. +type MarkerKey = (String, String, Vec); + +/// Parse-verified marker sets, as a compile reads each analysis text twice and processes recompile. +static MARKER_MEMO: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + /// `calls` moved `offset` bytes later. pub(crate) fn shifted(calls: Vec, offset: usize) -> impl Iterator { calls.into_iter().map(move |call| TypedCall { @@ -176,8 +185,32 @@ pub(crate) fn markable_calls<'c>( let AnalysisRoute::Native(parse_dialect) = analysis_route(Some(dialect)) else { return Cow::Owned(Vec::new()); }; + let key: MarkerKey = (dialect.to_owned(), sql.to_owned(), calls.to_vec()); + if let Some(kept) = marker_memo().get(&key) { + return Cow::Owned(kept.clone()); + } + let kept: Vec = parsed_markable_calls(sql, calls, (parse_dialect, marked)); + let mut memo = marker_memo(); + if memo.len() >= MARKER_MEMO_ENTRIES { + memo.clear(); + } + memo.insert(key, kept.clone()); + Cow::Owned(kept) +} + +/// The process's parse-verified marker sets, locked. +fn marker_memo() -> MutexGuard<'static, HashMap>> { + MARKER_MEMO.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// The calls whose markers parse as typed expressions, checked by parsing the marked SQL. +fn parsed_markable_calls( + sql: &str, + calls: &[TypedCall], + (parse_dialect, marked): (&'static dyn NativeDialect, usize), +) -> Vec { if lowered_markers(&typed_sql(sql, calls).0, parse_dialect) == Some(marked) { - return Cow::Borrowed(calls); + return calls.to_vec(); } let mut kept: Vec = Vec::with_capacity(calls.len()); let mut kept_marked: usize = 0; @@ -193,7 +226,7 @@ pub(crate) fn markable_calls<'c>( kept.pop(); } } - Cow::Owned(kept) + kept } /// Whether `typed_sql` wraps `call` in a marker. diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index 5d51555fc9..29e6b8dd67 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -145,3 +145,5 @@ pub(crate) const BIGQUERY_BIGNUMERIC_LIMITS: (u32, u32) = (38, 38); pub(crate) const BIGQUERY_BIG_DECIMAL_NAMES: [&str; 2] = ["BIGNUMERIC", "BIGDECIMAL"]; /// How a relation function such as `__ref("orders")` starts. pub(crate) const RELATION_FUNCTION_PREFIX: &str = "__"; +/// Parse-verified marker sets a process keeps before starting over; each holds one model's SQL. +pub(crate) const MARKER_MEMO_ENTRIES: usize = 8192; diff --git a/crates/sqlbuild-analysis/src/sql_macros/models.rs b/crates/sqlbuild-analysis/src/sql_macros/models.rs index 5657995e5c..5eaafa0a41 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/models.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/models.rs @@ -303,7 +303,7 @@ pub struct SlotCapture { } /// One expression macro call's expansion in a text: its byte range and the macro's `returns`. -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct TypedCall { pub start: usize, pub end: usize, diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs index 5f86bfd1f5..7ac928966d 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -196,6 +196,12 @@ fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns .collect(); let (typed, insertions) = typed_analysis_sql(test_case.sql, &calls, "duckdb"); assert_eq!(typed, test_case.expected_sql, "{}", test_case.description); + let (remembered, _) = typed_analysis_sql(test_case.sql, &calls, "duckdb"); + assert_eq!( + remembered, test_case.expected_sql, + "{}", + test_case.description + ); let removed: String = { let mut text: String = String::new(); let mut cursor: usize = 0; diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index e89e7360e8..0a0e5235bd 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1835,6 +1835,20 @@ "dest": "defer_sources_to", "group": 1 }, + { + "kind": "store", + "options": [ + "--plan-from" + ], + "dest": "plan_from", + "choices": [ + "live", + "synced_state" + ], + "default": null, + "help": "read warehouse state live, or from the target's synced copy checked against the warehouse first (default: the target's plan_from, else live)", + "group": 1 + }, { "kind": "store", "options": [ @@ -2277,6 +2291,20 @@ "dest": "defer_sources_to", "group": 1 }, + { + "kind": "store", + "options": [ + "--plan-from" + ], + "dest": "plan_from", + "choices": [ + "live", + "synced_state" + ], + "default": null, + "help": "read warehouse state live, or from the target's synced copy checked against the warehouse first (default: the target's plan_from, else live)", + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 88339e681a..991c78bae2 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -77,7 +77,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 191] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 192] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -141,6 +141,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 191] = [ ("as_target", NamespaceDefault::None), ("include_python", NamespaceDefault::Bool(true)), ("load_sources", NamespaceDefault::None), + ("plan_from", NamespaceDefault::None), ("reload", NamespaceDefault::Bool(false)), ("run_tests", NamespaceDefault::Bool(true)), ("run_audits", NamespaceDefault::Bool(true)), diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index c136da4699..8e3f224b8a 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -49,6 +49,18 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( attribute: "reach", expected_value: Value::Str("all".to_owned()), }, + ParsedValueTestCase { + description: "--plan-from names where a plan reads warehouse state", + argv: &["plan", "--plan-from", "synced_state"], + attribute: "plan_from", + expected_value: Value::Str("synced_state".to_owned()), + }, + ParsedValueTestCase { + description: "an unset --plan-from leaves the source to the target", + argv: &["build"], + attribute: "plan_from", + expected_value: Value::None, + }, ParsedValueTestCase { description: "an unset concurrency stays None", argv: &["build"], @@ -149,6 +161,12 @@ fn given_help_version_or_rejected_arguments_when_parsing_then_exits_after_writin expected_exit: 2, expected_last_write: "sqb build: error[C900]: argument --concurrency: invalid int value: 'x1'\n\n", }, + ExitTestCase { + description: "--plan-from names only live or synced_state", + argv: &["build", "--plan-from", "cache"], + expected_exit: 2, + expected_last_write: "sqb build: error[C900]: argument --plan-from: invalid choice: 'cache' (choose from 'live', 'synced_state')\n\n", + }, ExitTestCase { description: "an ambiguous abbreviation lists its matches", argv: &["build", "--allow-s"], diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 008107a2f0..2910f50607 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -110,7 +110,7 @@ pub(crate) const DAG_ROUTE: Route = routed( }, ); -const PLAN_FIELDS: [Field; 24] = [ +const PLAN_FIELDS: [Field; 25] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -135,6 +135,7 @@ const PLAN_FIELDS: [Field; 24] = [ same("selection_diagnostics"), same("timings"), same("ignore_audit_failures"), + same("plan_from"), ]; const PLAN_ARGUMENTS: [Field; 1] = [request(class("PlanCommandRequest"), &PLAN_FIELDS)]; pub(crate) const PLAN_ROUTE: Route = routed( @@ -144,13 +145,14 @@ pub(crate) const PLAN_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const BUILD_FIELDS: [Field; 39] = [ +const BUILD_FIELDS: [Field; 40] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), same("defer_to"), same("defer_clone_from"), same("defer_sources_to"), + same("plan_from"), SELECTED_TARGET, CURSOR_OVERRIDES, NO_COLOR, diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index a11bd722b1..683223e6dd 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -231,4 +231,11 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ decided_by: &["ignore_audit_failures"], }, }, + TargetField { + group: None, + name: "plan_from", + rule: MergeRule::Overlay { + decided_by: &["plan_from"], + }, + }, ]; diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index c83f1f4cc7..e520bb835e 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -5,9 +5,9 @@ use crate::project::_helpers::defaults::{ InheritableField, optional_retention_policy, optional_table_type, }; use crate::project::_helpers::fields::{ - bool_or_none, coerce_mapping, get, key_name, keyed, object, optional_nullable_bool, - optional_str, optional_strict_int, optional_text_field, string_mapping, text_or_none, - value_or_none, + bool_or_none, coerce_mapping, get, key_name, keyed, object, optional_choice, + optional_nullable_bool, optional_str, optional_strict_int, optional_text_field, string_mapping, + text_or_none, value_or_none, }; use crate::project::_helpers::keys::{ Place, reject_removed_virtual_keys, reject_unknown_mapping_keys, validate_allowed_keys, @@ -16,8 +16,8 @@ use crate::project::_helpers::sections::{fixed_duration_seconds, item_timeout}; use crate::project::_helpers::values::{int_below, is_int}; use crate::project::constants::{ CLONE_KEYS, CONCURRENCY_KEY, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, - IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, POOLS_KEY, RETENTION_MATERIALIZATIONS, - TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, + IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PLAN_FROM_KEY, PLAN_FROM_SOURCES, POOLS_KEY, + RETENTION_MATERIALIZATIONS, TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; @@ -432,9 +432,23 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded Loaded { + Ok(match get(mapping, PLAN_FROM_KEY) { + None => text_or_none(None), + Some(_) => text_or_none(Some(optional_choice( + mapping, + PLAN_FROM_KEY, + &PLAN_FROM_SOURCES, + PLAN_FROM_SOURCES[0], + )?)), + }) +} + /// `_load_local_targets`: each target's fields by name. pub(crate) fn load_target_fields( payload: Option<&ConfigValue>, diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index a6ad43ab52..654fcadcbf 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -195,7 +195,11 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 74] = [ pub(crate) const POOLS_KEY: &str = "pools"; /// The highest limit a pool may declare. pub(crate) const MAX_POOL_LIMIT: i64 = 10_000; -pub(crate) const TARGET_KEYS: [&str; 23] = [ +/// Where a target's plans read warehouse state: live, or a synced copy checked first. +pub(crate) const PLAN_FROM_KEY: &str = "plan_from"; +/// What `plan_from` may name; `live` is the default. +pub(crate) const PLAN_FROM_SOURCES: [&str; 2] = ["live", "synced_state"]; +pub(crate) const TARGET_KEYS: [&str; 24] = [ "connection", "catalog_connection", "vars", @@ -219,6 +223,7 @@ pub(crate) const TARGET_KEYS: [&str; 23] = [ "concurrency", POOLS_KEY, IGNORE_AUDIT_FAILURES_KEY, + PLAN_FROM_KEY, ]; pub(crate) const CONTROL_KEYS: [&str; 3] = ["location", "server", "sync_at_least_every"]; pub(crate) const CHANGES_KEYS: [&str; 2] = ["counts", "reach"]; diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 7cb96749fb..8c8a3fc77a 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -40,6 +40,28 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { "Expected 'catalog_connection' to be a non-empty string when provided", )), }, + ProjectFailureTestCase { + description: "a target may plan from its synced state", + project: "name = 'orders'\nadapter = 'duckdb'\n[targets.dev]\nplan_from = 'synced_state'\n", + local: "[targets.dev]\nplan_from = 'live'\n", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "plan_from names only live or synced_state", + project: "name = 'orders'\nadapter = 'duckdb'\n[targets.dev]\nplan_from = 'cache'\n", + local: "", + expected_failure: Some(ConfigFailure::plain( + "Expected 'plan_from' to be one of: live, synced_state", + )), + }, + ProjectFailureTestCase { + description: "a local target's plan_from names only live or synced_state", + project: "name = 'orders'\nadapter = 'duckdb'\n", + local: "[targets.dev]\nplan_from = 'cache'\n", + expected_failure: Some(ConfigFailure::plain( + "Expected 'plan_from' to be one of: live, synced_state", + )), + }, ProjectFailureTestCase { description: "an unknown key suggests the nearest supported key", project: "name = 'orders'\nadapter = 'duckdb'\n[scopes]\nenforce_placment = true\n", diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs index 5af09aa592..3a2ce7ce86 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -252,6 +252,12 @@ pub(crate) fn python_entry_lines( width, true, )]; + lines.extend( + entry + .wait_changes + .iter() + .map(|change| STYLE.muted(&format!(" {change}"))), + ); lines.extend(identity_diff_lines(entry)); lines }) diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 4e45437716..34817c0b7b 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -364,6 +364,8 @@ pub struct PythonPlanText { pub identity_status: String, pub source_diff: Vec, pub dependency_diff: Vec, + /// What the node now also waits on, or no longer, through its job and tag dependencies. + pub wait_changes: Vec, } /// One provider used by a selected Python surface. diff --git a/crates/sqlbuild-host/src/entries/_helpers/adapter.rs b/crates/sqlbuild-host/src/entries/_helpers/adapter.rs index d3cf753392..cdddda6eef 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/adapter.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/adapter.rs @@ -164,6 +164,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::Adapter, }, + HostEntry { + name: "SyncedCatalog", + kind: HostKind::Class, + edge: HostEdge::Adapter, + }, HostEntry { name: "InspectionQueryRecord", kind: HostKind::Class, diff --git a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs index fa4ea13ddc..8e77164e4a 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs @@ -234,6 +234,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::UserCode, }, + HostEntry { + name: "NodeSetRef", + kind: HostKind::Class, + edge: HostEdge::UserCode, + }, HostEntry { name: "PythonCheckSeverity", kind: HostKind::Class, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 1922abc77d..4c39f96c72 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -94,6 +94,7 @@ pub enum AdapterHost { ResolveRelationLocationQualifiedName, RunBoundedInspections, RunRecordedInspectionQuery, + SyncedCatalog, InspectionQueryRecord, StatementMetadataEffect, ConditionalResultNullability, @@ -174,6 +175,7 @@ pub enum UserCodeHost { AuditCase, RelationRenderer, SqlResourceRef, + NodeSetRef, PythonCheckSeverity, SqlResourceRefKind, LoadCustomRules, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index b1eb03479b..1f753dee9c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -53,3 +53,9 @@ pub(crate) const INGESTION_CHUNK_RECORDS: usize = 200; /// The audit result metadata field marking a failure its build let through. pub(crate) const IGNORED_KEY: &str = "ignored"; +/// The mirrored catalog of one location's target, with when each relation was observed. +pub(crate) const SYNCED_CATALOG_SQL: &str = "SELECT record_json, observed_at FROM sqb_warehouse_catalog \ + WHERE location_id=$1 AND target=$2 ORDER BY state_database, state_schema, event_id"; +/// The development locations of one target, by project path. +pub(crate) const DEVELOPMENT_LOCATIONS_SQL: &str = + "SELECT project_path, location_id FROM sqb_development_locations WHERE target=$1"; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs index e38b47279c..86928a28c0 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs @@ -1,5 +1,7 @@ pub mod preview_state_parts; +pub mod read_development_location; pub mod read_preview_copy; pub mod read_statistics_due; pub mod read_sync_due; +pub mod read_synced_catalog; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs new file mode 100644 index 0000000000..2ee11e3f77 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs @@ -0,0 +1,35 @@ +//! The location `sqb dev` keeps for a project's target, read without creating one. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::DEVELOPMENT_LOCATIONS_SQL; + +/// The development location of `target` whose project path `same_project` accepts, if any. +pub async fn read_development_location( + store: &ControlStore, + target: &str, + same_project: impl Fn(&str) -> bool, +) -> Result, ControlError> { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await?, + DEVELOPMENT_LOCATIONS_SQL, + vec![Parameter::Text(target.into())], + ) + .await?; + commit(transaction).await?; + rows.iter() + .find(|row| row["project_path"].as_str().is_some_and(&same_project)) + .map(|row| { + row["location_id"] + .as_str() + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Schema("invalid development location identity".into())) + }) + .transpose() +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs new file mode 100644 index 0000000000..c3104ec656 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs @@ -0,0 +1,41 @@ +//! A plan from synced state reads the target's mirrored catalog and when each relation was seen. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::SYNCED_CATALOG_SQL; +use crate::warehouse_metadata::models::{SyncedCatalogRelation, WarehouseCatalogRecord}; + +/// Every mirrored relation of the location's target, with the time its catalog scan observed it. +pub async fn read_synced_catalog( + store: &ControlStore, + (location_id, target): (Uuid, &str), +) -> Result, ControlError> { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await?, + SYNCED_CATALOG_SQL, + vec![Parameter::Id(location_id), Parameter::Text(target.into())], + ) + .await?; + commit(transaction).await?; + rows.into_iter() + .map(|mut row| { + let record: WarehouseCatalogRecord = + serde_json::from_value(row.remove("record_json").unwrap_or_default()).map_err( + |error| ControlError::Invalid(format!("invalid mirrored relation: {error}")), + )?; + let observed_at = serde_json::from_value(row.remove("observed_at").unwrap_or_default()) + .map_err(|error| { + ControlError::Invalid(format!("invalid mirrored observation time: {error}")) + })?; + Ok(SyncedCatalogRelation { + record, + observed_at, + }) + }) + .collect() +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 7a40c9a009..fc24cb88a2 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -738,6 +738,13 @@ pub struct WarehouseCatalogColumn { pub data_type: String, } +/// One mirrored relation and the time the catalog scan that recorded it observed it. +#[derive(Clone, Debug, PartialEq)] +pub struct SyncedCatalogRelation { + pub record: WarehouseCatalogRecord, + pub observed_at: DateTime, +} + /// Physical identity and ordered columns come from the adapter catalog, not fingerprints. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index 530161c02c..ca785c24e1 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -407,6 +407,7 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { FieldDefault::NativeCall(RecordId::SqlTestSelection), ), FieldSpec::new("source_lexical_syntax", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), ], }; const R_PYTHON_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index 86db23df0b..75fe5447c4 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -150,6 +150,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), ], }; const R_TARGET_SETTINGS: RecordSpec = RecordSpec { @@ -208,6 +209,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index f7f2d5cde5..ca6de97cfa 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -238,6 +238,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("defer_to", FieldDefault::None), FieldSpec::new("defer_clone_from", FieldDefault::None), FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("cursor_overrides", FieldDefault::None), FieldSpec::new("no_color", FieldDefault::False), @@ -560,6 +561,7 @@ const PLAN_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("no_sql_validation", FieldDefault::False), FieldSpec::new("defer_to", FieldDefault::None), FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("plan_from", FieldDefault::None), FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("as_target", FieldDefault::None), FieldSpec::new("cursor_overrides", FieldDefault::None), diff --git a/crates/sqlbuild-python-base/src/types.rs b/crates/sqlbuild-python-base/src/types.rs index d7009884c7..97b617f0fd 100644 --- a/crates/sqlbuild-python-base/src/types.rs +++ b/crates/sqlbuild-python-base/src/types.rs @@ -17,3 +17,9 @@ pub trait FromPositional: Sized { pub trait IntoPositional { fn into_positional<'py>(self, py: Python<'py>) -> PyResult>; } + +/// Each Python node's resolved job and tag dependencies, or why they did not resolve. +pub type NodeSetOutcome = Result< + Vec, + sqlbuild_analysis::graph::errors::SelectorError, +>; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs index 8fcac2dff0..5af92903d1 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs @@ -12,6 +12,7 @@ use pyo3::pyfunction; use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; +use sqlbuild_python_project::graph::main::python_nodes::resolve_node_sets::resolve_node_sets; use crate::compile_driver::_helpers::setup::target_defaults::TargetDefaults; use crate::compile_driver::_helpers::setup::target_defaults::apply_target_defaults; @@ -125,10 +126,12 @@ pub(crate) fn compiled_project( py: Python<'_>, request: ProjectRequest, ) -> PyResult> { - produced( + let project: Bound<'_, PyAny> = produced( FrontierStage::CompiledProject, python_compiled_project(py, &request)?, - ) + )?; + let _ = resolve_node_sets(request.discovered_inputs.bind(py), &project)?; + Ok(project) } /// `_build_python_compiled_project`. diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs index e1c0c868e3..436c151826 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs @@ -36,6 +36,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; @@ -367,6 +368,17 @@ fn macro_file<'py>( context.record(RecordId::DiscoveredMacroFile, &fields) } +/// The discovery session `discovered` inputs were read in, which keeps their resolved set dependencies. +pub(crate) fn discovered_session<'py>( + discovered: &Bound<'py, PyAny>, +) -> PyResult>> { + let session: Bound<'py, PyAny> = native_field(discovered, "native_session")?; + if session.is_none() { + return Ok(None); + } + Ok(Some(session.cast_into::()?)) +} + /// The session of this pass, created and memoised by the first declaration collection. pub(crate) fn discovery_session<'py>( project_dir: &Bound<'py, PyAny>, @@ -382,6 +394,7 @@ pub(crate) fn discovery_session<'py>( py, NativeDiscoverySession { inner: DiscoverySession::new(project_root(project_dir)?, declaration_file_options()), + node_sets: std::sync::Mutex::new(None), }, )?; memo.set_item(SESSION_MEMO_KEY, &created)?; diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs index 6d3dca266f..0467afb706 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs @@ -42,6 +42,8 @@ pub(crate) struct ModelTextRequest { #[derive(Debug)] pub(crate) struct NativeDiscoverySession { pub(crate) inner: sqlbuild_discovery::declaration_files::models::DiscoverySession, + /// The Python nodes' job and tag dependencies, as the latest compile of these inputs resolved them. + pub(crate) node_sets: std::sync::Mutex>, } /// One in-memory declaration file parse request; `kind` names the collection it belongs to. diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/keep_node_sets.rs b/crates/sqlbuild-python-front/src/discovery/main/records/keep_node_sets.rs new file mode 100644 index 0000000000..c93f30f00e --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/keep_node_sets.rs @@ -0,0 +1,22 @@ +//! Keep a compile's resolved job and tag dependencies with the discovered inputs it compiled. + +use std::sync::PoisonError; + +use crate::discovery::_helpers::records::declaration_records::discovered_session; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::types::NodeSetOutcome; + +/// Keep `outcome` in the discovery session `discovered` was read in, when it has one. +pub fn keep_node_sets( + discovered: &Bound<'_, PyAny>, + outcome: Option, +) -> PyResult<()> { + if let Some(session) = discovered_session(discovered)? { + *session + .get() + .node_sets + .lock() + .unwrap_or_else(PoisonError::into_inner) = outcome; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs index 3e85c0fcd2..87ce8c6245 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/mod.rs @@ -14,6 +14,7 @@ pub mod discover_sql_hook_files; pub mod discover_test_files; pub mod discovered_model_files; pub mod externally_described_models; +pub mod keep_node_sets; pub mod parse_sql_test_file; pub mod projected_text; pub mod register_declaration_records; @@ -22,3 +23,4 @@ pub mod register_model_records; pub mod register_sql_test_records; pub mod register_yaml_records; pub mod retained_discovery_session; +pub mod stored_node_sets; diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/stored_node_sets.rs b/crates/sqlbuild-python-front/src/discovery/main/records/stored_node_sets.rs new file mode 100644 index 0000000000..8c75209723 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/records/stored_node_sets.rs @@ -0,0 +1,19 @@ +//! The job and tag dependencies the latest compile of discovered inputs resolved. + +use std::sync::PoisonError; + +use crate::discovery::_helpers::records::declaration_records::discovered_session; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::types::NodeSetOutcome; + +/// The resolved set dependencies the latest compile of `discovered` kept, if any. +pub fn stored_node_sets(discovered: &Bound<'_, PyAny>) -> PyResult> { + Ok(discovered_session(discovered)?.and_then(|session| { + session + .get() + .node_sets + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + })) +} diff --git a/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs b/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs index ddf86cf94e..f9e3bffab9 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs @@ -19,12 +19,12 @@ pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } -/// Run one cursor statement; a failure leaves only that relation unavailable. -pub(crate) fn read_cursor_query<'py>( +/// Run one cursor statement and record it: `(rows, seconds, error)`, `rows` `None` on failure. +fn run_cursor_statement<'py>( py: Python<'py>, sql: &str, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { +) -> PyResult<(Bound<'py, PyAny>, f64, Bound<'py, PyAny>)> { let started = Instant::now(); let outcome = call_with( execute, @@ -32,19 +32,10 @@ pub(crate) fn read_cursor_query<'py>( ) .and_then(|result| result.call_method0("fetchall")); let elapsed = started.elapsed().as_secs_f64(); - let (row, error, count) = match outcome { + let (rows, error, count) = match outcome { Ok(rows) => { - let count = rows.len()?; - let row = if count > 0 { - rows.get_item(0)? - } else { - py.None().into_bound(py) - }; - ( - row, - py.None().into_bound(py), - count.into_pyobject(py)?.into_any(), - ) + let count = rows.len()?.into_pyobject(py)?.into_any(); + (rows, py.None().into_bound(py), count) } Err(error) if error.is_instance_of::(py) => { let value = error.value(py).clone().into_any(); @@ -67,14 +58,33 @@ pub(crate) fn read_cursor_query<'py>( ], )?; call_with(&host_entry(py, RECORD_QUERY)?, &[("record", record)])?; - Ok(PyTuple::new(py, [row, PyFloat::new(py, elapsed).into_any(), error])?.into_any()) + Ok((rows, elapsed, error)) +} + +/// Run and record one cursor statement: `(rows, seconds, error)`, rows cut to the first if `first_row`. +pub(crate) fn read_cursor_query<'py>( + py: Python<'py>, + sql: &str, + reader: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + first_row: bool, +) -> PyResult> { + let (rows, elapsed, error) = run_cursor_statement(py, sql, reader)?; + let rows = if !first_row { + rows + } else if !rows.is_none() && rows.len()? > 0 { + rows.get_item(0)? + } else { + py.None().into_bound(py) + }; + Ok(PyTuple::new(py, [rows, PyFloat::new(py, elapsed).into_any(), error])?.into_any()) } -/// A bounded-inspection task running one cursor statement. +/// A bounded-inspection task running `read_cursor_query` on one statement. pub(crate) fn bound_task<'py>( py: Python<'py>, sql: String, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + first_row: bool, ) -> PyResult> { let connection: Py = connection.clone().unbind(); let execute: Py = execute.clone().unbind(); @@ -86,7 +96,8 @@ pub(crate) fn bound_task<'py>( _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult> { let py = args.py(); - Ok(read_cursor_query(py, &sql, (connection.bind(py), execute.bind(py)))?.unbind()) + let reader = (connection.bind(py), execute.bind(py)); + Ok(read_cursor_query(py, &sql, reader, first_row)?.unbind()) }, )? .into_any()) diff --git a/crates/sqlbuild-python-planner/src/live_reads/_helpers/cursor_batches.rs b/crates/sqlbuild-python-planner/src/live_reads/_helpers/cursor_batches.rs new file mode 100644 index 0000000000..d36f05f790 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/_helpers/cursor_batches.rs @@ -0,0 +1,80 @@ +//! Cursor reads answered together in batched `UNION ALL` statements. + +use crate::live_reads::_helpers::bounded::bound_task; +use crate::live_reads::_helpers::bounded::run_bounded; +use crate::live_reads::types::CursorBatchOutcomes; +use crate::planner_warehouse::models::CursorBatch; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyFloat, PyTuple}; +use std::collections::HashMap; + +/// Run `batches` at most `concurrency` at a time: each answered read's outcome, and failed batches' reads. +pub(crate) fn run_cursor_batches<'py>( + py: Python<'py>, + batches: &[CursorBatch], + reader: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (concurrency, on_complete): (i64, Option>), +) -> PyResult> { + let tasks: Vec> = batches + .iter() + .map(|batch| bound_task(py, batch.sql.clone(), reader, false)) + .collect::>()?; + let results = run_bounded(py, tasks, concurrency.max(1), on_complete)?; + let mut answered: Vec<(usize, Bound<'py, PyAny>)> = Vec::new(); + let mut failed: Vec = Vec::new(); + for (batch, result) in batches.iter().zip(results) { + let rows: Bound<'py, PyAny> = result.get_item(0)?; + if !result.get_item(2)?.is_none() || rows.is_none() { + failed.extend(batch.reads.iter().map(|(index, _)| *index)); + continue; + } + let elapsed: f64 = result.get_item(1)?.extract()?; + let mut by_read: HashMap> = HashMap::new(); + for row in rows.try_iter()? { + let row: Bound<'py, PyAny> = row?; + let read: usize = read_position(&row.get_item(0)?)?; + by_read.insert(read, row); + } + for (position, (index, has_min)) in batch.reads.iter().enumerate() { + let row: Bound<'py, PyAny> = by_read + .remove(&position) + .unwrap_or_else(|| py.None().into_bound(py)); + let outcome = PyTuple::new( + py, + [ + bound_row(&row, *has_min)?, + PyFloat::new(py, elapsed).into_any(), + py.None().into_bound(py), + ], + )?; + answered.push((*index, outcome.into_any())); + } + } + Ok((answered, failed)) +} + +/// A batch row's `_read` position, which a driver may return as an integer or a decimal. +fn read_position(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(position) = value.extract::() { + return Ok(position); + } + let text: String = value.str()?.extract()?; + text.parse::().map_err(|_| { + PyValueError::new_err(format!("cursor batch row has no read position: {text}")) + }) +} + +/// A batch row `(_read, _min, _max)` as the row its read's own statement returns. +fn bound_row<'py>(row: &Bound<'py, PyAny>, has_min: bool) -> PyResult> { + let py = row.py(); + if row.is_none() { + return Ok(py.None().into_bound(py)); + } + let bounds: Vec> = if has_min { + vec![row.get_item(1)?, row.get_item(2)?] + } else { + vec![row.get_item(2)?] + }; + Ok(PyTuple::new(py, bounds)?.into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs index b3886f7bf0..ac232e5f26 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod bounded; +pub(crate) mod cursor_batches; diff --git a/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs b/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs index 8230420d86..44c7be36bb 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs @@ -10,5 +10,5 @@ pub fn bound_task<'py>( sql: String, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - bounded::bound_task(py, sql, (connection, execute)) + bounded::bound_task(py, sql, (connection, execute), true) } diff --git a/crates/sqlbuild-python-planner/src/live_reads/mod.rs b/crates/sqlbuild-python-planner/src/live_reads/mod.rs index 19d4b07806..3132bb75fc 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/mod.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-planner/src/live_reads/models.rs b/crates/sqlbuild-python-planner/src/live_reads/models.rs index 030c294279..6fb7db6992 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/models.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/models.rs @@ -6,6 +6,7 @@ use crate::live_reads::_helpers::bounded::eligible_task; use crate::live_reads::_helpers::bounded::read_cursor_query; use crate::live_reads::_helpers::bounded::run_bounded; use crate::live_reads::_helpers::bounded::text; +use crate::live_reads::_helpers::cursor_batches::run_cursor_batches; use crate::live_reads::constants::ACTIVE_CATALOG; use crate::live_reads::constants::EXECUTE; use crate::live_reads::constants::INSPECTION_CATALOG; @@ -14,7 +15,8 @@ use crate::live_reads::constants::RELATIONS_MAIN_LIST; use crate::live_reads::main::bound_task::bound_task; use crate::observations::main::capture_source_columns::capture_source_columns; use crate::planner_warehouse::models::{ - AdapterRead, EligibleCursorQuery, FingerprintRead, PlannerReads, Read, StateHistoryRead, + AdapterRead, CursorStatements, EligibleCursorQuery, FingerprintRead, PlannerReads, Read, + StateHistoryRead, }; use crate::planner_warehouse::types::CursorProgress; use crate::planner_warehouse::types::WarehouseReads; @@ -250,35 +252,61 @@ impl WarehouseReads for LiveReads { fn cursor_bounds<'py>( &self, py: Python<'py>, - statements: &dyn Fn() -> PyResult>, + statements: &dyn Fn() -> PyResult, progress: &dyn CursorProgress<'py>, ) -> PyResult>>> { - let statements: Vec = statements()?; + let CursorStatements { singles, batches } = statements()?; let connection: Bound<'py, PyAny> = self.connection(py); let execute: Bound<'py, PyAny> = self.method(py, "execute")?; let concurrency: i64 = self.inspection_concurrency(py)?; - let total: usize = statements.len(); - if concurrency <= 1 || total <= 1 { - let mut outcomes: Vec> = Vec::with_capacity(total); - for (index, sql) in statements.iter().enumerate() { + let total: usize = singles.len(); + let mut outcomes: Vec>> = vec![None; total]; + let pending: Vec = if batches.is_empty() || total <= 1 { + (0..total).collect() + } else { + progress.batched( + batches.len(), + (concurrency.max(1) as usize).min(batches.len()), + )?; + let sizes: Vec = batches.iter().map(|batch| batch.reads.len()).collect(); + let (answered, failed) = run_cursor_batches( + py, + &batches, + (&connection, &execute), + (concurrency, Some(progress.batch_callback(py, sizes)?)), + )?; + for (index, outcome) in answered { + outcomes[index] = Some(outcome); + } + failed + }; + if concurrency <= 1 || pending.len() <= 1 { + for index in pending { progress.starting(index)?; - let outcome = read_cursor_query(py, sql, (&connection, &execute))?; + let outcome = + read_cursor_query(py, &singles[index], (&connection, &execute), true)?; progress.finished(index, &outcome)?; - outcomes.push(outcome); + outcomes[index] = Some(outcome); + } + } else { + if batches.is_empty() { + progress.concurrent((concurrency as usize).min(pending.len()))?; + } + let tasks: Vec> = pending + .iter() + .map(|index| bound_task(py, singles[*index].clone(), (&connection, &execute))) + .collect::>()?; + let results = run_bounded( + py, + tasks, + concurrency, + Some(progress.completion_callback(py, pending.clone())?), + )?; + for (index, outcome) in pending.into_iter().zip(results) { + outcomes[index] = Some(outcome); } - return Ok(Read::Known(outcomes)); } - progress.concurrent((concurrency as usize).min(total))?; - let tasks: Vec> = statements - .into_iter() - .map(|sql| bound_task(py, sql, (&connection, &execute))) - .collect::>()?; - Ok(Read::Known(run_bounded( - py, - tasks, - concurrency, - Some(progress.completion_callback(py)?), - )?)) + Ok(Read::Known(outcomes.into_iter().flatten().collect())) } fn eligible_cursor_maxes<'py>( diff --git a/crates/sqlbuild-python-planner/src/live_reads/types.rs b/crates/sqlbuild-python-planner/src/live_reads/types.rs new file mode 100644 index 0000000000..4ed47b70cf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/types.rs @@ -0,0 +1,6 @@ +//! Type aliases of live warehouse reads. + +use pyo3::prelude::{Bound, PyAny}; + +/// Each answered read's position with its row, and the positions of reads whose batch failed. +pub type CursorBatchOutcomes<'py> = (Vec<(usize, Bound<'py, PyAny>)>, Vec); diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index 6e450f2ef4..1f1979f407 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -10,6 +10,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use crate::plan_output::_helpers::cursor_details::{CursorDetails, cursor_details, owner}; +use crate::plan_output::_helpers::plan_text::entry_wait_changes; use crate::plan_output::_helpers::plan_text::{model_config_diff, model_query_diff}; use crate::plan_output::_helpers::python_identity::{ definition_source_text, dependency_source_text, @@ -468,6 +469,11 @@ pub(crate) fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = + entry_wait_changes(entry, &native_field(entry, "name")?.extract::()?)?; + if !changes.is_empty() { + item.set_item("wait_changes", changes)?; + } Ok(item.into_any()) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 95f586c794..10a1bfb6ac 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -6,6 +6,7 @@ use crate::plan_output::main::items::items; use crate::plan_output::main::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::node_set_dependencies::main::wait_changes::wait_changes; use sqlbuild_core::cli_text::models::{ CliStyle, CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, PlanWarningText, ProviderUsageText, PythonPlanText, RenameHintText, SchemaFindingText, @@ -300,6 +301,13 @@ fn source_load_text(entry: &Bound<'_, PyAny>) -> PyResult { }) } +/// What a Python entry now also waits on, or no longer, since the build its identity was recorded by. +pub(crate) fn entry_wait_changes(entry: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let previous: Option = native_field(entry, "previous_metadata_json")?.extract()?; + let current: Option = native_field(entry, "current_metadata_json")?.extract()?; + Ok(wait_changes(name, previous.as_deref(), current.as_deref())) +} + /// Both identity diffs of a changed Python entry, each as `format_query_diff` lines. fn python_text(entry: &Bound<'_, PyAny>) -> PyResult { let identity_status: String = text(&native_field( @@ -331,8 +339,10 @@ fn python_text(entry: &Bound<'_, PyAny>) -> PyResult { } } let [source_diff, dependency_diff] = diffs; + let name: String = text(&native_field(entry, "name")?)?; Ok(PythonPlanText { - name: text(&native_field(entry, "name")?)?, + wait_changes: entry_wait_changes(entry, &name)?, + name, kind: text(&native_field(&native_field(entry, "kind")?, "value")?)?, phase: text(&native_field(&native_field(entry, "phase")?, "value")?)?, identity_status, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs index 5fa9706ce6..228316644c 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs @@ -6,7 +6,6 @@ use std::collections::HashSet; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyString, PyTuple}; -use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -58,7 +57,7 @@ pub(crate) fn tuple_of<'py>(py: Python<'py>, items: &[Py]) -> PyResult) -> PyResult { +pub(crate) fn fingerprint_key(fingerprint: &Bound<'_, PyAny>) -> PyResult { let target_name: Bound<'_, PyAny> = native_field(fingerprint, "target_name")?; let target: Option = if target_name.is_none() { None @@ -81,19 +80,6 @@ fn fingerprint_key(fingerprint: &Bound<'_, PyAny>) -> PyResult { }) } -pub(crate) fn newest<'py>( - py: Python<'py>, - candidates: &[Py], - (target, model_name): (Option<&EventRelation>, Option<&str>), -) -> PyResult>> { - let keys: Vec = candidates - .iter() - .map(|candidate| fingerprint_key(candidate.bind(py))) - .collect::>()?; - Ok(newest_fingerprint(&keys, target, model_name) - .map(|index| candidates[index].bind(py).clone())) -} - /// A fresh live-state inspection over the runtime's connection and planning database. pub(crate) fn inspection<'py>( runtime: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs index df3650bc04..71a880cdc0 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs @@ -82,7 +82,7 @@ fn manual_migration_requests<'py>( pub(crate) fn discover<'py>( (phase, fingerprints): (&MigrationPhase<'py>, &Bound<'py, PyAny>), - state: &Bound<'py, PyAny>, + (state, models): (&Bound<'py, PyAny>, &Bound<'py, PyDict>), schemas: &Bound<'py, PyAny>, ) -> PyResult> { let runtime = phase.runtime.clone(); @@ -104,7 +104,7 @@ pub(crate) fn discover<'py>( } let mut declarations: Vec> = Vec::new(); for request in &declared_requests { - let (origin_location, origin_model) = resolve_origin(request, &runtime, state)?; + let (origin_location, origin_model) = resolve_origin(request, (&runtime, models), state)?; declarations.push(ModelMigrationDeclaration { model_name: name_object(&request.model)?, origin_location, @@ -125,7 +125,7 @@ pub(crate) fn discover<'py>( pub(crate) fn finish<'py>( phase: &MigrationPhase<'py>, - state: &Bound<'py, PyAny>, + (state, models): (&Bound<'py, PyAny>, &Bound<'py, PyDict>), schemas: &Bound<'py, PyAny>, discovery: &ModelMigrationDiscovery<'py>, ) -> PyResult> { @@ -139,7 +139,7 @@ pub(crate) fn finish<'py>( Bound<'py, PyAny>, )> = Vec::new(); for request in &discovery.requests { - let (origin, origin_model) = resolve_origin(request, &runtime, state)?; + let (origin, origin_model) = resolve_origin(request, (&runtime, models), state)?; resolved.push((request, origin, origin_model)); } let mut entries: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs index a57764dc40..0a5d71a1e0 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs @@ -6,7 +6,7 @@ use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::PyErr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PySet, PySetMethods}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -30,17 +30,27 @@ pub(crate) fn location<'py>( new_record_with(runtime.py(), RecordId::CompiledRelationLocation, &fields) } +/// The project's models by name, the first model of a name winning, to resolve origins by name. +pub(crate) fn models_by_name<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { + let models: Bound<'py, PyDict> = PyDict::new(runtime.py()); + for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { + let item = item?; + let name = name_object(&item)?; + if !models.contains(&name)? { + models.set_item(name, item)?; + } + } + Ok(models) +} + fn named<'py>( name: &Bound<'py, PyAny>, destination: &Bound<'py, PyAny>, - runtime: &Bound<'py, PyAny>, + (runtime, models): (&Bound<'py, PyAny>, &Bound<'py, PyDict>), state: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { - let item = item?; - if name_object(&item)?.eq(name)? { - return Ok((native_field(&item, "destination")?, name.clone())); - } + if let Some(item) = models.get_item(name)? { + return Ok((native_field(&item, "destination")?, name.clone())); } let fingerprint = call_with( &native_field(state, "named_fingerprint")?, @@ -128,7 +138,7 @@ pub(crate) fn require_same_database( /// The origin location and origin model name of one migration request. pub(crate) fn resolve_origin<'py>( request: &ModelMigrationRequest<'py>, - runtime: &Bound<'py, PyAny>, + (runtime, models): (&Bound<'py, PyAny>, &Bound<'py, PyDict>), state: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = runtime.py(); @@ -167,7 +177,7 @@ pub(crate) fn resolve_origin<'py>( )?); } if parts.len() == 1 { - return named(&parts[0], &destination, runtime, state); + return named(&parts[0], &destination, (runtime, models), state); } let database = if parts.len() == maximum { parts[0].clone() diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs index 2409dc6fe7..a33747f556 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs @@ -2,6 +2,7 @@ use crate::planner_migrations::_helpers::model::model_planning::discover; use crate::planner_migrations::_helpers::model::model_planning::finish; +use crate::planner_migrations::_helpers::planning::planning_origin::models_by_name; use crate::planner_migrations::models::{MigrationPhase, ModelMigrationPlanning}; use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -19,6 +20,7 @@ pub fn plan_model_migrations<'py>( &runtime, Some(&native_field(&snapshot, "fingerprints")?), )?; - let discovery = discover((phase, fingerprints), &state, &schemas)?; - finish(phase, &state, &schemas, &discovery) + let models = models_by_name(&runtime)?; + let discovery = discover((phase, fingerprints), (&state, &models), &schemas)?; + finish(phase, (&state, &models), &schemas, &discovery) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs index 24f03dbbbf..9dbc21f7eb 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs @@ -1,8 +1,8 @@ //! Public records of planner migrations. use crate::planner_migrations::_helpers::discovery::state_inspection::RelationKey; +use crate::planner_migrations::_helpers::discovery::state_inspection::fingerprint_key; use crate::planner_migrations::_helpers::discovery::state_inspection::lowered; -use crate::planner_migrations::_helpers::discovery::state_inspection::newest; use crate::planner_migrations::_helpers::discovery::state_inspection::pending; use crate::planner_migrations::_helpers::discovery::state_inspection::relation_key; use crate::planner_migrations::_helpers::discovery::state_inspection::schema_or_empty; @@ -39,7 +39,8 @@ use sqlbuild_python_front::state_rows::constants::MODEL_KIND; use sqlbuild_python_front::state_rows::constants::OLD_NAME_KIND; use sqlbuild_python_front::state_rows::main::migrations::layout::layout; use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; -use sqlbuild_state::events::models::EventRelation; +use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; +use sqlbuild_state::events::models::{EventRelation, FingerprintKey}; use std::collections::HashMap; use std::collections::HashSet; @@ -192,6 +193,54 @@ pub(crate) struct DiscoveryRequest<'py> { pub fingerprints: Bound<'py, PyAny>, } +/// One schema's fingerprint positions by lowercased built relation name and by node name. +#[derive(Debug, Default)] +struct FingerprintPositions { + by_target: HashMap>, + by_node: HashMap>, +} + +impl FingerprintPositions { + fn of(keys: &[FingerprintKey]) -> Self { + let mut positions: Self = Self::default(); + for (position, key) in keys.iter().enumerate() { + if let Some(target) = &key.target { + positions + .by_target + .entry(target.name.to_lowercase()) + .or_default() + .push(position); + } + positions + .by_node + .entry(key.node_name.clone()) + .or_default() + .push(position); + } + positions + } + + /// Positions that built a relation named `target` or ran as `model_name`, in stored order. + fn candidates(&self, target: &str, model_name: Option<&str>) -> Vec { + let mut found: Vec = self + .by_target + .get(&target.to_lowercase()) + .into_iter() + .flatten() + .chain( + model_name + .and_then(|name| self.by_node.get(name)) + .into_iter() + .flatten(), + ) + .copied() + .collect(); + found.sort_unstable(); + found.dedup(); + found + } +} + /// Relations, fingerprints and migration events for one planning database. #[pyclass(module = "sqlbuild._native", name = "MigrationStateInspection")] #[derive(Debug)] @@ -202,6 +251,10 @@ pub struct MigrationStateInspection { read_schemas: HashSet, /// Model fingerprints per lowercased schema, in first-read order. fingerprints: Vec<(String, Vec>)>, + /// Each `fingerprints` entry's lookup keys, read from the fingerprints once. + fingerprint_keys: Vec>, + /// Each `fingerprints` entry's positions by lowercased built relation name and by node name. + fingerprint_positions: Vec, /// Positions in `fingerprints` (schema, fingerprint) by node name, in the order `fingerprints` lists them. fingerprints_by_name: HashMap>, events: Vec>, @@ -332,6 +385,8 @@ impl MigrationStateInspection { known_fingerprints: known.unbind(), read_schemas: HashSet::new(), fingerprints: Vec::new(), + fingerprint_keys: Vec::new(), + fingerprint_positions: Vec::new(), fingerprints_by_name: HashMap::new(), events: Vec::new(), event_schemas: HashSet::new(), @@ -557,14 +612,27 @@ impl MigrationStateInspection { } if state_tables.contains(&(key.clone(), FINGERPRINT_TABLE_NAME.to_owned())) { let models: Vec> = slf.borrow().model_fingerprints(py, &schema, &key)?; + let keys: Vec = models + .iter() + .map(|model| fingerprint_key(model.bind(py))) + .collect::>()?; let mut this: PyRefMut<'_, Self> = slf.borrow_mut(); match this .fingerprints - .iter_mut() - .find(|(stored, _)| *stored == key) + .iter() + .position(|(stored, _)| *stored == key) { - Some((_, stored)) => *stored = models, - None => this.fingerprints.push((key, models)), + Some(index) => { + this.fingerprints[index].1 = models; + this.fingerprint_positions[index] = FingerprintPositions::of(&keys); + this.fingerprint_keys[index] = keys; + } + None => { + this.fingerprints.push((key, models)); + this.fingerprint_positions + .push(FingerprintPositions::of(&keys)); + this.fingerprint_keys.push(keys); + } } read_fingerprints = true; } @@ -663,7 +731,10 @@ impl MigrationStateInspection { model_name: Option, ) -> PyResult>> { let key: String = lowered(&schema_or_empty(native_field(location, "schema")?))?; - let Some((_, candidates)) = self.fingerprints.iter().find(|(stored, _)| *stored == key) + let Some(index) = self + .fingerprints + .iter() + .position(|(stored, _)| *stored == key) else { return Ok(None); }; @@ -672,7 +743,19 @@ impl MigrationStateInspection { schema: native_field(location, "schema")?.extract()?, name: native_field(location, "name")?.extract()?, }; - newest(py, candidates, (Some(&target), model_name.as_deref())) + let positions: Vec = + self.fingerprint_positions[index].candidates(&target.name, model_name.as_deref()); + let keys: Vec = positions + .iter() + .map(|&position| self.fingerprint_keys[index][position].clone()) + .collect(); + Ok( + newest_fingerprint(&keys, Some(&target), model_name.as_deref()).map(|candidate| { + self.fingerprints[index].1[positions[candidate]] + .bind(py) + .clone() + }), + ) } /// The newest model fingerprint with this node name in any inspected schema. @@ -682,19 +765,21 @@ impl MigrationStateInspection { py: Python<'py>, model_name: &str, ) -> PyResult>> { - let named: Vec> = self + let named: Vec<(&Py, FingerprintKey)> = self .fingerprints_by_name .get(model_name) .into_iter() .flatten() - .filter_map(|(schema, position)| { - self.fingerprints - .get(*schema) - .and_then(|(_, fingerprints)| fingerprints.get(*position)) + .filter_map(|&(schema, position)| { + Some(( + self.fingerprints.get(schema)?.1.get(position)?, + self.fingerprint_keys.get(schema)?.get(position)?.clone(), + )) }) - .map(|fingerprint| fingerprint.clone_ref(py)) .collect(); - newest(py, &named, (None, Some(model_name))) + let keys: Vec = named.iter().map(|(_, key)| key.clone()).collect(); + Ok(newest_fingerprint(&keys, None, Some(model_name)) + .map(|position| named[position].0.bind(py).clone())) } /// Latest model fingerprints whose names are no longer in the project. diff --git a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs index 0a6b6af84f..3524cae4a7 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs @@ -73,9 +73,18 @@ pub const AUDIT_ENTRIES: &str = "audit_entries"; /// An audit plan entry's field saying its failure holds nothing back. pub const IGNORE_FAILURE: &str = "ignore_failure"; +/// Discovering the project's files and Python nodes; a location planning its published project skips it. +pub const DISCOVERING: &str = "Discovering the project"; + /// Compiling the project before planning; a location planning its published project skips it. pub const COMPILING: &str = "Compiling the project"; +/// Evaluating the configured rules; skipped where the rules ran when the project loaded. +pub const EVALUATING_RULES: &str = "Evaluating rules"; + +/// Opening the warehouse connection; a plan from a synced state opens none. +pub const CONNECTING: &str = "Connecting to the warehouse"; + /// The warehouse snapshot, model migrations and source relations, live or from a synced state. pub const READING_WAREHOUSE: &str = "Reading warehouse state"; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs index 877dabe529..90d51f4cfc 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs @@ -7,9 +7,11 @@ use crate::planner_resolution::main::safety::maximum_allowed_start::maximum_allo use crate::planner_strategy::main::lineage::resolve_lineage_reference::resolve_lineage_reference; use crate::planner_warehouse::_helpers::inspection::watermarks::validate_watermark_column; use crate::planner_warehouse::constants::{ - CURSOR_ALGEBRA_ERROR, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, LOG_DEBUG_EVENT, - PLANNER_LOGGER, + CURSOR_ALGEBRA_ERROR, CURSOR_BATCH_SIZE, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, + DATE_VALUE, LOG_DEBUG_EVENT, PLANNER_LOGGER, }; +use crate::planner_warehouse::models::CursorBatch; +use crate::planner_warehouse::models::CursorStatements; use crate::planner_warehouse::models::EligibleCursorQuery; use crate::planner_warehouse::models::Read; use crate::planner_warehouse::models::SnapshotRequest; @@ -19,6 +21,7 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_changes::constants::FORWARD_ONLY_POLICY; +use sqlbuild_analysis::planner_warehouse::main::cursor_batch_sql::cursor_batch_sql; use sqlbuild_analysis::planner_warehouse::main::cursor_bounds::cursor_bounds; use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use sqlbuild_analysis::planner_warehouse::main::cursor_query_identity::cursor_query_identity; @@ -586,17 +589,19 @@ fn report_cursor_query( let error = outcome.get_item(2)?; if error.is_none() { if !on_progress.is_none() { - on_progress.call1((format!( - "Inspected cursor bounds {identity} ({elapsed:.2}s)" - ),))?; + emit_progress( + on_progress, + format!("Inspected cursor bounds {identity} ({elapsed:.2}s)"), + )?; } return Ok(()); } let error_text: String = error.str()?.extract()?; if !on_progress.is_none() { - on_progress.call1((format!( - "Failed cursor bounds {identity} ({elapsed:.2}s): {error_text}" - ),))?; + emit_progress( + on_progress, + format!("Failed cursor bounds {identity} ({elapsed:.2}s): {error_text}"), + )?; } log_debug( py, @@ -614,9 +619,10 @@ fn report_cursor_query( ) } -fn completion_reporter<'py>( +/// The `on_complete` callback of statements where task `i` reads `queries[reads[i]]`. +fn reads_reporter<'py>( py: Python<'py>, - queries: Vec, + (queries, reads): (Vec, Vec), on_progress: &Bound<'py, PyAny>, ) -> PyResult> { let completed = Arc::new(AtomicUsize::new(0)); @@ -635,14 +641,68 @@ fn completion_reporter<'py>( .get_item("result")? .unwrap_or_else(|| py.None().into_bound(py)); let done = completed.fetch_add(1, Ordering::SeqCst) + 1; - let query = &queries[index]; - let identity = cursor_query_identity(query, (done, queries.len())); + let query = &queries[reads[index]]; + let identity = cursor_query_identity(query, (done, reads.len())); report_cursor_query(py, query, (&result, &identity), on_progress.bind(py)) }, )? .into_any()) } +/// The `on_complete` callback of batched statements answering `sizes[i]` reads each. +fn batch_reporter<'py>( + py: Python<'py>, + sizes: Vec, + on_progress: &Bound<'py, PyAny>, +) -> PyResult> { + let completed = Arc::new(AtomicUsize::new(0)); + let on_progress: Py = on_progress.clone().unbind(); + Ok(PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + let py = args.py(); + let on_progress = on_progress.bind(py); + if on_progress.is_none() { + return Ok(()); + } + let kwargs = kwargs.cloned().unwrap_or_else(|| PyDict::new(py)); + let index: usize = kwargs + .get_item("index")? + .map_or(Ok(0), |value| value.extract())?; + let Some(result) = kwargs.get_item("result")? else { + return Ok(()); + }; + let done = completed.fetch_add(1, Ordering::SeqCst) + 1; + let elapsed: f64 = result.get_item(1)?.extract()?; + let error = result.get_item(2)?; + let relations = sizes[index]; + let line = if error.is_none() { + format!( + "Inspected cursor bounds ({done}/{}): {relations} relations ({elapsed:.2}s)", + sizes.len() + ) + } else { + format!( + "Failed cursor bounds ({done}/{}): {relations} relations ({elapsed:.2}s): {}; reading each relation on its own", + sizes.len(), + error.str()? + ) + }; + emit_progress(on_progress, line)?; + Ok(()) + }, + )? + .into_any()) +} + +/// Write one progress `line` through the plan's `on_progress` callback. +fn emit_progress(on_progress: &Bound<'_, PyAny>, line: String) -> PyResult<()> { + on_progress.call1((line,))?; + Ok(()) +} + fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResult> { let mut values = Values::new(); if row.is_none() { @@ -675,11 +735,14 @@ fn execute_queries<'py>( queries: &[PhysicalCursorQuery], ) -> PyResult, HashSet)>> { let py = request.project.py(); - let statements = || -> PyResult> { - queries - .iter() - .map(|query| dialect_cursor_sql(&request.dialect, query)) - .collect() + let statements = || -> PyResult { + Ok(CursorStatements { + singles: queries + .iter() + .map(|query| dialect_cursor_sql(&request.dialect, query)) + .collect::>()?, + batches: dialect_cursor_batches(&request.dialect, queries)?, + }) }; let progress = (queries, &request.on_progress); let Read::Known(outcomes) = @@ -1075,12 +1138,15 @@ pub(crate) fn cursor_snapshots<'py>( .iter() .map(|query| query.min_tags.len() + query.max_tags.len()) .sum(); - request.on_progress.call1((format!( - "Gathered cursor bounds ({}/{logical} logical values; {} physical relation reads). ({:.2}s)", - values.len(), - queries.len(), - started.elapsed().as_secs_f64() - ),))?; + emit_progress( + &request.on_progress, + format!( + "Gathered cursor bounds ({}/{logical} logical values; {} physical relation reads). ({:.2}s)", + values.len(), + queries.len(), + started.elapsed().as_secs_f64() + ), + )?; } Ok((assemble(py, &models, (&values, &failed))?, false)) } @@ -1113,13 +1179,62 @@ pub(crate) fn dialect_cursor_sql( } } +/// `UNION ALL` batches answering `queries`, bounds read as `dialect_cursor_sql` reads them. +fn dialect_cursor_batches( + dialect: &Bound<'_, Dialect>, + queries: &[PhysicalCursorQuery], +) -> PyResult> { + if queries.len() <= 1 { + return Ok(Vec::new()); + } + let py = dialect.py(); + let render = dialect.syntax(CURSOR_TEXT_METHOD)?; + let failure: RefCell> = RefCell::new(None); + let as_text = |query: &PhysicalCursorQuery, expression: &str| -> String { + call_with( + &render, + &[ + ("expression", text(py, expression)), + ("cursor_type", text(py, &query.cursor_type)), + ], + ) + .and_then(|rendered| rendered.extract::()) + .unwrap_or_else(|error| { + failure.replace(Some(error)); + String::new() + }) + }; + let mut batches: Vec = Vec::new(); + for (chunk, start) in queries + .chunks(CURSOR_BATCH_SIZE) + .zip((0..).step_by(CURSOR_BATCH_SIZE)) + { + let members: Vec<&PhysicalCursorQuery> = chunk.iter().collect(); + batches.push(CursorBatch { + reads: chunk + .iter() + .enumerate() + .map(|(offset, query)| (start + offset, !query.min_tags.is_empty())) + .collect(), + sql: cursor_batch_sql(&members, &as_text), + }); + } + match failure.into_inner() { + Some(error) => Err(error), + None => Ok(batches), + } +} + /// Progress lines of one plan's cursor bound statements: the statements' queries and `on_progress`. impl<'py> CursorProgress<'py> for (&[PhysicalCursorQuery], &Bound<'py, PyAny>) { fn starting(&self, index: usize) -> PyResult<()> { let (queries, on_progress) = *self; if !on_progress.is_none() { let identity = cursor_query_identity(&queries[index], (index + 1, queries.len())); - on_progress.call1((format!("Inspecting cursor bounds {identity}..."),))?; + emit_progress( + on_progress, + format!("Inspecting cursor bounds {identity}..."), + )?; } Ok(()) } @@ -1127,10 +1242,32 @@ impl<'py> CursorProgress<'py> for (&[PhysicalCursorQuery], &Bound<'py, PyAny>) { fn concurrent(&self, concurrency: usize) -> PyResult<()> { let (queries, on_progress) = *self; if !on_progress.is_none() { - on_progress.call1((format!( - "Inspecting cursor bounds for {} relations ({concurrency} concurrent)...", - queries.len() - ),))?; + emit_progress( + on_progress, + format!( + "Inspecting cursor bounds for {} relations ({concurrency} concurrent)...", + queries.len() + ), + )?; + } + Ok(()) + } + + fn batched(&self, batches: usize, concurrency: usize) -> PyResult<()> { + let (queries, on_progress) = *self; + if !on_progress.is_none() { + let statements = if batches == 1 { + "statement" + } else { + "statements" + }; + emit_progress( + on_progress, + format!( + "Inspecting cursor bounds for {} relations in {batches} {statements} ({concurrency} concurrent)...", + queries.len() + ), + )?; } Ok(()) } @@ -1146,8 +1283,17 @@ impl<'py> CursorProgress<'py> for (&[PhysicalCursorQuery], &Bound<'py, PyAny>) { ) } - fn completion_callback(&self, py: Python<'py>) -> PyResult> { + fn completion_callback( + &self, + py: Python<'py>, + reads: Vec, + ) -> PyResult> { let (queries, on_progress) = *self; - completion_reporter(py, queries.to_vec(), on_progress) + reads_reporter(py, (queries.to_vec(), reads), on_progress) + } + + fn batch_callback(&self, py: Python<'py>, sizes: Vec) -> PyResult> { + let (_, on_progress) = *self; + batch_reporter(py, sizes, on_progress) } } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index 56000c8401..a580b7064f 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -386,17 +386,21 @@ fn gather_relations<'py>( Ok((result, states)) } +/// Listed columns of `relevant` nodes, the selection and the upstream its warnings check; else all. fn gather_columns<'py>( request: &SnapshotRequest<'py>, relations: &Bound<'py, PyDict>, + relevant: &Bound<'py, PyAny>, ) -> PyResult> { let py = request.project.py(); let mut node_names: HashSet = HashSet::new(); for node in destination_nodes(&request.project)? { - node_names.insert( - sqlbuild_python_project::resource_views::main::resource_reads::name::name(&node)? - .extract()?, - ); + if selected_node(relevant, &node)? { + node_names.insert( + sqlbuild_python_project::resource_views::main::resource_reads::name::name(&node)? + .extract()?, + ); + } } let mut needed: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); for (logical, relation) in relations.iter() { @@ -693,7 +697,7 @@ pub(crate) fn warehouse_snapshot<'py>( optional_names(py, query_schemas)?, optional_names(py, metadata_names.as_ref())?, )?; - let columns = gather_columns(request, &relations)?; + let columns = gather_columns(request, &relations, &relevant)?; let fingerprints = gather_fingerprints( request, &database, diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs index 7f31dc14a4..5da2153fcc 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs @@ -54,3 +54,6 @@ pub const PREVIEW_UNKNOWNS: [&str; 5] = [ /// What a preview's cursor reads leave to the run. pub const CURSOR_INPUTS: &str = "cursor inputs"; + +/// Cursor reads one `UNION ALL` statement answers, so a plan runs few statements, each small. +pub(crate) const CURSOR_BATCH_SIZE: usize = 50; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index 3f669b09b4..da5c4f3d27 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -427,6 +427,22 @@ pub struct EligibleCursorQuery { pub(crate) sql: String, } +/// One `UNION ALL` statement answering several cursor reads, one row per read led by `_read`. +#[derive(Debug)] +pub struct CursorBatch { + /// The reads it answers in `_read` order: each read's index and whether its row has a minimum. + pub(crate) reads: Vec<(usize, bool)>, + pub(crate) sql: String, +} + +/// A plan's cursor reads: one statement per read, and batches answering the reads together. +#[derive(Debug)] +pub struct CursorStatements { + pub(crate) singles: Vec, + /// Empty when the reads run one statement each. + pub(crate) batches: Vec, +} + /// The warehouse reads one plan runs through. #[pyclass(frozen, module = "sqlbuild._native", name = "WarehouseReads")] #[derive(Debug)] @@ -645,7 +661,7 @@ impl WarehouseReads for SyncedReads { fn cursor_bounds<'py>( &self, _py: Python<'py>, - _statements: &dyn Fn() -> PyResult>, + _statements: &dyn Fn() -> PyResult, _progress: &dyn CursorProgress<'py>, ) -> PyResult>>> { Ok(Read::Unknown(CURSOR_INPUTS.into())) diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs index a036a75f7f..cbf790687b 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs @@ -1,6 +1,7 @@ //! The warehouse reads planning consults, as traits. use crate::planner_warehouse::models::AdapterRead; +use crate::planner_warehouse::models::CursorStatements; use crate::planner_warehouse::models::EligibleCursorQuery; use crate::planner_warehouse::models::FingerprintRead; use crate::planner_warehouse::models::Read; @@ -15,10 +16,18 @@ pub trait CursorProgress<'py> { fn starting(&self, index: usize) -> PyResult<()>; /// Before statements run `concurrency` at a time. fn concurrent(&self, concurrency: usize) -> PyResult<()>; + /// Before `batches` statements answering every read run `concurrency` at a time. + fn batched(&self, batches: usize, concurrency: usize) -> PyResult<()>; /// After statement `index` ran, with its `(row, seconds, error)` outcome. fn finished(&self, index: usize, outcome: &Bound<'py, PyAny>) -> PyResult<()>; - /// The `on_complete(index=, result=)` callback concurrent statements report through. - fn completion_callback(&self, py: Python<'py>) -> PyResult>; + /// The `on_complete(index=, result=)` callback of statements where task `i` reads `reads[i]`. + fn completion_callback( + &self, + py: Python<'py>, + reads: Vec, + ) -> PyResult>; + /// The `on_complete(index=, result=)` callback of batches answering `sizes[i]` reads each. + fn batch_callback(&self, py: Python<'py>, sizes: Vec) -> PyResult>; } /// Everything a plan reads from the warehouse, live or from a synced state that may answer unknown. @@ -65,11 +74,11 @@ pub trait WarehouseReads: Send + Sync + std::fmt::Debug { /// Whether listing by schema and name finds relations a name-only listing would. fn lists_by_schema(&self, py: Python<'_>) -> PyResult; - /// Each statement's `(row, seconds, error)`; `statements` renders SQL only for reads that run it. + /// Each read's `(row, seconds, error)`; a failed batch's reads then run their own statements. fn cursor_bounds<'py>( &self, py: Python<'py>, - statements: &dyn Fn() -> PyResult>, + statements: &dyn Fn() -> PyResult, progress: &dyn CursorProgress<'py>, ) -> PyResult>>>; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index 50168e66a7..ce70fdee71 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -109,9 +109,16 @@ pub(crate) fn job_catalog(py: Python<'_>) -> PyResult> { /// Each job's selector tokens and each group's members declared for `target`, and the names a schedule or sensor starts there. fn inventory_catalog(inventory: &OrchestrationInventory, target: Option<&str>) -> Catalog { - let declared_for = |targets: &[String]| { + catalog_where(inventory, |targets: &[String]| { targets.is_empty() || target.is_some_and(|target| targets.iter().any(|name| name == target)) - }; + }) +} + +/// Each job's selector tokens and each group's members `declared_for` accepts the targets of. +pub(crate) fn catalog_where( + inventory: &OrchestrationInventory, + declared_for: impl Fn(&[String]) -> bool, +) -> Catalog { let tokens = |value: &str| value.split_whitespace().map(str::to_owned).collect(); let mut catalog: BTreeMap = BTreeMap::new(); let mut triggered: BTreeSet = BTreeSet::new(); @@ -159,7 +166,7 @@ fn inventory_catalog(inventory: &OrchestrationInventory, target: Option<&str>) - ); } for sensor in &inventory.sensors { - if sensor.targets.as_deref().is_none_or(declared_for) { + if sensor.targets.as_deref().is_none_or(&declared_for) { triggered.extend( sensor .job_references diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs new file mode 100644 index 0000000000..6510eb7a92 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs @@ -0,0 +1,14 @@ +//! Every job and group a project declares, for resolving `job(...)` dependencies. + +use crate::compiled_project::_helpers::job_catalog::catalog_where; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::graph::models::JobSelection; +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use std::collections::BTreeMap; + +/// Every job and group the project declares, whatever targets they are declared for. +pub fn declared_jobs(project_dir: &Bound<'_, PyAny>) -> PyResult> { + let inventory: OrchestrationInventory = orchestration_declarations(project_dir)?; + Ok(catalog_where(&inventory, |_| true).0) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs index 0cb998a7ee..efde8565a4 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -1,5 +1,6 @@ pub mod config_values; pub mod declarations; +pub mod declared_jobs; pub mod entry_schema_fact; pub mod family_fact; pub mod graph_with_jobs; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs index 4388c92d11..58ff9d5b96 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod python_lifecycle; pub(crate) mod python_node_graph; pub(crate) mod python_selection; +pub(crate) mod set_dependencies; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs index ccd8d7dd2e..32638fb9af 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs @@ -4,14 +4,21 @@ use crate::graph::main::python_nodes::python_node_graph::python_node_graph; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use sqlbuild_analysis::node_set_dependencies::main::set_dependency_metadata::set_dependency_metadata; +use sqlbuild_analysis::node_set_dependencies::models::{GraphNode, ResolvedNode, ResolvedSet}; +use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNode}; use sqlbuild_host::entries::models::UserCodeHost; +use std::collections::HashMap; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::main::python_files::node_identity::node_identity; use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; @@ -32,6 +39,106 @@ pub(crate) struct Vocabulary<'py> { pub(crate) kinds: Bound<'py, PyAny>, pub(crate) sql_ref: Bound<'py, PyAny>, pub(crate) providers: Bound<'py, PyDict>, + /// Each node's resolved `job(...)` and `tag(...)` dependencies; empty for the explicit graph. + pub(crate) sets: Vec, + /// Each discovered node's kind, such as `task`, by name. + pub(crate) node_kinds: HashMap, +} + +impl Vocabulary<'_> { + /// Each set `name` declares, as `(set label, end node labels)` its identity records. + fn recorded_sets(&self, name: &str) -> Vec<(String, Vec)> { + self.sets + .iter() + .filter(|node| node.name == name) + .flat_map(|node| &node.sets) + .map(|set| self.recorded_set(set)) + .collect() + } + + /// One set's label and its end node labels. + fn recorded_set(&self, set: &ResolvedSet) -> (String, Vec) { + let ends: Vec = set.ends.iter().map(|end| self.end_label(end)).collect(); + ( + format!("{} '{}'", set.dependency.kind, set.dependency.name), + ends, + ) + } + + /// `model 'orders'` or `task 'export_orders'`. + fn end_label(&self, end: &GraphNode) -> String { + match end { + GraphNode::Sql((kind, name)) => format!("{kind} '{name}'"), + GraphNode::Python(name) => format!( + "{} '{name}'", + self.node_kinds.get(name).map_or("node", String::as_str) + ), + } + } + + /// The end nodes of every set dependency `name` declares. + fn set_ends<'a>(&'a self, name: &'a str) -> impl Iterator + 'a { + self.sets + .iter() + .filter(move |node| node.name == name) + .flat_map(|node| &node.sets) + .flat_map(|set| &set.ends) + } +} + +/// A `SqlResourceRef` to each SQL end node of `name`'s sets that it does not already declare. +fn set_sql_refs<'py>( + vocabulary: &Vocabulary<'py>, + name: &str, + mut declared: Vec<(String, String)>, +) -> PyResult>> { + let py: Python<'py> = vocabulary.sql_ref.py(); + let kinds: Bound<'py, PyAny> = host_entry(py, UserCodeHost::SqlResourceRefKind)?; + let mut references: Vec> = Vec::new(); + for end in vocabulary.set_ends(name) { + let GraphNode::Sql(key) = end else { + continue; + }; + if declared.contains(key) { + continue; + } + declared.push(key.clone()); + references.push(call_with( + &vocabulary.sql_ref, + &[ + ("kind", native_field(&kinds, &key.0.to_uppercase())?), + ("name", PyString::new(py, &key.1).into_any()), + ], + )?); + } + Ok(references) +} + +/// `identity` with `recorded` sets in its metadata, so a later plan can list wait changes. +fn with_recorded_sets<'py>( + identity: Bound<'py, PyAny>, + recorded: &[(String, Vec)], +) -> PyResult> { + if recorded.is_empty() || identity.is_none() { + return Ok(identity); + } + let metadata: String = native_field(&identity, "metadata_json")?.extract()?; + let updated: String = set_dependency_metadata(&metadata, recorded); + replaced( + &identity, + &[( + "metadata_json", + PyString::new(identity.py(), &updated).into_any(), + )], + ) +} + +/// `(kind, name)` of a `SqlResourceRef`. +fn sql_ref_key(reference: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + native_field(&native_field(reference, "kind")?, "value")?.extract()?, + native_field(reference, "name")?.extract()?, + )) } fn attribute_value<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { @@ -100,14 +207,19 @@ pub(crate) fn python_node<'py>( if identity.is_none() { identity = node_identity(py, (kind, documented), &get)?; } + let name: String = get(NAME_KEY)?.extract()?; + identity = with_recorded_sets(identity, &vocabulary.recorded_sets(&name))?; let depends_on: Bound<'py, PyAny> = get("depends_on")?; let mut sql_deps: Vec> = Vec::new(); + let mut declared: Vec<(String, String)> = Vec::new(); for dependency in depends_on.try_iter()? { let dependency: Bound<'py, PyAny> = dependency?; if dependency.is_instance(&vocabulary.sql_ref)? { + declared.push(sql_ref_key(&dependency)?); sql_deps.push(dependency); } } + sql_deps.extend(set_sql_refs(vocabulary, &name, declared)?); let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("kind", kind_member), ("file_path", get("file_path")?), @@ -133,8 +245,9 @@ pub(crate) fn python_node<'py>( pub(crate) fn upstream_names( py: Python<'_>, nodes: &[Bound<'_, PyAny>], - sql_ref: &Bound<'_, PyAny>, + vocabulary: &Vocabulary<'_>, ) -> PyResult>> { + let sql_ref: &Bound<'_, PyAny> = &vocabulary.sql_ref; let by_key: Bound<'_, PyDict> = PyDict::new(py); for node in nodes { by_key.set_item(native_field(node, "function")?, native_field(node, "name")?)?; @@ -161,11 +274,80 @@ pub(crate) fn upstream_names( names.push(upstream.extract()?); } } + let name: String = native_field(node, "name")?.extract()?; + for end in vocabulary.set_ends(&name) { + if let GraphNode::Python(upstream) = end + && !names.contains(upstream) + { + names.push(upstream.clone()); + } + } Ok(names) }) .collect() } +/// Each discovered node's lowercase kind, by name. +fn node_kinds(discovered_inputs: &Bound<'_, PyAny>) -> PyResult> { + let mut kinds: HashMap = HashMap::new(); + for (collection, kind, _) in KINDS { + for discovered in native_field(discovered_inputs, collection)?.try_iter()? { + kinds.insert( + native_field(&discovered?, NAME_KEY)?.extract()?, + kind.to_lowercase(), + ); + } + } + Ok(kinds) +} + +/// The Python-node graph of only the dependencies the nodes name one by one. +pub(crate) fn explicit_python_node_graph<'py>( + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + graph_with_sets(discovered_inputs, Vec::new()) +} + +/// The executable Python-node graph with each of `sets` an edge to its end nodes. +pub(crate) fn graph_with_sets<'py>( + discovered_inputs: &Bound<'py, PyAny>, + sets: Vec, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let providers: Bound<'py, PyDict> = PyDict::new(py); + for provider in native_field(discovered_inputs, "providers")?.try_iter()? { + let provider: Bound<'py, PyAny> = provider?; + providers.set_item(native_field(&provider, "name")?, provider)?; + } + let vocabulary: Vocabulary<'py> = Vocabulary { + node: record_class(py, RecordId::DiscoveredPythonNode)?, + kinds: host_entry(py, UserCodeHost::PythonNodeKind)?, + sql_ref: host_entry(py, UserCodeHost::SqlResourceRef)?, + providers, + node_kinds: if sets.is_empty() { + HashMap::new() + } else { + node_kinds(discovered_inputs)? + }, + sets, + }; + let mut nodes: Vec> = Vec::new(); + within_identity_pass(py, || { + for (collection, kind, documented) in KINDS { + for discovered in native_field(discovered_inputs, collection)?.try_iter()? { + nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); + } + } + Ok(()) + })?; + let inventory: Vec = nodes + .iter() + .zip(upstream_names(py, &nodes, &vocabulary)?) + .map(|(node, upstream)| inventory_node(node, upstream)) + .collect::>()?; + graph_record(py, &nodes, graph_indexes(&inventory)) +} + pub(crate) fn inventory_node( node: &Bound<'_, PyAny>, upstream: Vec, diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs new file mode 100644 index 0000000000..30e2fb4665 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/set_dependencies.rs @@ -0,0 +1,46 @@ +//! The discovered Python nodes declaring `job(...)` or `tag(...)` dependencies. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::node_set_dependencies::models::{DependantNode, NodeSetDependency}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Discovered function collections and the node kind each holds. +const COLLECTIONS: [(&str, &str); 4] = [ + ("loader_functions", "loader"), + ("task_functions", "task"), + ("asset_functions", "asset"), + ("check_functions", "check"), +]; + +/// Each discovered Python node declaring a `job(...)` or `tag(...)` dependency. +pub(crate) fn dependants(discovered: &Bound<'_, PyAny>) -> PyResult> { + let node_set: Bound<'_, PyAny> = host_entry(discovered.py(), UserCodeHost::NodeSetRef)?; + let mut found: Vec = Vec::new(); + for (collection, kind) in COLLECTIONS { + for function in native_field(discovered, collection)?.try_iter()? { + let function: Bound<'_, PyAny> = function?; + let mut sets: Vec = Vec::new(); + for dependency in native_field(&function, "depends_on")?.try_iter()? { + let dependency: Bound<'_, PyAny> = dependency?; + if dependency.is_instance(&node_set)? { + sets.push(NodeSetDependency { + kind: native_field(&native_field(&dependency, "kind")?, "value")? + .extract()?, + name: native_field(&dependency, "name")?.extract()?, + exclude: native_field(&dependency, "exclude")?.extract()?, + }); + } + } + if !sets.is_empty() { + found.push(DependantNode { + name: native_field(&function, "name")?.extract()?, + kind: kind.to_owned(), + sets, + }); + } + } + } + Ok(found) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs index e0c9f75bcb..26ca9f0d63 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs @@ -6,5 +6,6 @@ pub mod python_node_names; pub mod register_python_lifecycle; pub mod register_python_node_graph; pub mod register_python_selection; +pub mod resolve_node_sets; pub mod resolve_python_check_excludes; pub mod run_selection_from_inputs; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs index 6fc4c5c0ea..790c948091 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs @@ -4,12 +4,15 @@ use crate::graph::_helpers::python_nodes::python_selection::{ allows_round_trips, project_sources, selection_graph, }; use crate::graph::main::python_nodes::python_node_graph::python_node_graph; +use crate::graph::main::python_nodes::resolve_node_sets::resolve_node_sets; use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph}; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::python_selection::main::python_sql_graph_error::python_sql_graph_error; use sqlbuild_analysis::python_selection::models::{PythonSelectionGraph, SqlSelectionGraph}; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::types::NodeSetOutcome; +use sqlbuild_python_front::discovery::main::records::stored_node_sets::stored_node_sets; /// The project's first whole-graph boundary error across SQL resources and Python nodes, as every selection reports it. pub fn project_graph_error( @@ -19,6 +22,13 @@ pub fn project_graph_error( let py: Python<'_> = project_graph.py(); let project_graph: &Bound<'_, CompiledProjectGraph> = project_graph.cast()?; let project: &Bound<'_, PyAny> = project_graph.get().project(py); + let sets: Option = match stored_node_sets(discovered_inputs)? { + Some(outcome) => Some(outcome), + None => resolve_node_sets(discovered_inputs, project)?, + }; + if let Some(Err(error)) = sets { + return Ok(Some(error)); + } let sources: Vec<(String, Option)> = project_sources(project)?; let nodes: PythonSelectionGraph = selection_graph(&python_node_graph(discovered_inputs)?)?; let native: &Bound<'_, NativeProjectGraph> = project_graph.get().native(py); diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs index 53c6bd7bfa..be1152468b 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs @@ -1,51 +1,19 @@ //! The internal executable Python-node graph of discovered project inputs. -use crate::graph::_helpers::python_nodes::python_node_graph::KINDS; -use crate::graph::_helpers::python_nodes::python_node_graph::Vocabulary; -use crate::graph::_helpers::python_nodes::python_node_graph::graph_record; -use crate::graph::_helpers::python_nodes::python_node_graph::inventory_node; -use crate::graph::_helpers::python_nodes::python_node_graph::python_node; -use crate::graph::_helpers::python_nodes::python_node_graph::upstream_names; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; -use sqlbuild_analysis::python_selection::models::InventoryNode; -use sqlbuild_host::entries::models::UserCodeHost; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; +use crate::graph::_helpers::python_nodes::python_node_graph::graph_with_sets; +use crate::graph::_helpers::python_nodes::python_selection::planner_error; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::node_set_dependencies::models::ResolvedNode; +use sqlbuild_python_front::discovery::main::records::stored_node_sets::stored_node_sets; -/// The internal executable Python-node graph of discovered project inputs. +/// The executable Python-node graph, each resolved `job(...)` and `tag(...)` an edge to its end nodes. pub fn python_node_graph<'py>( discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = discovered_inputs.py(); - let providers: Bound<'py, PyDict> = PyDict::new(py); - for provider in native_field(discovered_inputs, "providers")?.try_iter()? { - let provider: Bound<'py, PyAny> = provider?; - providers.set_item(native_field(&provider, "name")?, provider)?; - } - let vocabulary: Vocabulary<'py> = Vocabulary { - node: record_class(py, RecordId::DiscoveredPythonNode)?, - kinds: host_entry(py, UserCodeHost::PythonNodeKind)?, - sql_ref: host_entry(py, UserCodeHost::SqlResourceRef)?, - providers, + let sets: Vec = match stored_node_sets(discovered_inputs)? { + Some(Ok(resolved)) => resolved, + Some(Err(error)) => return Err(planner_error(discovered_inputs.py(), error)), + None => Vec::new(), }; - let mut nodes: Vec> = Vec::new(); - within_identity_pass(py, || { - for (collection, kind, documented) in KINDS { - for discovered in native_field(discovered_inputs, collection)?.try_iter()? { - nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); - } - } - Ok(()) - })?; - let inventory: Vec = nodes - .iter() - .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?) - .map(|(node, upstream)| inventory_node(node, upstream)) - .collect::>()?; - graph_record(py, &nodes, graph_indexes(&inventory)) + graph_with_sets(discovered_inputs, sets) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs new file mode 100644 index 0000000000..a494b6d3c8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs @@ -0,0 +1,58 @@ +//! Python nodes' `job(...)` and `tag(...)` dependencies, resolved once a compile has the SQL graph. + +use std::collections::BTreeMap; + +use crate::compiled_project::main::declared_jobs::declared_jobs; +use crate::graph::_helpers::python_nodes::python_node_graph::explicit_python_node_graph; +use crate::graph::_helpers::python_nodes::python_selection::{ + allows_round_trips, project_sources, selection_graph, +}; +use crate::graph::_helpers::python_nodes::set_dependencies::dependants; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::graph::models::JobSelection; +use sqlbuild_analysis::node_set_dependencies::main::resolve_node_set_dependencies::resolve_node_set_dependencies; +use sqlbuild_analysis::node_set_dependencies::models::DependantNode; +use sqlbuild_analysis::python_selection::models::{PythonSelectionGraph, SqlSelectionGraph}; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::types::NodeSetOutcome; +use sqlbuild_python_front::discovery::main::records::keep_node_sets::keep_node_sets; + +/// Resolve and keep the set dependencies of `discovered` over `project`'s graph, errors included. +pub fn resolve_node_sets( + discovered: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult> { + let found: Vec = dependants(discovered)?; + let outcome: Option = if found.is_empty() { + None + } else { + let native: Bound<'_, NativeProjectGraph> = + native_field(project, "lineage_graph")?.cast_into()?; + let sources: Vec<(String, Option)> = project_sources(project)?; + let python: PythonSelectionGraph = + selection_graph(&explicit_python_node_graph(discovered)?)?; + let project_dir: Bound<'_, PyAny> = native_field(discovered, "project_dir")?; + let catalog: BTreeMap = if project_dir.is_none() { + BTreeMap::new() + } else { + declared_jobs(&project_dir)? + }; + let allow_warehouse_round_trips: bool = allows_round_trips(project)?; + Some(compiler_guard(|| { + Ok(resolve_node_set_dependencies( + SqlSelectionGraph { + graph: native.get().graph(), + sources: &sources, + allow_warehouse_round_trips, + }, + &python, + &found, + &catalog, + )) + })?) + }; + keep_node_sets(discovered, outcome.clone())?; + Ok(outcome) +} diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index 52e20da977..76de4d0736 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -37,6 +37,9 @@ use sqlbuild_analysis::graph::models::BuildResources; use sqlbuild_analysis::graph::models::EdgeDirection; use sqlbuild_analysis::graph::models::GraphIndexes; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::node_set_dependencies::models::GraphNode; +use sqlbuild_analysis::node_set_dependencies::models::ResolvedNode; +use sqlbuild_analysis::node_set_dependencies::models::ResolvedSet; use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; @@ -48,6 +51,7 @@ use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::discovery::main::records::stored_node_sets::stored_node_sets; use std::cell::RefCell; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -398,6 +402,8 @@ pub struct ProjectSelection { pub(crate) sources: Vec<(String, Option)>, allow_warehouse_round_trips: bool, pub nodes: PythonSelectionGraph, + /// Each Python node's resolved `job(...)` and `tag(...)` dependencies. + pub sets: Vec, } impl ProjectSelection { @@ -406,6 +412,10 @@ impl ProjectSelection { sources: project_sources(project)?, allow_warehouse_round_trips: allows_round_trips(project)?, nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, + sets: match stored_node_sets(discovered_inputs)? { + Some(Ok(sets)) => sets, + _ => Vec::new(), + }, }) } @@ -444,6 +454,31 @@ impl ProjectSelection { .map(|node| format!("{}:{name}", node.kind)) } + /// A Python node's published `set_dependencies`: each job or tag it depends on and its end node keys. + pub fn set_dependencies(&self, name: &str) -> Option { + let resolved = self.sets.iter().find(|resolved| resolved.name == name)?; + let sets: Vec = + resolved.sets.iter().map(|set| self.set_json(set)).collect(); + Some(serde_json::Value::Array(sets)) + } + + /// One published job or tag dependency with the keys of its end nodes. + fn set_json(&self, set: &ResolvedSet) -> serde_json::Value { + let mut ends: Vec = Vec::new(); + for end in &set.ends { + match end { + GraphNode::Sql((kind, name)) => ends.push(format!("{kind}:{name}")), + GraphNode::Python(name) => ends.extend(self.node_key(name)), + } + } + serde_json::json!({ + "kind": set.dependency.kind, + "name": set.dependency.name, + "exclude": set.dependency.exclude, + "ends": ends, + }) + } + /// Each Python node's published key and the keys it reads: its upstream Python nodes and its SQL deps. pub fn node_deps(&self) -> Vec<(String, BTreeSet)> { let upstream: BTreeMap<&str, &[String]> = self diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs index 960edd15e0..9d5bee8dad 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod orchestration_triggers; pub(crate) mod runs_state; +pub(crate) mod synced_catalog; diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/synced_catalog.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/synced_catalog.rs new file mode 100644 index 0000000000..5810d1f104 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/synced_catalog.rs @@ -0,0 +1,115 @@ +//! Reads and writes a plan from synced state shares: the control store's mirrored catalog and stderr notes. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::warehouse_metadata::main::location_reads::read_development_location::read_development_location; +use sqlbuild_orchestration::warehouse_metadata::main::location_reads::read_synced_catalog::read_synced_catalog; +use sqlbuild_orchestration::warehouse_metadata::models::SyncedCatalogRelation; + +use crate::native_host::constants::{CONTROL_STORE_FILE, CONTROL_STORE_VARIABLE, SYNCED_STATE}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; + +/// Whether `--plan-from`, or else the target's `plan_from`, asks for the synced state. +pub(crate) fn plans_from_synced_state( + discovered: &Bound<'_, PyAny>, + options: &Bound<'_, PyAny>, + target: &Bound<'_, PyAny>, +) -> PyResult { + let requested: Option = native_field(options, "plan_from")?.extract()?; + if let Some(requested) = requested { + return Ok(requested == SYNCED_STATE); + } + if target.is_none() { + return Ok(false); + } + let config: Bound<'_, PyAny> = resolve_target_config( + discovered.py(), + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + target, + )?; + let configured: Option = native_field(&config, "plan_from")?.extract()?; + Ok(configured.as_deref() == Some(SYNCED_STATE)) +} + +/// Write `text` to Python's stderr, where the plan's other progress goes, and flush it. +pub(crate) fn write_stderr(py: Python<'_>, text: &str) -> PyResult<()> { + let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; + stderr.call_method1("write", (text,))?; + stderr.call_method0("flush")?; + Ok(()) +} + +pub(crate) fn planning_live(py: Python<'_>, reason: &str) -> PyResult<()> { + write_stderr(py, &format!("Planning live: {reason}\n")) +} + +/// The mirrored catalog the project's control store holds for `target`'s development location. +pub(crate) fn synced_relations( + project_dir: &Path, + target: &str, +) -> Result, String> { + let url: String = match std::env::var(CONTROL_STORE_VARIABLE) { + Ok(url) => url, + Err(_) => { + let path: PathBuf = project_dir.join(CONTROL_STORE_FILE); + if !path.is_file() { + return Err("the project has no control store".to_owned()); + } + path.to_string_lossy().into_owned() + } + }; + let project: PathBuf = + std::fs::canonicalize(project_dir).unwrap_or_else(|_| project_dir.to_path_buf()); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| error.to_string())?; + let read = runtime.block_on(async { + let store = open_store(&url).await.map_err(|error| error.to_string())?; + let location = read_development_location(&store, target, |path| { + std::fs::canonicalize(path).is_ok_and(|path| path == project) + }) + .await + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("no location syncs target '{target}' for this project"))?; + read_synced_catalog(&store, (location, target)) + .await + .map_err(|error| error.to_string()) + }); + runtime.shutdown_timeout(Duration::from_secs(1)); + read +} + +/// `(database, schema, name, columns, observed_at)` of one mirrored relation. +pub(crate) fn synced_row<'py>( + py: Python<'py>, + relation: &SyncedCatalogRelation, +) -> PyResult> { + let record = &relation.record; + let columns: Vec<(String, String)> = record + .columns + .iter() + .map(|column| (column.name.clone(), column.data_type.clone())) + .collect(); + PyTuple::new( + py, + [ + record.state_database.clone().into_pyobject(py)?.into_any(), + record.state_schema.clone().into_pyobject(py)?.into_any(), + record.name.clone().into_pyobject(py)?.into_any(), + columns.into_pyobject(py)?.into_any(), + relation + .observed_at + .to_rfc3339() + .into_pyobject(py)? + .into_any(), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/native_host/constants.rs b/crates/sqlbuild-python-project/src/native_host/constants.rs new file mode 100644 index 0000000000..2c58c12343 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/constants.rs @@ -0,0 +1,12 @@ +//! Names a plan from synced state uses to find and attach the target's mirrored catalog. + +/// The `plan_from` source that plans from the target's synced copy of warehouse state. +pub(crate) const SYNCED_STATE: &str = "synced_state"; +/// The variable naming a control store other than the project's own. +pub(crate) const CONTROL_STORE_VARIABLE: &str = "SQLBUILD_CONTROL_DATABASE_URL"; +/// The project's own control store, beside its other `sqb dev` state. +pub(crate) const CONTROL_STORE_FILE: &str = ".sqlbuild/control.sqlite3"; +/// The adapter method that checks a synced copy against the warehouse; only Snowflake has it. +pub(crate) const RELATION_CHANGES_METHOD: &str = "read_relation_changes"; +/// The adapter attribute inspection reads the synced catalog from. +pub(crate) const SYNCED_CATALOG_ATTRIBUTE: &str = "synced_catalog"; diff --git a/crates/sqlbuild-python-project/src/native_host/main/attach_synced_catalog.rs b/crates/sqlbuild-python-project/src/native_host/main/attach_synced_catalog.rs new file mode 100644 index 0000000000..8ed31f9f80 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/attach_synced_catalog.rs @@ -0,0 +1,66 @@ +//! Give the adapter the target's synced catalog when a plan reads from synced state. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyList, PyTuple}; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_orchestration::warehouse_metadata::models::SyncedCatalogRelation; + +use crate::native_host::_helpers::synced_catalog::{ + planning_live, plans_from_synced_state, synced_relations, synced_row, +}; +use crate::native_host::constants::{RELATION_CHANGES_METHOD, SYNCED_CATALOG_ATTRIBUTE}; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Give `adapter` the target's synced catalog when the plan reads from synced state. +pub fn attach_synced_catalog( + adapter: &Bound<'_, PyAny>, + (discovered, compiled, options): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + if adapter.hasattr(SYNCED_CATALOG_ATTRIBUTE)? { + adapter.setattr(SYNCED_CATALOG_ATTRIBUTE, py.None())?; + } + let target: Bound<'_, PyAny> = + native_field(&native_field(compiled, "project")?, "effective_target_name")?; + if !plans_from_synced_state(discovered, options, &target)? { + return Ok(()); + } + let checks: bool = adapter.hasattr(RELATION_CHANGES_METHOD)? + && adapter.getattr(RELATION_CHANGES_METHOD)?.is_callable(); + if !checks { + let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; + return planning_live( + py, + &format!("the {adapter_name} adapter cannot check a synced copy"), + ); + } + let project_dir: Option = native_field(discovered, "project_dir")?.extract()?; + let target: Option = target.extract()?; + let (Some(project_dir), Some(target)) = (project_dir, target) else { + return planning_live(py, "the project directory or target is unknown"); + }; + let loaded: Result, String> = + py.detach(|| synced_relations(&project_dir, &target)); + match loaded { + Ok(relations) if !relations.is_empty() => { + let rows: Vec> = relations + .iter() + .map(|relation| synced_row(py, relation)) + .collect::>()?; + let catalog: Bound<'_, PyAny> = call_with( + &host_entry(py, AdapterHost::SyncedCatalog)?.getattr("from_rows")?, + &[("rows", PyList::new(py, rows)?.into_any())], + )?; + adapter.setattr(SYNCED_CATALOG_ATTRIBUTE, catalog) + } + Ok(_) => planning_live( + py, + "the control store holds no synced catalog for this target", + ), + Err(reason) => planning_live(py, &reason), + } +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/mod.rs b/crates/sqlbuild-python-project/src/native_host/main/mod.rs index 2da4595b3e..97b1fd6b68 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/mod.rs @@ -1,4 +1,6 @@ +pub mod attach_synced_catalog; pub mod compile_calendars; pub mod compile_sensors; pub mod register_runs_state; +pub mod report_synced_catalog; pub mod run_recorder; diff --git a/crates/sqlbuild-python-project/src/native_host/main/report_synced_catalog.rs b/crates/sqlbuild-python-project/src/native_host/main/report_synced_catalog.rs new file mode 100644 index 0000000000..7fd881d59e --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/report_synced_catalog.rs @@ -0,0 +1,22 @@ +//! Write what a plan from synced state found when it checked the catalog. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +use crate::native_host::_helpers::synced_catalog::write_stderr; +use crate::native_host::constants::SYNCED_CATALOG_ATTRIBUTE; + +/// Write what the synced state checks found, once the plan is made. +pub fn report_synced_catalog(adapter: &Bound<'_, PyAny>) -> PyResult<()> { + if !adapter.hasattr(SYNCED_CATALOG_ATTRIBUTE)? { + return Ok(()); + } + let catalog: Bound<'_, PyAny> = adapter.getattr(SYNCED_CATALOG_ATTRIBUTE)?; + if catalog.is_none() { + return Ok(()); + } + let report: Option = catalog.call_method0("report")?.extract()?; + match report { + Some(line) => write_stderr(adapter.py(), &format!("{line}\n")), + None => Ok(()), + } +} diff --git a/crates/sqlbuild-python-project/src/native_host/mod.rs b/crates/sqlbuild-python-project/src/native_host/mod.rs index 9fbdf0b3ef..cbe3fe33b0 100644 --- a/crates/sqlbuild-python-project/src/native_host/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/mod.rs @@ -1,5 +1,6 @@ //! Model lookup and orchestration triggers. pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs index 3efb430ca5..48d875394f 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs @@ -2,7 +2,7 @@ use crate::semantic_shapes::main::declared_types::declared_types; use crate::semantic_shapes::main::function_names::function_names; -use std::collections::HashSet; +use std::collections::{BTreeSet, HashMap, HashSet}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -19,6 +19,8 @@ use crate::semantic_shapes::_helpers::shapes::model_inputs; use crate::semantic_shapes::_helpers::shapes::project_shapes; use crate::semantic_shapes::_helpers::shapes::published_shape; use crate::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; +use sqlbuild_analysis::analysis_route::main::same_column_type::same_column_type; +use sqlbuild_analysis::source_drift::main::reads_added_columns::reads_added_columns; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; @@ -91,7 +93,12 @@ pub(crate) fn source_binding_diagnostics<'py>( && EXACT_UNQUOTED_DIALECTS.contains(&dialect.extract::()?.as_str()); let folds_upper: bool = !dialect.is_none() && dialect.extract::()?.as_str() == UPPERCASE_FOLDING_DIALECT; - let mut changed: HashSet = HashSet::new(); + let comparison: Comparison = Comparison { + dialect: dialect.extract()?, + sensitive, + folds_upper, + }; + let mut changed: HashMap = HashMap::new(); for item in mapping_items(&columns)?.try_iter()? { let (name, value): (String, Object<'py>) = item?.extract()?; if !value.is_truthy()? { @@ -107,21 +114,29 @@ pub(crate) fn source_binding_diagnostics<'py>( )) }) .collect::>()?; - if sensitive || adds_physical_evidence(&physical, shapes.get_item(&name)?.as_ref())? { - changed.insert(name.clone()); + let bound: Vec<(String, Option)> = physical + .iter() + .map(|(column, column_type)| { + let listed_quoted: bool = folds_upper && unquoted_column_name(column) != column; + let name: String = if exact_unquoted + || listed_quoted + || (folds_upper && is_folded_identifier(column)) + { + column.clone() + } else { + format!("\"{}\"", column.replace('"', "\"\"")) + }; + (name, column_type.clone()) + }) + .collect(); + let known: Option> = shapes.get_item(&name)?; + let listed: &[(String, Option)] = if sensitive { &bound } else { &physical }; + if let Some(drift) = comparison.drift(listed, known.as_ref())? { + changed.insert(name.clone(), drift); } let quoted: Bound<'py, PyDict> = PyDict::new(py); - for (column, column_type) in physical { - let listed_quoted: bool = folds_upper && unquoted_column_name(&column) != column; - let name: String = if exact_unquoted - || listed_quoted - || (folds_upper && is_folded_identifier(&column)) - { - column - } else { - format!("\"{}\"", column.replace('"', "\"\"")) - }; - quoted.set_item(name, column_type)?; + for (column, column_type) in bound { + quoted.set_item(column, column_type)?; } shapes.set_item(name, quoted)?; } @@ -137,7 +152,7 @@ pub(crate) fn source_binding_diagnostics<'py>( &project, &request.get("selected_keys")?, (&changed, &shapes), - &profile, + (&profile, &comparison), )?; if !physical_catalog.is_none() { physical_catalog = physical_catalog.call_method1("with_relations", (&shapes,))?; @@ -145,40 +160,18 @@ pub(crate) fn source_binding_diagnostics<'py>( validated_diagnostics(&project, &models, (&shapes, &physical_catalog), &dialect) } -/// Selected models reading changed sources, re-analysed in dependency order; shapes updated. +/// Selected models whose result inspected columns could change, re-analysed in dependency order. fn rebound_models<'py>( project: &Object<'py>, selected_keys: &Object<'py>, - (changed, shapes): (&HashSet, &Bound<'py, PyDict>), - profile: &Object<'py>, + (changed, shapes): (&HashMap, &Bound<'py, PyDict>), + (profile, comparison): (&Object<'py>, &Comparison), ) -> PyResult>> { let py: Python<'py> = project.py(); - let by_name: Bound<'py, PyDict> = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { - let model: Object<'py> = model?; - by_name.set_item(native_field(&model, "name")?, &model)?; - } - let graph: Bound<'py, PyDict> = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { - let model: Object<'py> = model?; - let mut upstream: Vec> = Vec::new(); - for reference in native_field(&model, "references")?.try_iter()? { - let reference: Object<'py> = reference?; - let name: Object<'py> = native_field(&reference, "ref_name")?; - if native_field(&reference, "ref_kind")?.eq(REF_KIND)? && by_name.contains(&name)? { - upstream.push(name); - } - } - graph.set_item(native_field(&model, "name")?, PyTuple::new(py, upstream)?)?; - } - let order: Object<'py> = py - .import(GRAPHLIB_MODULE)? - .getattr("TopologicalSorter")? - .call1((graph,))? - .call_method0("static_order")?; + let (by_name, order) = dependency_order(project)?; let query_class: Object<'py> = record_class(py, RecordId::NativeColumnQuery)?; let unknown: Object<'py> = enum_member(py, RecordId::InferredNullability, "UNKNOWN")?; - let mut affected: HashSet = HashSet::new(); + let mut affected: HashMap = HashMap::new(); let mut models: Vec> = Vec::new(); for name in order.try_iter()? { let model: Object<'py> = by_name.as_any().get_item(name?)?; @@ -189,23 +182,11 @@ fn rebound_models<'py>( { continue; } - let mut reads_changed: bool = false; - for reference in native_field(&model, "references")?.try_iter()? { - let reference: Object<'py> = reference?; - let kind: String = native_field(&reference, "ref_kind")?.str()?.extract()?; - let upstream: String = native_field(&reference, "ref_name")?.extract()?; - if (kind == SOURCE_KIND && changed.contains(&upstream)) - || (kind == REF_KIND && affected.contains(&upstream)) - { - reads_changed = true; - break; - } - } - if !reads_changed { + if !needs_reanalysis(&model, (changed, &affected), comparison)? { continue; } let model_name: String = native_field(&model, "name")?.extract()?; - affected.insert(model_name.clone()); + let known: Option> = shapes.get_item(&model_name)?; models.push(model.clone()); let inputs: Bound<'py, PyDict> = model_inputs(&model, shapes)?; let nullability: Bound<'py, PyDict> = PyDict::new(py); @@ -258,17 +239,86 @@ fn rebound_models<'py>( }; columns.set_item(native_field(&column, "name")?, column_type)?; } - shapes.set_item( - &model_name, - published_shape(&model, &columns, &inputs, profile)?, - )?; - } else if shapes.contains(&model_name)? { - shapes.del_item(&model_name)?; + let shape: Bound<'py, PyDict> = published_shape(&model, &columns, &inputs, profile)?; + let listed: Vec<(String, Option)> = mapping_items(&shape)? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()?; + if let Some(drift) = comparison.drift(&listed, known.as_ref())? { + affected.insert(model_name.clone(), drift); + } + shapes.set_item(&model_name, shape)?; + } else { + affected.insert(model_name.clone(), Drift::Changed); + if shapes.contains(&model_name)? { + shapes.del_item(&model_name)?; + } } } Ok(models) } +/// The project's models by name, and their names in dependency order. +fn dependency_order<'py>(project: &Object<'py>) -> PyResult<(Bound<'py, PyDict>, Object<'py>)> { + let py: Python<'py> = project.py(); + let by_name: Bound<'py, PyDict> = PyDict::new(py); + for model in native_field(project, "models")?.try_iter()? { + let model: Object<'py> = model?; + by_name.set_item(native_field(&model, "name")?, &model)?; + } + let graph: Bound<'py, PyDict> = PyDict::new(py); + for model in native_field(project, "models")?.try_iter()? { + let model: Object<'py> = model?; + let mut upstream: Vec> = Vec::new(); + for reference in native_field(&model, "references")?.try_iter()? { + let reference: Object<'py> = reference?; + let name: Object<'py> = native_field(&reference, "ref_name")?; + if native_field(&reference, "ref_kind")?.eq(REF_KIND)? && by_name.contains(&name)? { + upstream.push(name); + } + } + graph.set_item(native_field(&model, "name")?, PyTuple::new(py, upstream)?)?; + } + let order: Object<'py> = py + .import(GRAPHLIB_MODULE)? + .getattr("TopologicalSorter")? + .call1((graph,))? + .call_method0("static_order")?; + Ok((by_name, order)) +} + +/// Whether `model` reads a changed input, or inputs whose added columns could change its result. +fn needs_reanalysis( + model: &Object<'_>, + (changed, affected): (&HashMap, &HashMap), + comparison: &Comparison, +) -> PyResult { + let mut drifted: Vec = Vec::new(); + let mut added: BTreeSet = BTreeSet::new(); + for reference in native_field(model, "references")?.try_iter()? { + let reference: Object<'_> = reference?; + let kind: String = native_field(&reference, "ref_kind")?.str()?.extract()?; + let upstream: String = native_field(&reference, "ref_name")?.extract()?; + let drift: Option<&Drift> = match kind.as_str() { + SOURCE_KIND => changed.get(&upstream), + REF_KIND => affected.get(&upstream), + _ => None, + }; + match drift { + Some(Drift::Changed) => return Ok(true), + Some(Drift::Added(names)) => { + drifted.push(upstream); + added.extend(names.iter().cloned()); + } + None => {} + } + } + if drifted.is_empty() { + return Ok(false); + } + comparison.reads_added(model, &drifted, &added.into_iter().collect::>()) +} + /// Each rebound model's binding diagnostics, located in its authored file. fn validated_diagnostics<'py>( project: &Object<'py>, @@ -410,40 +460,99 @@ fn positions<'py>( Ok(Bound::new(py, BindingPositions::new(request.extract()?)?)?.into_any()) } -/// Whether inspected columns differ from the known shape; untyped names match any type. -fn adds_physical_evidence( - physical: &[(String, Option)], - known: Option<&Object<'_>>, -) -> PyResult { - let Some(known) = known else { - return Ok(true); - }; - let known: Vec<(String, String)> = mapping_items(known)? - .try_iter()? - .map(|item| item?.extract()) - .collect::>()?; - let physical_names: HashSet<&str> = physical.iter().map(|(name, _)| name.as_str()).collect(); - let known_names: HashSet<&str> = known.iter().map(|(name, _)| name.as_str()).collect(); - if physical_names != known_names { - return Ok(true); +/// How a relation's columns differ from what compiling assumed of them. +enum Drift { + /// Every known column is there as it was, and these columns are new. + Added(Vec), + /// A known column is missing, retyped or listed twice, or nothing was known. + Changed, +} + +/// How this dialect binds and stores columns, to compare listed columns with known ones. +struct Comparison { + dialect: Option, + /// Whether names bind case-sensitively, compared by `bound_identity`. + sensitive: bool, + folds_upper: bool, +} + +impl Comparison { + fn identity(&self, name: &str) -> String { + if self.sensitive { + bound_identity(name, self.folds_upper) + } else { + name.to_owned() + } } - for (name, physical_type) in physical { - let Some(physical_type) = physical_type.as_deref() else { - continue; + + /// How `listed` columns differ from the `known` shape; `None` when they bind and store alike. + fn drift( + &self, + listed: &[(String, Option)], + known: Option<&Object<'_>>, + ) -> PyResult> { + let Some(known) = known else { + return Ok(Some(Drift::Changed)); }; - if physical_type.is_empty() { - continue; + let known: HashMap = mapping_items(known)? + .try_iter()? + .map(|item| { + let (name, column_type): (String, String) = item?.extract()?; + Ok((self.identity(&name), column_type)) + }) + .collect::>()?; + let mut seen: HashSet = HashSet::new(); + let mut added: Vec = Vec::new(); + for (name, listed_type) in listed { + let identity: String = self.identity(name); + if !seen.insert(identity.clone()) { + return Ok(Some(Drift::Changed)); + } + match known.get(&identity) { + None => added.push(identity), + Some(known_type) => { + let listed_type: &str = listed_type.as_deref().unwrap_or_default(); + if !listed_type.is_empty() + && !same_column_type(self.dialect.as_deref(), known_type, listed_type) + { + return Ok(Some(Drift::Changed)); + } + } + } } - let known_type: Option<&String> = known - .iter() - .rev() - .find(|(known_name, _)| known_name == name) - .map(|(_, known_type)| known_type); - if known_type.is_some_and(|known_type| known_type != physical_type) { - return Ok(true); + if known.keys().any(|name| !seen.contains(name)) { + return Ok(Some(Drift::Changed)); } + Ok((!added.is_empty()).then_some(Drift::Added(added))) + } + + /// Whether `model`'s result could change once the `drifted` relations gain `added` columns. + fn reads_added( + &self, + model: &Object<'_>, + drifted: &[String], + added: &[String], + ) -> PyResult { + let Some(dialect) = self.dialect.as_deref() else { + return Ok(true); + }; + let query_sql: String = native_field(model, "query_sql")?.extract()?; + let placeholders: Object<'_> = placeholders(model)?; + let placeholders: Vec<(String, String)> = if placeholders.is_none() { + Vec::new() + } else { + mapping_items(&placeholders)? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()? + }; + Ok(reads_added_columns( + dialect, + (&query_sql, &placeholders), + drifted, + added, + )) } - Ok(false) } /// `{str(key): str(value)}` of a dict `placeholders` config, else None. @@ -462,6 +571,19 @@ fn placeholders<'py>(model: &Object<'py>) -> PyResult> { Ok(placeholders.into_any()) } +/// The column a name binds to on a case-sensitive dialect: quoted exactly, unquoted case-folded. +fn bound_identity(name: &str, folds_upper: bool) -> String { + match name + .strip_prefix('"') + .and_then(|rest| rest.strip_suffix('"')) + .filter(|inner| !inner.is_empty()) + { + Some(inner) => inner.replace("\"\"", "\""), + None if folds_upper => name.to_uppercase(), + None => name.to_lowercase(), + } +} + /// Whether a listed column is a plain identifier in one case, so it binds as an unquoted name. fn is_folded_identifier(name: &str) -> bool { let mut chars = name.chars(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index d16b491703..9cbf89ba27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -67,6 +67,7 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult( }; if !rules_ran_at_load() { let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; - let graph: Bound<'py, PyAny> = project_graph(&project)?; - run_rules_gate( - (discovered, adapter, expansion_reuse.as_any()), - (&options, &on_progress), - (&project, &graph), - )?; + timed(EVALUATING_RULES, || { + let graph: Bound<'py, PyAny> = project_graph(&project)?; + run_rules_gate( + (discovered, adapter, expansion_reuse.as_any()), + (&options, &on_progress), + (&project, &graph), + ) + })?; } planned( (discovered, adapter), @@ -297,10 +301,12 @@ fn planned<'py>( ) } None => { - let opened: Bound<'py, PyAny> = - connection_with_hooks(adapter, &connection_config, hooks)?; + let opened: Bound<'py, PyAny> = timed(CONNECTING, || { + connection_with_hooks(adapter, &connection_config, hooks) + })?; connection = Some(opened.clone()); let reads = LiveReads::new(adapter, &opened, None).handle(adapter.py())?; + attach_synced_catalog(adapter, (discovered, compiled, options))?; (opened, reads) } }; @@ -325,6 +331,7 @@ fn planned<'py>( tracker.setattr("planning_seconds", planning_seconds)?; } let result: Bound<'py, PyAny> = result?; + report_synced_catalog(adapter)?; write_plan_capture(&native_field(&result, "plan_output")?)?; replaced( &result, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index f556259c9f..f2e7efbe18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -168,6 +168,9 @@ fn python_nodes( ))); }; published.definition["deps"] = json!(deps); + if let Some(sets) = selected.set_dependencies(&node.name) { + published.definition["set_dependencies"] = sets; + } published.definition["tags"] = json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); if let Some(source) = selected.intermediate_source(&node.kind, &node.name) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index d2e0d8eeb5..d9fc02c4d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -1,8 +1,7 @@ //! `sqb plan`: compile the plan pipeline and print the plan as text or JSON. use crate::bindings::_helpers::command_runtime::audit_failures::resolve_ignore_audit_failures; -use sqlbuild_python_planner::planner_phases::constants::BUILD_HISTORY; -use sqlbuild_python_planner::planner_phases::constants::WRITING; +use sqlbuild_python_planner::planner_phases::constants::{BUILD_HISTORY, DISCOVERING, WRITING}; use sqlbuild_python_planner::planner_phases::main::timing::timed::timed; use sqlbuild_python_planner::planner_phases::models::PlanTimingScope; use sqlbuild_python_planner::planner_phases::models::PlanTimings; @@ -66,7 +65,7 @@ const DEFAULT_SECTION_ENTRIES: i64 = 50; const ERROR_SEVERITY: &str = "error"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; /// Request fields passed to the compile pipeline under the same name. -const PIPELINE_FIELDS: [&str; 12] = [ +const PIPELINE_FIELDS: [&str; 13] = [ "no_sql_validation", "no_cache", "verbose", @@ -79,6 +78,7 @@ const PIPELINE_FIELDS: [&str; 12] = [ "cli_vars", "max_microbatches", "selection_diagnostics", + "plan_from", ]; fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { @@ -164,16 +164,18 @@ fn resolve_invocation<'py>( }; let discovered: Bound<'py, PyAny> = match project { PlanProject::Held { discovered, .. } => discovered.clone(), - PlanProject::Compiled => discover_project_inputs( - py, - Some(keywords( + PlanProject::Compiled => timed(DISCOVERING, || { + discover_project_inputs( py, - &[ - ("project_dir", project_dir.clone()), - ("extract_output_column_locations", flag(py, false)), - ], - )?), - )?, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", flag(py, false)), + ], + )?), + ) + })?, }; validate_preview_target( py, diff --git a/fensu.toml b/fensu.toml index 6058ce6bab..9fec3e7dee 100644 --- a/fensu.toml +++ b/fensu.toml @@ -92,6 +92,11 @@ modules = ["sqlbuild.adapter.discovery.main.resolve_adapter"] symbols = ["resolve_adapter"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.classes.synced_catalog"] +symbols = ["SyncedCatalog"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.adapter.relations.main._get_columns_for_inspection"] symbols = ["get_columns_for_inspection"] @@ -669,7 +674,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl [[targets.python.dead_code.roots]] modules = ["sqlbuild.python_nodes.models"] -symbols = ["AuditCase", "SqlResourceRef"] +symbols = ["AuditCase", "NodeSetRef", "SqlResourceRef"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] @@ -1490,6 +1495,12 @@ path = "src/sqlbuild/runtime/user_code/classes/user_code_finder.py" symbols = ["UserCodeFinder.find_spec"] reason = "importlib calls meta-path finders positionally as find_spec(fullname, path, target)." +[[targets.python.rule_exceptions]] +rule = "FFS120" +path = "src/sqlbuild/refs.py" +symbols = ["job", "tag"] +reason = "Like model(name), job and tag take the job or tag name positionally, as authors write depends_on=job(\"orders_job\")." + [[targets.python.rule_exceptions]] rule = "FFS120" path = "src/sqlbuild/assets.py" @@ -1960,6 +1971,12 @@ path = "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection. symbols = ["connection_config"] reason = "A once-per-process warning is process state by definition: a command resolves its connection at startup for provider setup and again for its own work, and threading a flag through every caller would not guarantee once. Approved 17 Nov for this module only." +[[targets.rust.rule_exceptions]] +rule = "FPRSS130" +path = "crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs" +symbols = ["marker_memo"] +reason = "Verifying typed markers parses the model once per marked call; the result depends only on the SQL, the calls and the dialect, so a bounded process-wide memo serves a compile's repeated reads and recompiles in long-lived processes." + [[targets.rust.rule_exceptions]] rule = "FPRSS130" path = "crates/sqlbuild-python-base/src/native_threads/_helpers/exit_signal.rs" @@ -2602,7 +2619,7 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs=0", "crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs=2", "crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs=1", - "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=11", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=7", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs=6", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs=3", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs=2", @@ -3149,6 +3166,7 @@ module_layers = [ "native_host/live_reads=plan", "native_host/orchestration_sensors=commands", "native_host/setup_context=commands", + "native_host/synced_catalog=plan", "native_host/warehouse_metadata=commands", "observations=plan", "output_capture=registry", @@ -3992,3 +4010,15 @@ reason = "Each registers its own module's Python functions; only the add-functio [[dupes.allowlist]] paths = ["crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs"] reason = "source_start and source_end apply opposite boundary rules: a start inside a rewrite maps to the start of the replaced text, while an exclusive end inside or closing one maps to its end, and an end at a removal maps to where the removed text began. Only the partition-and-shift shape is alike." + +[[dupes.allowlist]] +paths = ["src/sqlbuild/python_nodes/main/_apply_loader.py", "src/sqlbuild/python_nodes/main/apply_asset.py", "src/sqlbuild/python_nodes/main/apply_check.py", "src/sqlbuild/python_nodes/main/apply_hook.py", "src/sqlbuild/python_nodes/main/apply_task.py", "src/sqlbuild/assets.py", "src/sqlbuild/python_nodes/_helpers/loader_authoring.py"] +reason = "Each is one public Python node decorator with its own keyword arguments and definition record. Dependency normalisation and description resolution go through the shared normalize_python_node_dependencies and resolve_description; only the decorate-and-attach shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/rules/_helpers/configuration/catalogue.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs"] +reason = "Each reads a two-field key from a different record type (a SqlResourceRef's kind value, a CompiledObjectKey's resource type, a rule catalogue or graph row); only the read-two-fields shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-orchestration/src/credentials/main/locations/location_named.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_development_location.rs"] +reason = "One reads the active location with a name, the other the development location of a target whose project path matches; each is one control-store read with its own SQL and filter, and only the read-then-decode-the-identity shape is alike." diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 5ccea0bc09..129b506831 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -64,6 +64,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--reach` | With `--changed`: `job` stops at models another scheduled job refreshes, `all` builds everything downstream | | `--defer-to` | Resolve unselected model references against another target | | `--defer-sources-to` | Read managed source data from another target | +| `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](../concepts/project-configuration.md#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](../concepts/testing-reference.md#time-limits) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index a68f01bd5f..66d69eead9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -34,6 +34,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--no-python` | Exclude read-side Python tasks and assets from the plan | | `--defer-to` | Resolve unselected model references against another target | +| `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](../concepts/project-configuration.md#planning-from-synced-state) | | `--as` | Compile for another target and show its plan, including [model migration](../concepts/models/migrations.md) decisions, using the current connection. Inspection only. | | `--json` | Output the plan as JSON | | `--timings` | After the plan, print how long each planning step took; see [Timings](#timings) | @@ -142,20 +143,23 @@ When a removed column and an added column look related but are not an exact rena `--timings` prints how long each planning step took, in pipeline order, after the plan: ``` -Plan timings seconds share - Compiling the project 2.1 20% - Reading warehouse state 3.3 31% - Detecting changes 2.4 23% - Checking source freshness 0.2 2% - Building plan entries 1.6 15% - Preparing test fixtures 0.1 1% - Assembling the plan 0.4 4% - Writing the plan 0.3 3% - Other 0.2 2% - Total 10.6 +Plan timings seconds share + Discovering the project 1.2 10% + Compiling the project 2.1 17% + Evaluating rules 0.6 5% + Connecting to the warehouse 0.3 2% + Reading warehouse state 3.3 26% + Detecting changes 2.4 19% + Checking source freshness 0.2 2% + Building plan entries 1.6 13% + Preparing test fixtures 0.1 1% + Assembling the plan 0.4 3% + Writing the plan 0.3 2% + Other 0.1 1% + Total 12.6 ``` -`Reading build history` appears with `--changed`. `Preparing test fixtures` counts only the time the plan waits for them, because they are prepared alongside the other steps. `Other` is the time no step covers. A plan a location makes for the web UI records the same steps in its plan document. +`Reading build history` appears with `--changed`. A plan a location makes for the web UI from its synced warehouse state uses the project it already discovered, compiled and checked against the rules, and opens no connection, so those steps do not appear there. `Preparing test fixtures` counts only the time the plan waits for them, because they are prepared alongside the other steps. `Other` is the time no step covers. A plan a location makes for the web UI records the same steps in its plan document. ## Exit codes diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 57d1c97021..6b7b5893af 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -188,6 +188,7 @@ A target is a named build context - the database and schema you build into, plus | `warehouses` | Snowflake warehouse per command group: `build`, `query` and `sync` (a location's warehouse syncs). See [Command-group warehouses](adapters/snowflake.md#command-group-warehouses) | | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](audits.md#ignoring-audit-failures) | +| `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | ```toml [connections.warehouse] @@ -281,6 +282,37 @@ The value must be an integer of at least `1`; anything else fails with `D001`. ` `8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and uses the active target's value. +### Planning from synced state + +Every plan reads the warehouse's catalog: which tables and views exist and their columns. On a +large project that is most of a plan's time. A target can instead plan from the copy of the +catalog that `sqb dev` keeps synced in the project's control store, after checking it against +the warehouse: + +```toml +[targets.dev] +schema = "analytics_dev" +plan_from = "synced_state" +``` + +`--plan-from live` or `--plan-from synced_state` on `sqb plan` or `sqb build` overrides the target +for one invocation. The default is `live`; SQLBuild cannot tell which target is production, so +planning from synced state is something a target opts into. + +At the start of the plan, one `INFORMATION_SCHEMA.TABLES` read per database lists the schemas the +copy holds, with each table's last DDL. The plan uses that listing, so tables created or dropped +since the last sync are seen. It reads columns from the copy, except for tables whose last DDL is +later than the copy's observation of them (allowing five minutes for clock skew), which it +describes afresh. When more than 128 tables changed, it reads the schema's columns live instead. +Cursor bounds and incremental state are still read live. + +The plan reads live, and says why on stderr (`Planning live: ...`), when the target's adapter +isn't Snowflake (the only adapter that can check the copy), the project has no control store, no +location syncs the target for this project, the copy is empty, or the check fails. Otherwise it reports `Planned from synced state: checked in 1.2s, 3 tables refreshed`. The +control store is `.sqlbuild/control.sqlite3` in the project, or the database +`SQLBUILD_CONTROL_DATABASE_URL` names. Planning never writes to it; only the location's sync +refreshes the copy. + ### Clone policies Targets can declare whether they allow cloning to or from: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index d5991ed34e..1d108783cb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -133,7 +133,7 @@ Every decorator accepts the same organizational metadata: | Argument | Description | |----------|-------------| -| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references) | +| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references or a `job()`/`tag()`; see [SQL References](sql-references.md#depending-on-a-job-or-a-tag)) | | `tags` | Labels for selection, filtering, and catalog grouping | | `group` | A display/catalog grouping string | | `description` | Required human-readable docs; defaults to the function docstring ([Descriptions](../models/configuration.md#descriptions)) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md index 54c656c706..44a6114391 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/sql-references/ - Declaring a reference - Resolving with ctx.relation - Hard-coded relation names +- Depending on a job or a tag - model vs source - Where references are allowed @@ -119,6 +120,61 @@ This is a guard against accidental hard-coding, not a sandbox. SQL sent directly the project's sources. To turn both checks off while migrating a project, set [`[references] enforce_explicit = false`](../project-configuration.md#references). +## Depending on a job or a tag + +Naming every upstream model ties a node to model names that change with each rename or split. +`job()` and `tag()` from `sqlbuild.refs` depend on a whole set instead: + +```python +from sqlbuild.refs import job, tag +from sqlbuild.tasks import task + +@task(depends_on=(job("orders_job", exclude="tag:experimental"), tag("finance_inputs"))) +def publish_revenue(ctx): + """Publish revenue.""" + ... +``` + +- `job(name)` depends on the job's **end nodes**: the models, seeds, sources, loaders, tasks and + assets of its selection that nothing else in the selection reads. Everything else in the job is + upstream of an end node, so this waits for the whole job without listing every edge. A job group + name depends on the union of its jobs. +- `tag(name)` depends on the end nodes of every node carrying the tag. +- `exclude=` takes the usual [selectors](../selectors.md) (names, `tag:`, `path:`, + `kind:`) and removes those nodes before the set is reduced to its end nodes. + +Compile resolves each into ordinary edges to the end nodes, so planning, `--changed`, selector +expansion (`+task:publish_revenue`), `sqb dag` and job group waits treat them like named +dependencies. A model named directly and also reached through a job is one edge. Jobs resolve +whatever targets they are declared for. + +The edges only order the node. `ctx.relation(...)` still resolves only the `model()`, `source()` +and `seed()` references the node names, and column lineage stops at these edges. + +When a job's or tag's end nodes change, the plan says so under the node, against the end nodes +it was last built with: + +```text +Python read-side (1) +└── publish_revenue task (unchanged) + └── publish_revenue now also waits on model 'refund_totals' through job 'orders_job' +``` + +`--json` lists the same lines as `wait_changes` on the Python node. + +Compile fails when: + +- the job or tag does not exist, or the set is empty after `exclude`; +- a node of the set reads the dependant's output, directly or through other nodes. The error names + the node that reads back and suggests moving it to another job or excluding it; +- a [job group](../orchestration.md#group-jobs) would wait in a loop: job groups wait job + by job, so if the dependant's job waits for the job it depends on while that job reads a node of + the dependant's job, neither can start. The error names the nodes that close the loop; move + them to another job; +- a loader declares one: loaders depend on loaders, tasks and assets named one by one; +- the name has a code location prefix such as `job("billing:nightly_job")`, which is reserved for + depending on other locations' jobs. + ## model vs source | Reference | Resolves to | @@ -131,7 +187,7 @@ set [`[references] enforce_explicit = false`](../project-configuration.md#refere ## Where references are allowed -- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`. +- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`, and depend on a `job()` or `tag()`. - **Python hooks** declare the models, sources, and seeds they read with `@hook(reads=...)`; see [Python Hooks](../models/hooks/python.md#declared-reads). - **Loaders** may not reference SQL resources; they read sources with `ctx.source(...)` and upstream loaders with `ctx.loader(...)`. - A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index cb48515dc1..7b61bd1b3a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6568,6 +6568,7 @@ class BuildCommandRequest(NativeRecord): defer_to: str | None defer_clone_from: str | None defer_sources_to: str | None + plan_from: str | None selected_target: str | None cursor_overrides: CursorOverrides | None no_color: bool @@ -6611,6 +6612,7 @@ class BuildCommandRequest(NativeRecord): defer_to: str | None = None, defer_clone_from: str | None = None, defer_sources_to: str | None = None, + plan_from: str | None = None, selected_target: str | None = None, cursor_overrides: CursorOverrides | None = None, no_color: bool = False, @@ -6979,6 +6981,7 @@ class PlanCommandRequest(NativeRecord): no_sql_validation: bool defer_to: str | None defer_sources_to: str | None + plan_from: str | None selected_target: str | None as_target: str | None cursor_overrides: CursorOverrides | None @@ -7005,6 +7008,7 @@ class PlanCommandRequest(NativeRecord): no_sql_validation: bool = False, defer_to: str | None = None, defer_sources_to: str | None = None, + plan_from: str | None = None, selected_target: str | None = None, as_target: str | None = None, cursor_overrides: CursorOverrides | None = None, @@ -10242,6 +10246,7 @@ class CompilePipelineOptions(NativeRecord): accepts_unit_test_selectors: bool sql_test_selection: SqlTestSelection source_lexical_syntax: SqlLexicalSyntax | None + plan_from: str | None ignore_audit_failures: bool def __init__( self, @@ -10270,6 +10275,7 @@ class CompilePipelineOptions(NativeRecord): accepts_unit_test_selectors: bool = ..., sql_test_selection: SqlTestSelection = ..., source_lexical_syntax: SqlLexicalSyntax | None = ..., + plan_from: str | None = ..., ignore_audit_failures: bool = ..., ) -> None: ... @@ -11999,6 +12005,7 @@ class TargetConfig(NativeRecord): concurrency: int | None catalog_connection: str | None ignore_audit_failures: bool | None + plan_from: str | None def __init__( self, connection: dict[str, object] = ..., @@ -12024,6 +12031,7 @@ class TargetConfig(NativeRecord): concurrency: int | None = ..., catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., + plan_from: str | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -12052,6 +12060,7 @@ class LocalTargetConfig(NativeRecord): concurrency: int | None catalog_connection: str | None ignore_audit_failures: bool | None + plan_from: str | None def __init__( self, connection: dict[str, object] = ..., @@ -12078,6 +12087,7 @@ class LocalTargetConfig(NativeRecord): concurrency: int | None = ..., catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., + plan_from: str | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index c6f7428a5a..f215c77928 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -63,6 +63,7 @@ "sqlbuild.adapter.relations.main.record_inspection_query", "record_inspection_query", ), + "SyncedCatalog": ("sqlbuild.adapter.relations.classes.synced_catalog", "SyncedCatalog"), "build_relation_lookup": ( "sqlbuild.adapter.relations.main.relation_lookup", "build_relation_lookup", @@ -282,6 +283,7 @@ "AuditCase": ("sqlbuild.python_nodes.models", "AuditCase"), "RELATION_RENDERER": ("sqlbuild.python_nodes.models", "RELATION_RENDERER"), "SqlResourceRef": ("sqlbuild.python_nodes.models", "SqlResourceRef"), + "NodeSetRef": ("sqlbuild.python_nodes.models", "NodeSetRef"), "PythonCheckSeverity": ("sqlbuild.python_nodes.types", "PythonCheckSeverity"), "SqlResourceRefKind": ("sqlbuild.python_nodes.types", "SqlResourceRefKind"), "load_custom_rules": ( diff --git a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py index 870d551b88..bfb387c150 100644 --- a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py +++ b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py @@ -5,6 +5,7 @@ import threading import time from collections.abc import Callable +from dataclasses import replace from typing import Any, cast from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo @@ -13,18 +14,29 @@ current_relation_metadata_cache, ) from sqlbuild.adapter.relations.classes.relation_metadata_cache import RelationMetadataCache +from sqlbuild.adapter.relations.classes.remembered_flight import RememberedFlight +from sqlbuild.adapter.relations.classes.synced_catalog import ( + SyncedCatalog, +) from sqlbuild.adapter.relations.constants import ( INSPECTION_DESCRIBED_NAME_LIMIT, INSPECTION_IN_LIST_LIMIT, SCHEMA_LISTING_MAX_AGE_SECONDS, + SYNCED_CATALOG_ATTRIBUTE, + SYNCED_REFRESH_LIMIT, ) from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections from sqlbuild.adapter.relations.models import ( InspectionQueryRecord, + ListedRelation, + RelationChange, SchemaColumnListing, SchemaRelationListing, + SeededSchema, ) from sqlbuild.adapter.relations.types import ( + PartedRelationListingReader, + RelationChangeReader, RelationListingScoper, SchemaScopedMetadataReader, ViewDefinitionBatchReader, @@ -50,19 +62,37 @@ def __init__(self, *, adapter: Any, connection: Any) -> None: self._capped_column_schemas: set[_SchemaKey] = set() self._remembered: dict[tuple[str, ...], object] = {} self._remember_lock: threading.Lock = threading.Lock() + self._in_flight: dict[tuple[str, ...], RememberedFlight] = {} self._relation_requests: dict[_RelationRequestKey, tuple[RelationInfo, ...]] = {} self._relation_columns: dict[_RelationIdentity, tuple[ColumnInfo, ...] | None] = {} self._view_definitions: dict[_RelationIdentity, str | None] = {} + synced: object = getattr(adapter, SYNCED_CATALOG_ATTRIBUTE, None) + self._synced: SyncedCatalog | None = synced if isinstance(synced, SyncedCatalog) else None def remember[ValueT](self, *, key: tuple[str, ...], compute: Callable[[], ValueT]) -> ValueT: - """Compute one adapter-owned metadata fact at most once per planning invocation.""" + """Compute one adapter-owned metadata fact at most once; concurrent callers share it.""" with self._remember_lock: if key in self._remembered: return cast(ValueT, self._remembered[key]) - value: ValueT = compute() + flight: RememberedFlight | None = self._in_flight.get(key) + owner: bool = flight is None + if flight is None: + flight = RememberedFlight() + self._in_flight[key] = flight + if not owner: + return cast(ValueT, flight.wait()) + try: + value: ValueT = compute() + except BaseException as error: + with self._remember_lock: + del self._in_flight[key] + flight.fail(error) + raise with self._remember_lock: self._remembered.setdefault(key, value) + del self._in_flight[key] + flight.succeed(value) return value @property @@ -126,21 +156,156 @@ def prefetch_schema_listings( key: _SchemaKey = self._schema_key(database=database, schema=schema) if key not in self._relation_listings: missing.setdefault(key, (database, schema)) - listings: list[SchemaRelationListing | None] = run_bounded_inspections( - tasks=tuple( - self._bind_relation_listing( - reader=reader, database=database, schema=schema, best_effort=best_effort - ) - for database, schema in missing.values() - ), - concurrency=self._concurrency, - ) + for key in self._seed_synced(reader=reader, missing=missing): + del missing[key] + listings: list[SchemaRelationListing | None] + if isinstance(reader, PartedRelationListingReader) and ( + current_relation_metadata_cache(adapter=self.adapter) is None + ): + listings = self._read_parted_listings( + reader=reader, scopes=tuple(missing.values()), best_effort=best_effort + ) + else: + listings = run_bounded_inspections( + tasks=tuple( + self._bind_relation_listing( + reader=reader, database=database, schema=schema, best_effort=best_effort + ) + for database, schema in missing.values() + ), + concurrency=self._concurrency, + ) missing_key: _SchemaKey listing: SchemaRelationListing | None for missing_key, listing in zip(missing, listings, strict=True): if listing is not None: self._relation_listings[missing_key] = listing + def _seed_synced( + self, + *, + reader: SchemaScopedMetadataReader, + missing: dict[_SchemaKey, tuple[str | None, str]], + ) -> tuple[_SchemaKey, ...]: + """List the synced copy's schemas with one check per database, columns from the copy.""" + + synced: SyncedCatalog | None = self._synced + if synced is None or not isinstance(self.adapter, RelationChangeReader): + return () + by_database: dict[str | None, list[tuple[_SchemaKey, str]]] = {} + key: _SchemaKey + database: str | None + schema: str + for key, (database, schema) in missing.items(): + if synced.covers(database=database, schema=schema): + by_database.setdefault(database, []).append((key, schema)) + seeded: list[_SchemaKey] = [] + scoped: list[tuple[_SchemaKey, str]] + for database, scoped in by_database.items(): + started: float = time.monotonic() + try: + changes: dict[str, tuple[RelationChange, ...]] = self.adapter.read_relation_changes( + connection=self.connection, + database=database, + schemas=tuple(schema for _, schema in scoped), + ) + except Exception as error: + synced.record_failure(reason=f"the synced state check failed ({error})") + self._synced = None + return tuple(seeded) + for key, schema in scoped: + listed: tuple[RelationChange, ...] = changes.get( + reader.metadata_name_key(schema), () + ) + if not listed: + continue + seed: SeededSchema = synced.seed(database=database, schema=schema, changes=listed) + self._relation_listings[key] = replace(seed.relations, database=database) + if len(seed.stale) <= SYNCED_REFRESH_LIMIT: + try: + columns: dict[str, tuple[ColumnInfo, ...]] = self._refreshed_columns( + reader=reader, seed=seed + ) + except Exception as error: + del self._relation_listings[key] + synced.record_failure(reason=f"the synced state refresh failed ({error})") + self._synced = None + return tuple(seeded) + self._column_listings[key] = SchemaColumnListing( + database=database, schema=schema, columns_by_stored_name=columns + ) + seeded.append(key) + synced.record_check( + seconds=time.monotonic() - started, + relations=len(seed.relations.entries), + refreshed=len(seed.stale), + ) + return tuple(seeded) + + def _refreshed_columns( + self, *, reader: SchemaScopedMetadataReader, seed: SeededSchema + ) -> dict[str, tuple[ColumnInfo, ...]]: + """The synced columns, each relation whose definition changed since then read afresh.""" + + fresh: list[tuple[ColumnInfo, ...] | None] = run_bounded_inspections( + tasks=tuple( + self._bind_relation_columns(reader=reader, relation=stale.relation) + for stale in seed.stale + ), + concurrency=self._concurrency, + ) + columns: dict[str, tuple[ColumnInfo, ...]] = dict(seed.columns) + stale: ListedRelation + read: tuple[ColumnInfo, ...] | None + for stale, read in zip(seed.stale, fresh, strict=True): + if read is not None: + columns[stale.stored_name] = read + return columns + + def _read_parted_listings( + self, + *, + reader: PartedRelationListingReader, + scopes: tuple[tuple[str | None, str], ...], + best_effort: bool, + ) -> list[SchemaRelationListing | None]: + """Run every schema's listing reads together; a best-effort failure loses its schema.""" + + parts: list[tuple[int, Callable[[], object]]] = [] + failed: set[int] = set() + index: int + database: str | None + schema: str + for index, (database, schema) in enumerate(scopes): + try: + reads: tuple[Callable[[], object], ...] = reader.relation_listing_reads( + connection=self.connection, database=database, schema=schema + ) + except Exception: + if not best_effort: + raise + failed.add(index) + continue + parts.extend((index, read) for read in reads) + results: list[object] = run_bounded_inspections( + tasks=tuple(_guarded_read(read=read, best_effort=best_effort) for _, read in parts), + concurrency=self._concurrency, + ) + by_scope: dict[int, list[object]] = {} + result: object + for (index, _), result in zip(parts, results, strict=True): + if result is _FAILED_READ: + failed.add(index) + by_scope.setdefault(index, []).append(result) + return [ + None + if index in failed + else reader.relation_listing_from_reads( + database=database, schema=schema, results=tuple(by_scope.get(index, ())) + ) + for index, (database, schema) in enumerate(scopes) + ] + def prefetch_view_definitions( self, *, requests: tuple[tuple[str | None, str, tuple[str, ...]], ...] ) -> None: @@ -471,6 +636,23 @@ def _schema_key(self, *, database: str | None, schema: str) -> _SchemaKey: ) +_FAILED_READ: object = object() + + +def _guarded_read(*, read: Callable[[], object], best_effort: bool) -> Callable[[], object]: + """``read``, answering a failure with ``_FAILED_READ`` when the listing is best effort.""" + + def run() -> object: + try: + return read() + except Exception: + if best_effort: + return _FAILED_READ + raise + + return run + + def _flatten_relations(chunks: list[tuple[RelationInfo, ...]]) -> tuple[RelationInfo, ...]: relations: list[RelationInfo] = [] chunk: tuple[RelationInfo, ...] diff --git a/src/sqlbuild/adapter/relations/classes/remembered_flight.py b/src/sqlbuild/adapter/relations/classes/remembered_flight.py new file mode 100644 index 0000000000..c53abe6212 --- /dev/null +++ b/src/sqlbuild/adapter/relations/classes/remembered_flight.py @@ -0,0 +1,34 @@ +"""One remembered catalog computation in progress, which callers of the same key wait for.""" + +from __future__ import annotations + +import threading + + +class RememberedFlight: + """A computation's value or failure, shared with callers that asked while it ran.""" + + def __init__(self) -> None: + self._done: threading.Event = threading.Event() + self._value: object = None + self._error: BaseException | None = None + + def succeed(self, value: object) -> None: + """Share ``value`` with every waiting caller.""" + + self._value = value + self._done.set() + + def fail(self, error: BaseException) -> None: + """Raise ``error`` in every waiting caller.""" + + self._error = error + self._done.set() + + def wait(self) -> object: + """The computation's value once it finishes, raising its failure.""" + + _ = self._done.wait() + if self._error is not None: + raise self._error + return self._value diff --git a/src/sqlbuild/adapter/relations/classes/synced_catalog.py b/src/sqlbuild/adapter/relations/classes/synced_catalog.py new file mode 100644 index 0000000000..a05bcc153f --- /dev/null +++ b/src/sqlbuild/adapter/relations/classes/synced_catalog.py @@ -0,0 +1,121 @@ +"""A target's mirrored catalog, which a plan uses once the warehouse confirms it is current.""" + +from __future__ import annotations + +import sys +import threading +from datetime import datetime + +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.relations.constants import SYNCED_DDL_CLOCK_SKEW +from sqlbuild.adapter.relations.models import ( + ListedRelation, + RelationChange, + SchemaRelationListing, + SeededSchema, + SyncedRelation, +) +from sqlbuild.adapter.relations.types import SyncedRow + + +class SyncedCatalog: + """The mirror of one target's catalog, by lowercased database and schema.""" + + def __init__(self, *, relations: tuple[SyncedRelation, ...]) -> None: + self._schemas: dict[tuple[str | None, str], dict[str, list[SyncedRelation]]] = {} + relation: SyncedRelation + for relation in relations: + self._schemas.setdefault( + _scope_key(database=relation.database, schema=relation.schema), {} + ).setdefault(relation.name.lower(), []).append(relation) + self._lock: threading.Lock = threading.Lock() + self._checked: list[tuple[float, int, int]] = [] + self._failure: str | None = None + + @classmethod + def from_rows(cls, *, rows: list[SyncedRow]) -> SyncedCatalog: + """Each mirrored `(database, schema, name, columns, observed_at)` row as one catalog.""" + + return cls(relations=tuple(_synced_relation(row=row) for row in rows)) + + def covers(self, *, database: str | None, schema: str) -> bool: + """Whether the mirror holds any relation of this schema.""" + + return _scope_key(database=database, schema=schema) in self._schemas + + def seed( + self, *, database: str | None, schema: str, changes: tuple[RelationChange, ...] + ) -> SeededSchema: + """The schema as the check lists it, with mirrored columns where no DDL followed them.""" + + mirrored: dict[str, list[SyncedRelation]] = self._schemas.get( + _scope_key(database=database, schema=schema), {} + ) + entries: list[ListedRelation] = [] + columns: dict[str, tuple[ColumnInfo, ...]] = {} + stale: list[ListedRelation] = [] + names: dict[str, int] = {} + change: RelationChange + for change in changes: + names[change.listed.stored_name.lower()] = ( + names.get(change.listed.stored_name.lower(), 0) + 1 + ) + for change in changes: + entries.append(change.listed) + candidates: list[SyncedRelation] = mirrored.get(change.listed.stored_name.lower(), []) + current: bool = ( + len(candidates) == 1 + and names[change.listed.stored_name.lower()] == 1 + and change.last_ddl is not None + and change.last_ddl <= candidates[0].observed_at - SYNCED_DDL_CLOCK_SKEW + ) + if current: + columns[change.listed.stored_name] = candidates[0].columns + else: + stale.append(change.listed) + return SeededSchema( + relations=SchemaRelationListing( + database=database, schema=schema, entries=tuple(entries) + ), + columns=columns, + stale=tuple(stale), + ) + + def record_check(self, *, seconds: float, relations: int, refreshed: int) -> None: + """Note one check: how long it took, the relations it listed and those re-read.""" + + with self._lock: + self._checked.append((seconds, relations, refreshed)) + + def record_failure(self, *, reason: str) -> None: + """Note that the warehouse could not confirm the mirror, so the plan reads it live.""" + + with self._lock: + self._failure = self._failure or reason + print(f"Planning live: {reason}", file=sys.stderr, flush=True) + + def report(self) -> str | None: + """One line on what the checks found, or None when nothing was checked.""" + + with self._lock: + if not self._checked: + return None + seconds: float = sum(check[0] for check in self._checked) + refreshed: int = sum(check[2] for check in self._checked) + noun: str = "table" if refreshed == 1 else "tables" + return f"Planned from synced state: checked in {seconds:.1f}s, {refreshed} {noun} refreshed" + + +def _scope_key(*, database: str | None, schema: str) -> tuple[str | None, str]: + return (None if database is None else database.lower(), schema.lower()) + + +def _synced_relation(*, row: SyncedRow) -> SyncedRelation: + database, schema, name, columns, observed_at = row + return SyncedRelation( + database=database, + schema=schema, + name=name, + columns=tuple(ColumnInfo(name=column, type=kind) for column, kind in columns), + observed_at=datetime.fromisoformat(observed_at), + ) diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 89549a0934..6afe58c72e 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -1,5 +1,7 @@ """Shared relation metadata limits.""" +from datetime import timedelta + INSPECTION_DESCRIBED_NAME_LIMIT: int = 3 INSPECTION_IN_LIST_LIMIT: int = 50 METADATA_NAME_FILTER_LIMIT: int = 250 @@ -9,3 +11,6 @@ SCHEMA_LISTING_LOOKUP_THRESHOLD: int = 2 SCHEMA_LISTING_MAX_AGE_SECONDS: float = 60.0 UNKNOWN_RELATION_READER: str = "unknown user" +SYNCED_CATALOG_ATTRIBUTE: str = "synced_catalog" +SYNCED_REFRESH_LIMIT: int = 128 +SYNCED_DDL_CLOCK_SKEW: timedelta = timedelta(minutes=5) diff --git a/src/sqlbuild/adapter/relations/models.py b/src/sqlbuild/adapter/relations/models.py index d9762f4f60..9689310e37 100644 --- a/src/sqlbuild/adapter/relations/models.py +++ b/src/sqlbuild/adapter/relations/models.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import datetime from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo @@ -15,6 +16,15 @@ class ListedRelation: relation: RelationInfo +@dataclass(frozen=True) +class RelationChange: + """One listed relation and when its data and its definition last changed, as UTC.""" + + listed: ListedRelation + last_altered: datetime | None + last_ddl: datetime | None + + @dataclass(frozen=True) class SchemaRelationListing: """Every relation one database schema exposes to the inspecting role.""" @@ -61,3 +71,23 @@ class RunSchemaListing: global_version: int name_versions: dict[str, int] expires_at: float | None = None + + +@dataclass(frozen=True) +class SyncedRelation: + """One mirrored relation: its columns and when the catalog scan observed them.""" + + database: str | None + schema: str + name: str + columns: tuple[ColumnInfo, ...] + observed_at: datetime + + +@dataclass(frozen=True) +class SeededSchema: + """One checked schema's listing, its columns, and the relations whose columns are stale.""" + + relations: SchemaRelationListing + columns: dict[str, tuple[ColumnInfo, ...]] + stale: tuple[ListedRelation, ...] diff --git a/src/sqlbuild/adapter/relations/types.py b/src/sqlbuild/adapter/relations/types.py index 4863de5887..f5e9f0db57 100644 --- a/src/sqlbuild/adapter/relations/types.py +++ b/src/sqlbuild/adapter/relations/types.py @@ -5,13 +5,20 @@ from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable if TYPE_CHECKING: + from collections.abc import Callable + from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo - from sqlbuild.adapter.relations.models import SchemaColumnListing, SchemaRelationListing + from sqlbuild.adapter.relations.models import ( + RelationChange, + SchemaColumnListing, + SchemaRelationListing, + ) type RelationCacheKey = tuple[str, str | None, str | None, str] type RelationCacheVersion = tuple[int, int] type SchemaListingScope = tuple[str | None, str] +type SyncedRow = tuple[str | None, str, str, list[tuple[str, str]], str] class RelationLocation(Protocol): @@ -69,6 +76,34 @@ def metadata_name_key(self, name: str) -> str: ... +@runtime_checkable +class PartedRelationListingReader(Protocol): + """Schema-scoped reader whose relation listing is independent reads that may run together.""" + + def relation_listing_reads( + self, *, connection: Any, database: str | None, schema: str + ) -> tuple[Callable[[], object], ...]: + """The reads ``read_schema_relation_listing`` runs one after another for one schema.""" + ... + + def relation_listing_from_reads( + self, *, database: str | None, schema: str, results: tuple[object, ...] + ) -> SchemaRelationListing: + """The schema's listing from its reads' results, in ``relation_listing_reads`` order.""" + ... + + +@runtime_checkable +class RelationChangeReader(Protocol): + """Adapter that lists schemas with when each relation's data and definition last changed.""" + + def read_relation_changes( + self, *, connection: Any, database: str | None, schemas: tuple[str, ...] + ) -> dict[str, tuple[RelationChange, ...]]: + """Each schema's relations, by the schema's metadata name key, in one read.""" + ... + + @runtime_checkable class RelationChangeTokenReader(Protocol): """Adapter that reads a table's change token without starting warehouse compute.""" diff --git a/src/sqlbuild/adapters/snowflake/_helpers/metadata_types.py b/src/sqlbuild/adapters/snowflake/_helpers/metadata_types.py index 2d1846bc3c..bec56a4c72 100644 --- a/src/sqlbuild/adapters/snowflake/_helpers/metadata_types.py +++ b/src/sqlbuild/adapters/snowflake/_helpers/metadata_types.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from functools import lru_cache from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.snowflake.constants import NUMBER_TYPE_NAME, TEXT_TYPE_NAMES @@ -11,6 +12,7 @@ "FIXED": NUMBER_TYPE_NAME, "REAL": "FLOAT", } +_SHOW_TYPE_CACHE_SIZE: int = 4096 _INVALID_SHOW_TYPE_MESSAGE: str = "Snowflake SHOW COLUMNS returned invalid type metadata" @@ -38,8 +40,15 @@ def information_schema_type( def show_columns_type(raw_data_type: object) -> str: """Render the same type string as INFORMATION_SCHEMA for one SHOW COLUMNS data_type JSON.""" + return _show_type(str(raw_data_type)) + + +@lru_cache(maxsize=_SHOW_TYPE_CACHE_SIZE) +def _show_type(raw_data_type: str) -> str: + """`show_columns_type` of one JSON text; a schema repeats a few hundred texts in 100K+ rows.""" + try: - decoded: object = json.loads(str(raw_data_type)) + decoded: object = json.loads(raw_data_type) except (json.JSONDecodeError, TypeError) as error: raise AdapterUserError(message=_INVALID_SHOW_TYPE_MESSAGE) from error if not isinstance(decoded, dict) or not isinstance(decoded.get("type"), str): diff --git a/src/sqlbuild/adapters/snowflake/_helpers/relation_changes.py b/src/sqlbuild/adapters/snowflake/_helpers/relation_changes.py new file mode 100644 index 0000000000..0d64f0ac0a --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/relation_changes.py @@ -0,0 +1,50 @@ +"""Map INFORMATION_SCHEMA.TABLES rows onto the relations SHOW TABLES and SHOW VIEWS list.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc +from sqlbuild.adapter.relations.models import ListedRelation, RelationChange + +_RELATION_TYPES: dict[str, str] = { + "BASE TABLE": "base table", + "LOCAL TEMPORARY": "temporary table", + "TEMPORARY TABLE": "temporary table", + "EXTERNAL TABLE": "external table", + "EVENT TABLE": "event table", + "VIEW": "view", + "MATERIALIZED VIEW": "materialized view", +} +_VIEW_TYPES: frozenset[str] = frozenset({"view", "materialized view"}) +_YES: str = "YES" + + +def relation_change_from_row(*, row: tuple[Any, ...], database: str | None) -> RelationChange: + """One relation as SHOW would list it, with its last data change and last DDL in UTC.""" + + stored_name: str = str(row[0]) + table_type: str = str(row[2] or "").upper() + relation_type: str = _RELATION_TYPES.get(table_type, table_type.lower()) + view: bool = relation_type in _VIEW_TYPES + retention: str = str(row[7] if row[7] is not None else "").strip() + return RelationChange( + listed=ListedRelation( + stored_name=stored_name, + relation=RelationInfo( + database=database, + schema=str(row[1]).lower(), + name=stored_name.lower(), + relation_type=relation_type, + is_transient=None if view else str(row[3] or "").upper() == _YES, + created_at=relation_age_timestamp_utc(row[4]), + last_altered_at=None, + retention_days=None + if view or not retention.lstrip("-").isdigit() + else int(retention), + ), + ), + last_altered=relation_age_timestamp_utc(row[5]), + last_ddl=relation_age_timestamp_utc(row[6]), + ) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 65d20ccf5d..ed578f251e 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -121,6 +121,7 @@ ) from sqlbuild.adapter.relations.models import ( ListedRelation, + RelationChange, SchemaColumnListing, SchemaRelationListing, ) @@ -143,6 +144,7 @@ information_schema_type, show_columns_type, ) +from sqlbuild.adapters.snowflake._helpers.relation_changes import relation_change_from_row from sqlbuild.adapters.snowflake._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.snowflake._helpers.show_metadata import ( is_missing_object_error, @@ -186,6 +188,7 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS _EXACT_COLUMN_INSPECTION_LIMIT: int = 32 +_CONCURRENT_EXACT_COLUMN_INSPECTION_LIMIT: int = 200 _STATE_TABLE_RETENTION_ATTRIBUTE: str = "_state_table_retention_fallbacks_state" _METADATA_INSPECTION_CONCURRENCY: int = 8 _SHOW_RESULT_LIMIT: int = 10_000 @@ -201,6 +204,10 @@ "table_name, table_schema, table_type, is_transient, created, last_altered, retention_time" ) _BULK_COLUMN_RELATION_CHUNK_SIZE: int = 2_000 +_RELATION_CHANGE_COLUMNS: str = ( + "table_name, table_schema, table_type, is_transient, created, last_altered, last_ddl, " + "retention_time" +) class SnowflakeAdapter( @@ -223,7 +230,7 @@ class SnowflakeAdapter( statement_limit_excludes_queueing: ClassVar[bool] = True metadata_inspection_concurrency: ClassVar[int] = _METADATA_INSPECTION_CONCURRENCY concurrent_ddl_limit: ClassVar[int] = 16 - exact_column_inspection_limit: ClassVar[int] = _EXACT_COLUMN_INSPECTION_LIMIT + exact_column_inspection_limit: ClassVar[int] = _CONCURRENT_EXACT_COLUMN_INSPECTION_LIMIT max_identifier_length: ClassVar[int] = 255 supports_session_warehouse: ClassVar[bool] = True @@ -2270,6 +2277,34 @@ def _relation_info_from_row(*, row: tuple[Any, ...], database: str | None) -> Re retention_days=None if row[6] is None else int(row[6]), ) + def read_relation_changes( + self, *, connection: _SnowflakeConnection, database: str | None, schemas: tuple[str, ...] + ) -> dict[str, tuple[RelationChange, ...]]: + """List schemas from INFORMATION_SCHEMA.TABLES, whose LAST_DDL moves with each DDL.""" + + placeholders: str = ", ".join(["%s"] * len(schemas)) + query: str = ( + f"SELECT {_RELATION_CHANGE_COLUMNS} " + f"FROM {self._information_schema_relation(database=database, name='tables')} " + f"WHERE table_schema IN ({placeholders})" + ) + params: list[str] = [self._information_schema_identifier(schema) for schema in schemas] + if database is not None: + query += " AND table_catalog = %s" + params.append(self._information_schema_identifier(database)) + rows: list[tuple[Any, ...]] = self._fetch_inspection_rows( + connection=connection, query=query, params=tuple(params) + ) + changes: dict[str, list[RelationChange]] = { + self.metadata_name_key(schema): [] for schema in schemas + } + row: tuple[Any, ...] + for row in rows: + changes.setdefault(self.metadata_name_key(str(row[1])), []).append( + relation_change_from_row(row=row, database=database) + ) + return {schema: tuple(listed) for schema, listed in changes.items()} + def read_relation_change_token( self, *, connection: _SnowflakeConnection, database: str | None, schema: str, name: str ) -> str | None: @@ -2291,24 +2326,45 @@ def read_schema_relation_listing( ) -> SchemaRelationListing: """List one schema's tables and views with SHOW, which needs no running warehouse.""" + return self.relation_listing_from_reads( + database=database, + schema=schema, + results=tuple( + read() + for read in self.relation_listing_reads( + connection=connection, database=database, schema=schema + ) + ), + ) + + def relation_listing_reads( + self, *, connection: _SnowflakeConnection, database: str | None, schema: str + ) -> tuple[Callable[[], object], ...]: + """SHOW TABLES and SHOW VIEWS of one schema, independent reads that may run together.""" + show_database: str = self._show_database( connection=connection, database=database, schema=schema ) scope: str = self._show_schema_scope( connection=connection, database=database, schema=schema ) - tables: list[dict[str, object]] = self._fetch_paged_show_rows( - connection=connection, - query=f"SHOW TABLES IN SCHEMA {scope}", - show_database=show_database, - schema=schema, - ) - views: list[dict[str, object]] = self._fetch_paged_show_rows( - connection=connection, - query=f"SHOW VIEWS IN SCHEMA {scope}", - show_database=show_database, - schema=schema, + return tuple( + partial( + self._fetch_paged_show_rows, + connection=connection, + query=f"SHOW {kind} IN SCHEMA {scope}", + show_database=show_database, + schema=schema, + ) + for kind in ("TABLES", "VIEWS") ) + + def relation_listing_from_reads( + self, *, database: str | None, schema: str, results: tuple[object, ...] + ) -> SchemaRelationListing: + """One schema's listing from its SHOW TABLES and SHOW VIEWS rows.""" + + tables, views = cast(tuple[list[dict[str, object]], list[dict[str, object]]], results) entries: dict[str, ListedRelation] = {} row: dict[str, object] for row in tables: diff --git a/src/sqlbuild/assets.py b/src/sqlbuild/assets.py index 296ae1f5d2..c242b54be1 100644 --- a/src/sqlbuild/assets.py +++ b/src/sqlbuild/assets.py @@ -12,6 +12,7 @@ from sqlbuild.python_nodes.models import ( AssetDefinition, ColumnLineageRef, + NodeSetRef, RetryPolicy, SqlResourceRef, ) @@ -26,8 +27,9 @@ def asset( name: str | None = None, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef] = (), + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), tags: Sequence[str] = (), group: str | None = None, description: str | None = None, diff --git a/src/sqlbuild/checks.py b/src/sqlbuild/checks.py index 3f5872f6b3..8c14db63b7 100644 --- a/src/sqlbuild/checks.py +++ b/src/sqlbuild/checks.py @@ -7,7 +7,7 @@ from sqlbuild.executor.python_nodes.models import CheckContext, PythonCheckResult from sqlbuild.python_nodes.main.apply_check import apply_check from sqlbuild.python_nodes.main.read_check_definition import read_check_definition -from sqlbuild.python_nodes.models import CheckDefinition, SqlResourceRef +from sqlbuild.python_nodes.models import CheckDefinition, NodeSetRef, SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity __all__ = ("CheckContext", "PythonCheckResult", "check", "get_check_definition") @@ -17,8 +17,9 @@ def check( *, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef], + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef], name: str | None = None, severity: str | PythonCheckSeverity = PythonCheckSeverity.ERROR, tags: Sequence[str] = (), diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py index 9ea32d11dd..47a53cd8b3 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py @@ -17,7 +17,7 @@ from sqlbuild.python_nodes.main.read_check_definition import read_check_definition from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition from sqlbuild.python_nodes.main.read_task_definition import read_task_definition -from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.models import NodeSetRef, SqlResourceRef type _PythonNode = ( DiscoveredLoaderFunction @@ -69,6 +69,7 @@ def _node(*, kind: str, node: _PythonNode, owners: dict[object, int]) -> dict[st if isinstance(dependency, SqlResourceRef) else (None, owners.get(dependency), _decorated_name(dependency)) for dependency in node.depends_on + if not isinstance(dependency, NodeSetRef) ], } diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 74426fec85..4501569516 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -41,6 +41,7 @@ ) from sqlbuild.python_nodes.main.calculate_retry_delay import calculate_retry_delay from sqlbuild.python_nodes.models import ( + NodeSetRef, RetryPolicy, SqlResourceRef, ) @@ -367,7 +368,9 @@ def _missing_upstream_result( """Why the node cannot run: a Python upstream outside this run that has never succeeded.""" python_dependencies: tuple[Callable[..., object], ...] = tuple( - dependency for dependency in node.depends_on if not isinstance(dependency, SqlResourceRef) + dependency + for dependency in node.depends_on + if not isinstance(dependency, (SqlResourceRef, NodeSetRef)) ) if len(upstream_results) == len(python_dependencies): return None diff --git a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py index ec6adc9837..2ada3b7eff 100644 --- a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py +++ b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py @@ -2,16 +2,17 @@ from collections.abc import Callable -from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.models import NodeSetRef, SqlResourceRef def normalize_python_node_dependencies( value: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef], -) -> tuple[Callable[..., object] | SqlResourceRef, ...]: - if callable(value) or isinstance(value, SqlResourceRef): + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef], +) -> tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...]: + if callable(value) or isinstance(value, (SqlResourceRef, NodeSetRef)): return (value,) return tuple(value) diff --git a/src/sqlbuild/python_nodes/main/apply_asset.py b/src/sqlbuild/python_nodes/main/apply_asset.py index 708ea4220c..308482d44e 100644 --- a/src/sqlbuild/python_nodes/main/apply_asset.py +++ b/src/sqlbuild/python_nodes/main/apply_asset.py @@ -16,6 +16,7 @@ from sqlbuild.python_nodes.models import ( AssetDefinition, ColumnLineageRef, + NodeSetRef, RetryPolicy, SqlResourceRef, ) @@ -28,8 +29,9 @@ def apply_asset( name: str | None = None, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef] = (), + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), tags: Sequence[str] = (), group: str | None = None, description: str | None = None, @@ -40,7 +42,7 @@ def apply_asset( ) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: """Apply SQLBuild asset metadata to a Python function.""" - normalized_deps: tuple[Callable[..., object] | SqlResourceRef, ...] = ( + normalized_deps: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = ( normalize_python_node_dependencies(depends_on) ) diff --git a/src/sqlbuild/python_nodes/main/apply_check.py b/src/sqlbuild/python_nodes/main/apply_check.py index 7c2f8efc0e..643528cdca 100644 --- a/src/sqlbuild/python_nodes/main/apply_check.py +++ b/src/sqlbuild/python_nodes/main/apply_check.py @@ -10,7 +10,7 @@ normalize_python_node_dependencies, ) from sqlbuild.python_nodes._helpers.description_resolution import resolve_description -from sqlbuild.python_nodes.models import CheckDefinition, SqlResourceRef +from sqlbuild.python_nodes.models import CheckDefinition, NodeSetRef, SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity @@ -18,8 +18,9 @@ def apply_check( *, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef], + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef], name: str | None = None, severity: str | PythonCheckSeverity = PythonCheckSeverity.ERROR, tags: Sequence[str] = (), @@ -29,7 +30,7 @@ def apply_check( ) -> Callable[[Callable[..., object]], Callable[..., object]]: """Apply SQLBuild check metadata to a Python function.""" - normalized_deps: tuple[Callable[..., object] | SqlResourceRef, ...] = ( + normalized_deps: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = ( normalize_python_node_dependencies(depends_on) ) diff --git a/src/sqlbuild/python_nodes/main/apply_task.py b/src/sqlbuild/python_nodes/main/apply_task.py index 0abe98f455..83363f2b44 100644 --- a/src/sqlbuild/python_nodes/main/apply_task.py +++ b/src/sqlbuild/python_nodes/main/apply_task.py @@ -10,7 +10,7 @@ normalize_python_node_dependencies, ) from sqlbuild.python_nodes._helpers.description_resolution import resolve_description -from sqlbuild.python_nodes.models import RetryPolicy, SqlResourceRef, TaskDefinition +from sqlbuild.python_nodes.models import NodeSetRef, RetryPolicy, SqlResourceRef, TaskDefinition def apply_task( @@ -19,8 +19,9 @@ def apply_task( name: str | None = None, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef] = (), + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), tags: Sequence[str] = (), group: str | None = None, description: str | None = None, @@ -29,7 +30,7 @@ def apply_task( ) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: """Apply SQLBuild task metadata to a Python function.""" - normalized_deps: tuple[Callable[..., object] | SqlResourceRef, ...] = ( + normalized_deps: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = ( normalize_python_node_dependencies(depends_on) ) diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py index 6ecf319de8..7d47dde0bd 100644 --- a/src/sqlbuild/python_nodes/models.py +++ b/src/sqlbuild/python_nodes/models.py @@ -14,7 +14,7 @@ ) from sqlbuild.compiler.auditing.types import AuditSeverity from sqlbuild.python_nodes.constants import RELATION_RENDERER -from sqlbuild.python_nodes.types import PythonCheckSeverity, SqlResourceRefKind +from sqlbuild.python_nodes.types import NodeSetKind, PythonCheckSeverity, SqlResourceRefKind @dataclass(frozen=True) @@ -33,6 +33,21 @@ def __str__(self) -> str: return renderer(self) +@dataclass(frozen=True) +class NodeSetRef: + """A dependency on a job's or tag's end nodes, resolved into node edges at compile.""" + + kind: NodeSetKind + name: str + exclude: str | None = None + + def __str__(self) -> str: + """Render as the authoring call.""" + + exclude: str = "" if self.exclude is None else f', exclude="{self.exclude}"' + return f'{self.kind.value}("{self.name}"{exclude})' + + @dataclass(frozen=True) class ColumnLineageRef: """Graph-based upstream column reference for declared Python-node lineage.""" @@ -126,7 +141,7 @@ class TaskDefinition: """Metadata attached to a decorated SQLBuild task function.""" name: str - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = () + depends_on: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = () tags: tuple[str, ...] = () group: str | None = None description: str | None = None @@ -139,7 +154,7 @@ class AssetDefinition: """Metadata attached to a decorated SQLBuild asset function.""" name: str - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = () + depends_on: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] = () tags: tuple[str, ...] = () group: str | None = None description: str | None = None @@ -154,7 +169,7 @@ class CheckDefinition: """Metadata attached to a decorated SQLBuild check function.""" name: str - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + depends_on: tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] severity: PythonCheckSeverity = PythonCheckSeverity.ERROR tags: tuple[str, ...] = () group: str | None = None diff --git a/src/sqlbuild/python_nodes/types.py b/src/sqlbuild/python_nodes/types.py index ba01ad5969..5300b68fb0 100644 --- a/src/sqlbuild/python_nodes/types.py +++ b/src/sqlbuild/python_nodes/types.py @@ -38,6 +38,13 @@ class PythonCheckSeverity(StrEnum): WARN = "warn" +class NodeSetKind(StrEnum): + """What a set dependency names: a job's selection or every node carrying a tag.""" + + JOB = "job" + TAG = "tag" + + class SqlResourceRefKind(StrEnum): """Typed reference target for SQL graph resources.""" diff --git a/src/sqlbuild/refs.py b/src/sqlbuild/refs.py index ff596e8d74..94050fb166 100644 --- a/src/sqlbuild/refs.py +++ b/src/sqlbuild/refs.py @@ -1,9 +1,21 @@ """Public typed references for SQL graph resources.""" +from sqlbuild.python_nodes.models import NodeSetRef as NodeSetRef from sqlbuild.python_nodes.models import SqlResourceRef as SqlResourceRef +from sqlbuild.python_nodes.types import NodeSetKind as NodeSetKind from sqlbuild.python_nodes.types import SqlResourceRefKind as SqlResourceRefKind -__all__ = ("SqlResourceRef", "SqlResourceRefKind", "model", "seed", "source") +__all__ = ( + "NodeSetKind", + "NodeSetRef", + "SqlResourceRef", + "SqlResourceRefKind", + "job", + "model", + "seed", + "source", + "tag", +) def model(name: str) -> SqlResourceRef: @@ -22,3 +34,15 @@ def seed(name: str) -> SqlResourceRef: """Return a typed reference to a SQLBuild seed.""" return SqlResourceRef(kind=SqlResourceRefKind.SEED, name=name) + + +def job(name: str, *, exclude: str | None = None) -> NodeSetRef: + """Depend on a job's end nodes: its selection less `exclude` that nothing else in it reads.""" + + return NodeSetRef(kind=NodeSetKind.JOB, name=name, exclude=exclude) + + +def tag(name: str, *, exclude: str | None = None) -> NodeSetRef: + """Depend on the end nodes of every node carrying the tag, less `exclude`.""" + + return NodeSetRef(kind=NodeSetKind.TAG, name=name, exclude=exclude) diff --git a/src/sqlbuild/tasks.py b/src/sqlbuild/tasks.py index 007b573892..5ce20863f4 100644 --- a/src/sqlbuild/tasks.py +++ b/src/sqlbuild/tasks.py @@ -8,7 +8,7 @@ from sqlbuild.executor.python_nodes.models import TaskContext from sqlbuild.python_nodes.main.apply_task import apply_task from sqlbuild.python_nodes.main.read_task_definition import read_task_definition -from sqlbuild.python_nodes.models import RetryPolicy, SqlResourceRef, TaskDefinition +from sqlbuild.python_nodes.models import NodeSetRef, RetryPolicy, SqlResourceRef, TaskDefinition __all__ = ("SkipMode", "TaskContext", "get_task_definition", "task") @@ -19,8 +19,9 @@ def task( name: str | None = None, depends_on: Callable[..., object] | SqlResourceRef - | tuple[Callable[..., object] | SqlResourceRef, ...] - | list[Callable[..., object] | SqlResourceRef] = (), + | NodeSetRef + | tuple[Callable[..., object] | SqlResourceRef | NodeSetRef, ...] + | list[Callable[..., object] | SqlResourceRef | NodeSetRef] = (), tags: Sequence[str] = (), group: str | None = None, description: str | None = None, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index ac1eafad7f..f11a5b6db6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -82,10 +82,17 @@ class PreviewPerformanceTestCase: built_share: float expected_unbuilt: tuple[int, int] expected_definition_changes: tuple[int, int] + #: The warm preview's wall time, from queueing the request to the stored plan. expected_max_warm_seconds: float #: The warm preview's share of the location's publish, a linear 10K compile on one machine. expected_max_warm_to_publish: float expected_max_check_seconds: float + #: A live plan of the same warehouse, compile included, as a multiple of the location's publish. + expected_max_live_to_publish: float + #: Selected models analysed again against inspected source columns in the live plan. + expected_source_rebinding: tuple[int, int] + #: Every statement the live plan sends the warehouse, cursor batches included. + expected_max_warehouse_statements: int @dataclass(frozen=True) @@ -94,6 +101,7 @@ class PlanTimingsE2ETestCase: plan_args: tuple[str, ...] expected_first_step: str expected_last_step: str + expected_steps: tuple[str, ...] = () @dataclass(frozen=True) @@ -101,3 +109,13 @@ class PlanWithoutTimingsE2ETestCase: description: str plan_args: tuple[str, ...] expected_table_printed: bool + + +@dataclass(frozen=True) +class PlanFromE2ETestCase: + """Where a DuckDB plan is asked to read warehouse state, and what it says on stderr.""" + + description: str + plan_args: tuple[str, ...] + expected_stderr: str + unexpected_stderr: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index 5e42aa6063..1fd65c5fa5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -10,15 +10,23 @@ import sys import time from bisect import bisect_right -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import closing, contextmanager from dataclasses import dataclass from itertools import accumulate from pathlib import Path -from typing import Any +from typing import Any, cast from urllib.request import Request, urlopen from uuid import uuid4 +import sqlbuild._native as native_module +from sqlbuild._native import ( + CompilePipelineOptions, + CompilePipelineResult, + discover_project_inputs, + run_compile_pipeline, +) +from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.e2e.src.sqlbuild.cli.commands.main.lineage.helpers import DIAMOND_LAYERS from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( REPO_ROOT, @@ -36,6 +44,15 @@ location_process, snapshot_api, ) +from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( + offline_cursor_bound_relations, +) +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + FakeColumn, + FakeRelation, + OfflineSnowflakeAdapter, + RecordingSnowflakeWarehouse, +) DIAMOND_DATABASE_FILE: str = "orders.duckdb" _DIAMOND_PROJECT_TOML: str = ( @@ -479,3 +496,111 @@ def timed_plan( elapsed: float = time.monotonic() - started document: dict[str, Any] = json.loads(result)["outcome"]["document"] return elapsed, document + + +# ---- a live plan of the DuckShop 10K project against its synced warehouse ---- + +#: The database and schema the synced state places built models, seeds and state tables in. +_LIVE_TARGET: tuple[str, str] = ("DUCKSHOP_10K", "DEV") +#: One source table in this many carries a column its declaration lacks, as warehouses drift. +DRIFT_EVERY: int = 50 +_DRIFT_COLUMN: FakeColumn = FakeColumn(name="_LOADED_AT", data_type="TIMESTAMP_NTZ") +_NUMBER_TYPE: re.Pattern[str] = re.compile(r"NUMBER\((\d+),(\d+)\)") +_RELATION_TYPES: dict[str, str] = {"TABLE": "BASE TABLE", "VIEW": "VIEW"} +#: Declared string types Snowflake stores, and lists, as TEXT of its default length. +_LISTED_TYPES: dict[str, str] = {"VARCHAR": "TEXT", "STRING": "TEXT", "TEXT": "TEXT"} +_TEXT_LENGTHS: dict[str, int] = dict.fromkeys(_LISTED_TYPES, 16_777_216) + + +@dataclass(frozen=True) +class LivePlan: + """One live plan's wall time and the work it did against the offline warehouse.""" + + seconds: float + model_entries: int + source_rebinding_analyses: int + cursor_relations: int + cursor_statements: int + warehouse_statements: int + + +def _number_column(name: str, declared: re.Match[str]) -> FakeColumn: + precision, scale = declared.groups() + return FakeColumn( + name=name, + data_type="NUMBER", + numeric_precision=int(precision), + numeric_scale=int(scale), + character_maximum_length=None, + ) + + +def _column(column: dict[str, str]) -> FakeColumn: + """The warehouse column a declared column lists as, its type spelled as Snowflake lists it.""" + + declared: str = column["data_type"].upper() + number: re.Match[str] | None = _NUMBER_TYPE.fullmatch(declared) + builders: dict[bool, Callable[[], FakeColumn]] = { + True: lambda: _number_column(column["name"], cast(re.Match[str], number)), + False: lambda: FakeColumn( + name=column["name"], + data_type=_LISTED_TYPES.get(declared, declared), + character_maximum_length=_TEXT_LENGTHS.get(declared), + ), + } + return builders[number is not None]() + + +def _relation(record: dict[str, Any], drifted: frozenset[str]) -> FakeRelation: + location: str = f"{record['state_schema']}.{record['name']}" + columns: tuple[FakeColumn, ...] = tuple(map(_column, record["columns"])) + return FakeRelation( + database=record["state_database"], + schema=record["state_schema"], + name=record["name"], + table_type=_RELATION_TYPES[record["relation_type"]], + columns=columns + {True: (_DRIFT_COLUMN,), False: ()}[location in drifted], + ) + + +def offline_relations(catalog: list[dict[str, Any]]) -> tuple[FakeRelation, ...]: + """The synced catalog's relations, every `DRIFT_EVERY`th source with an undeclared column.""" + + sources: list[str] = sorted( + f"{record['state_schema']}.{record['name']}" + for record in filter( + lambda record: (record["state_database"], record["state_schema"]) != _LIVE_TARGET, + catalog, + ) + ) + drifted: frozenset[str] = frozenset(sources[::DRIFT_EVERY]) + return tuple(_relation(record, drifted) for record in catalog) + + +def live_plan(project: Path, catalog: list[dict[str, Any]]) -> LivePlan: + """Compile and plan `project` in this process, as `sqb plan` does, against `catalog`.""" + + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse( + relations=offline_relations(catalog), current_database=_LIVE_TARGET[0] + ) + _ = native_module.compile_stage_work_info(reset=True) + started: float = time.monotonic() + inputs: Any = discover_project_inputs(project_dir=project) + result: CompilePipelineResult = run_compile_pipeline( + discovered_inputs=inputs, + adapter=OfflineSnowflakeAdapter(warehouse=warehouse), + options=CompilePipelineOptions( + no_cache=True, connection_config=dict(inputs.project_config.connection) + ), + hooks=ConnectionHooks(), + ) + seconds: float = time.monotonic() - started + stages: dict[str, int] = native_module.compile_stage_work_info() + return LivePlan( + seconds=seconds, + model_entries=len(result.plan_output.model_entries), + source_rebinding_analyses=stages.get("source_rebinding_analyses", 0), + cursor_relations=len(offline_cursor_bound_relations(warehouse)), + cursor_statements=len(warehouse.queries_of_kind("cursor_bounds")), + warehouse_statements=len(warehouse.queries), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py index 42845dc31e..8b622abd76 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py @@ -16,6 +16,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( DiamondPlanE2ETestCase, DiamondPlanJsonE2ETestCase, + PlanFromE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( prepare_changed_incremental_diamond_project, @@ -91,3 +92,37 @@ def test_given_changed_diamond_root_when_planning_json_then_lists_each_model_onc reasons: dict[str, object] = {str(model["name"]): model["reason"] for model in models} assert len(models) == len(reasons) assert reasons == test_case.expected_reasons + + +@pytest.mark.parametrize( + "test_case", + [ + PlanFromE2ETestCase( + description="planning from synced state on DuckDB plans live and says why", + plan_args=("--plan-from", "synced_state"), + expected_stderr="Planning live: the duckdb adapter cannot check a synced copy\n", + unexpected_stderr="Planned from synced state", + ), + PlanFromE2ETestCase( + description="planning live says nothing about synced state", + plan_args=(), + expected_stderr="", + unexpected_stderr="Planning live", + ), + ], + ids=lambda case: case.description, +) +def test_given_duckdb_target_when_planning_from_synced_state_then_plans_live_and_says_why( + test_case: PlanFromE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_changed_incremental_diamond_project(tmp_path=tmp_path) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", *test_case.plan_args), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert test_case.expected_stderr in result.stderr, result.stderr + assert test_case.unexpected_stderr not in result.stderr, result.stderr + assert "Query changed (1)" in result.stdout, result.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py index bbb2512559..2c621ff0a7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py @@ -6,6 +6,12 @@ only the edited models and what reads them run. A quadratic lookup over those 10K models or 13K relations, in planning or in the copy it plans from, shows here first. +Each case also plans the project live, as `sqb plan` does, against an offline Snowflake warehouse +holding the same synced catalog, a few sources carrying an undeclared column. It bounds the models +analysed again against inspected source columns, the warehouse statements sent, cursor batching +and the live plan's time as a multiple of the location's publish, which compiles the same +project on the same machine. + It runs in the 10,000-model cell of the dense compile job, after the compile guards. """ @@ -24,6 +30,8 @@ DUCKSHOP_10K_SETUP_SECONDS, DUCKSHOP_10K_TARGET, PREVIEW_DONE, + LivePlan, + live_plan, mirror_synced_state, published_keys, served_duckshop_location, @@ -36,6 +44,9 @@ provision_person, ) +#: Cursor reads one batched statement answers. +_CURSOR_BATCH_SIZE: int = 50 + @pytest.mark.performance @pytest.mark.preview_performance @@ -47,18 +58,24 @@ built_share=1.0, expected_unbuilt=(0, 0), expected_definition_changes=(40, 200), - expected_max_warm_seconds=30.0, + expected_max_warm_seconds=10.0, expected_max_warm_to_publish=0.45, expected_max_check_seconds=0.5, + expected_max_live_to_publish=1.8, + expected_source_rebinding=(25, 150), + expected_max_warehouse_statements=1_150, ), PreviewPerformanceTestCase( description="one model in twenty never built", built_share=0.95, expected_unbuilt=(300, 700), expected_definition_changes=(40, 200), - expected_max_warm_seconds=30.0, + expected_max_warm_seconds=10.0, expected_max_warm_to_publish=0.45, expected_max_check_seconds=0.5, + expected_max_live_to_publish=1.8, + expected_source_rebinding=(25, 150), + expected_max_warehouse_statements=1_150, ), ), ids=lambda case: case.description, @@ -111,6 +128,7 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then session=session, ) checks.append(time.monotonic() - started) + live: LivePlan = live_plan(project, batches["catalog"]) reasons: list[str] = [run["reason"] for run in document["changed"]["runs"]] unbuilt: int = reasons.count("relation missing") definition_changes: int = sum( @@ -122,7 +140,12 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then f"duckshop 10k preview ({test_case.description}): cold={cold:.2f}s warm={warm:.2f}s " f"publish={served.published_seconds:.2f}s warm_to_publish={warm_to_publish:.2f} " f"check_median={check_seconds * 1000:.0f}ms runs={len(reasons)} unbuilt={unbuilt} " - f"definition_changes={definition_changes} models={len(document['models'])}", + f"definition_changes={definition_changes} models={len(document['models'])} " + f"live={live.seconds:.2f}s live_to_publish={live.seconds / served.published_seconds:.2f} " + f"live_entries={live.model_entries} rebinding={live.source_rebinding_analyses} " + f"cursor_relations={live.cursor_relations} cursor_statements={live.cursor_statements} " + f"warehouse_statements={live.warehouse_statements} " + f"warm_steps={json.dumps(document.get('timings', {}).get('steps'))}", flush=True, ) assert document.get("error") is None and len(document["models"]) >= 10_000 @@ -135,8 +158,23 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then <= definition_changes <= test_case.expected_definition_changes[1] ), "only the edited models changed definition: the synced hashes match the planner's" - assert warm < test_case.expected_max_warm_seconds + assert warm < test_case.expected_max_warm_seconds, ( + "a warm preview of 10K models must stay under ten seconds" + ) assert warm_to_publish < test_case.expected_max_warm_to_publish, ( "planning grew faster than compiling: look for a lookup over every model or relation" ) assert check_seconds < test_case.expected_max_check_seconds + assert live.model_entries >= 10_000 + assert ( + test_case.expected_source_rebinding[0] + <= live.source_rebinding_analyses + <= test_case.expected_source_rebinding[1] + ), "only models whose result a source's new or changed columns can change are analysed again" + assert live.cursor_statements == -(-live.cursor_relations // _CURSOR_BATCH_SIZE), ( + "cursor bounds are read in full batches" + ) + assert live.warehouse_statements <= test_case.expected_max_warehouse_statements + assert live.seconds / served.published_seconds < test_case.expected_max_live_to_publish, ( + "a live plan grew faster than compiling and publishing the same project" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py index 1409cc61c8..76e3ebaefb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py @@ -24,8 +24,15 @@ PlanTimingsE2ETestCase( description="--timings prints each step with Other last, then the total", plan_args=("--timings",), - expected_first_step="Compiling the project", + expected_first_step="Discovering the project", expected_last_step="Other", + expected_steps=( + "Discovering the project", + "Compiling the project", + "Evaluating rules", + "Connecting to the warehouse", + "Reading warehouse state", + ), ), ], ids=lambda case: case.description, @@ -41,6 +48,10 @@ def test_given_timings_when_planning_then_the_steps_add_up_to_the_total( assert len(table.steps) == table.row_count, output assert table.steps[0][0] == test_case.expected_first_step, output assert table.steps[-1][0] == test_case.expected_last_step, output + step_names: list[str] = [name for name, _ in table.steps] + assert list(filter(set(test_case.expected_steps).__contains__, step_names)) == list( + test_case.expected_steps + ), output assert "Reading build history" not in output, output assert sum(seconds for _, seconds in table.steps) == pytest.approx( table.total_seconds, abs=0.05 * len(table.steps) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_verbose_inspection.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_verbose_inspection.py index 5376a0ac92..ad243f0398 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_verbose_inspection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_verbose_inspection.py @@ -19,7 +19,9 @@ _INSPECTION_LINE: re.Pattern[str] = re.compile(r"^ inspect +\d+\.\d{2}s \d+ rows? \S", re.M) _METADATA_LINE: re.Pattern[str] = re.compile(r"^ inspect .*list_relations\(", re.M) -_CURSOR_LINE: re.Pattern[str] = re.compile(r"^ inspect .*SELECT CAST\(MAX\(", re.M) +_CURSOR_LINE: re.Pattern[str] = re.compile( + r"^ inspect .*SELECT (?:\d+ AS _read, )?CAST\((?:MIN|MAX)\(", re.M +) _TOTAL_LINE: re.Pattern[str] = re.compile(r"^Inspection queries: (\d+) \(", re.M) diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 34c7b96fbf..18a234ac3f 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index f55a9bd700..2abc0dd5ca 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -62,6 +62,27 @@ class NativeCatalogCase: expected_sql: str = "SELECT * FROM __sqlbuild_table_function_table_fn__orders" +@dataclass(frozen=True) +class PhysicalEvidenceCase: + """Listed source columns on a case-sensitive dialect, and whether they rebind readers.""" + + description: str + adapter: Callable[[], BaseAdapter] + dialect: str + listed: tuple[tuple[str, str], ...] + expected_rebinding: bool + expected_codes: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AddedColumnsCase: + """Models over a source the warehouse lists with one more column, and which are re-analysed.""" + + description: str + models: tuple[tuple[str, str], ...] + expected_reanalysed: int + + @dataclass(frozen=True) class SemanticCompileCase: description: str @@ -400,3 +421,32 @@ class AdapterRecoveryCase: ("B002", "downstream"), ("B212", "upstream"), ) + + +@dataclass(frozen=True) +class SetDependencyPlanTestCase: + """A task's `job(...)` or `tag(...)` dependency and the models planning its upstream selects.""" + + description: str + depends_on: str + expected_models: tuple[str, ...] + expected_sql_deps: tuple[str, ...] + + +@dataclass(frozen=True) +class SetDependencyErrorTestCase: + """A `job(...)` or `tag(...)` dependency compile refuses, and what its error says.""" + + description: str + depends_on: str + job_files: tuple[tuple[str, str], ...] + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class WaitChangeTestCase: + """A job selection edited after a build, and the wait changes the next plan lists.""" + + description: str + edited_select: str + expected_changes: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 4dde272237..e51e7ecab9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -244,3 +244,46 @@ def postgres_semantic_shapes( resolved_connection=_OFFLINE_POSTGRES_CONNECTION, ) return semantic_shapes(project=project) + + +def set_dependency_project( + *, depends_on: str, job_files: tuple[tuple[str, str], ...] = () +) -> dict[str, str]: + """Four models, a `nightly_job` over three of them, and a `report` task on `depends_on`.""" + + model: str = "MODEL (description 'Test model {name}.', materialized table{tags});\n{sql}\n" + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "set_dependencies"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = ":memory:"\n\n[targets.dev]\nschema = "dev"\n' + ), + "models/stg_orders.sql": model.format(name="stg_orders", tags="", sql="SELECT 1 AS id"), + "models/orders.sql": model.format( + name="orders", tags=", tags [finance]", sql='SELECT id FROM __ref("stg_orders")' + ), + "models/order_items.sql": model.format( + name="order_items", tags=", tags [finance]", sql='SELECT id FROM __ref("stg_orders")' + ), + "models/customers.sql": model.format(name="customers", tags="", sql="SELECT 2 AS id"), + "orchestration/jobs/nightly_job.toml": ( + 'name = "nightly_job"\ndescription = "Nightly."\n' + 'select = "stg_orders orders order_items"\n' + ), + "python/tasks/report.py": ( + "from sqlbuild.refs import job, model, tag\nfrom sqlbuild.tasks import task\n\n\n" + f"@task(depends_on={depends_on})\n" + 'def report(ctx):\n """Test task report."""\n return None\n' + ), + } + files.update(dict(job_files)) + return files + + +def python_sql_deps(discovered_inputs: DiscoveredProjectInputs, name: str) -> tuple[str, ...]: + """`kind:name` of each SQL resource a Python node's graph edges read, in order.""" + + graph: Any = native_module.build_python_node_graph(discovered_inputs=discovered_inputs) + return tuple( + f"{reference.kind.value}:{reference.name}" + for reference in graph.nodes_by_name[name].sql_deps + ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 0b17af264c..46abce3a20 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -16,13 +16,22 @@ ) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + AddedColumnsCase, NativeCatalogCase, + PhysicalEvidenceCase, ValidationCacheCase, ) +_DECLARED_ORDERS: tuple[tuple[str, str], ...] = (("order_id", "INTEGER"), ("status", "VARCHAR")) +_CUSTOMERS_MODEL: tuple[str, str] = ( + "customers", + "SELECT 1 AS customer_id, 'Ada' AS customer_name", +) + @pytest.mark.parametrize( "test_case", @@ -194,6 +203,215 @@ def test_given_case_sensitive_snowflake_when_rebinding_listed_columns_then_unquo assert tuple(item.code for item in diagnostics) == test_case.expected_codes +@pytest.mark.parametrize( + "test_case", + [ + PhysicalEvidenceCase( + "Snowflake listed names equal to declared names", + SnowflakeAdapter, + "snowflake", + _DECLARED_ORDERS, + expected_rebinding=False, + ), + PhysicalEvidenceCase( + "Snowflake new listed column an explicit reader cannot see", + SnowflakeAdapter, + "snowflake", + (*_DECLARED_ORDERS, ("region", "VARCHAR")), + expected_rebinding=False, + ), + PhysicalEvidenceCase( + "Snowflake declared column missing from the listing", + SnowflakeAdapter, + "snowflake", + (("order_id", "INTEGER"), ("region", "VARCHAR")), + expected_rebinding=True, + expected_codes=("B002",), + ), + PhysicalEvidenceCase( + "Snowflake lowercase stored name rejects unquoted reader", + SnowflakeAdapter, + "snowflake", + (('"order_id"', "INTEGER"), ("status", "VARCHAR")), + expected_rebinding=True, + expected_codes=("B002",), + ), + PhysicalEvidenceCase( + "Snowflake default length and precision spellings of the declared types", + SnowflakeAdapter, + "snowflake", + (("order_id", "NUMBER(38,0)"), ("status", "VARCHAR(16777216)")), + expected_rebinding=False, + ), + PhysicalEvidenceCase( + "Snowflake shorter listed VARCHAR", + SnowflakeAdapter, + "snowflake", + (("order_id", "NUMBER(38,0)"), ("status", "VARCHAR(10)")), + expected_rebinding=True, + ), + PhysicalEvidenceCase( + "Snowflake changed column type", + SnowflakeAdapter, + "snowflake", + (("order_id", "VARCHAR"), ("status", "VARCHAR")), + expected_rebinding=True, + ), + PhysicalEvidenceCase( + "Postgres listed names equal to declared names", + PostgresAdapter, + "postgres", + _DECLARED_ORDERS, + expected_rebinding=False, + ), + PhysicalEvidenceCase( + "Postgres mixed-case stored name rejects unquoted reader", + PostgresAdapter, + "postgres", + (("Order_Id", "INTEGER"), ("status", "VARCHAR")), + expected_rebinding=True, + expected_codes=("B002",), + ), + ], + ids=lambda case: case.description, +) +def test_given_case_sensitive_dialect_when_listed_columns_bind_as_declared_then_skips_rebinding( + test_case: PhysicalEvidenceCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{test_case.dialect}"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n" + " contract: enforced\n columns:\n" + + "".join( + f" - name: {name}\n type: {kind}\n" for name, kind in _DECLARED_ORDERS + ) + ) + (tmp_path / "models/report.sql").write_text( + "MODEL (description 'Test model report.', database warehouse, schema analytics);\n" + 'SELECT order_id, status FROM __source("orders")' + ) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=test_case.adapter() + ) + _ = sql_analysis_models.compile_stage_work_info(reset=True) + + diagnostics: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( + project=project, + columns={"orders": tuple(ColumnInfo(name, kind) for name, kind in test_case.listed)}, + profile=ExpressionInferenceProfile(sql_analysis_dialect=test_case.dialect), + selected_keys=frozenset(model.key for model in project.models), + ) + + rebinding: int = sql_analysis_models.compile_stage_work_info().get( + "source_rebinding_analyses", 0 + ) + assert (rebinding > 0) is test_case.expected_rebinding + assert tuple(item.code for item in diagnostics) == test_case.expected_codes + + +@pytest.mark.parametrize( + "test_case", + [ + AddedColumnsCase( + "a star reader is analysed again", + (("report", 'SELECT * FROM __source("orders")'),), + expected_reanalysed=1, + ), + AddedColumnsCase( + "an explicit reader of the source alone keeps its analysis", + (("report", 'SELECT order_id, status FROM __source("orders")'),), + expected_reanalysed=0, + ), + AddedColumnsCase( + "an unqualified name in a join with the source is analysed again", + ( + _CUSTOMERS_MODEL, + ( + "report", + 'SELECT order_id, customer_name FROM __source("orders") AS o ' + 'JOIN __ref("customers") AS c ON o.order_id = c.customer_id', + ), + ), + expected_reanalysed=1, + ), + AddedColumnsCase( + "qualified names in a join with the source keep their analysis", + ( + _CUSTOMERS_MODEL, + ( + "report", + 'SELECT o.order_id, c.customer_name FROM __source("orders") AS o ' + 'JOIN __ref("customers") AS c ON o.order_id = c.customer_id', + ), + ), + expected_reanalysed=0, + ), + AddedColumnsCase( + "an explicit reader of a star reader keeps its analysis", + ( + ("staged", 'SELECT * FROM __source("orders")'), + ("report", 'SELECT order_id FROM __ref("staged")'), + ), + expected_reanalysed=1, + ), + AddedColumnsCase( + "a star reader of a star reader is analysed again", + ( + ("staged", 'SELECT * FROM __source("orders")'), + ("report", 'SELECT * FROM __ref("staged")'), + ), + expected_reanalysed=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_source_gaining_columns_when_rebinding_then_reanalyses_models_they_can_change( + test_case: AddedColumnsCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n" + " contract: enforced\n columns:\n" + + "".join( + f" - name: {name}\n type: {kind}\n" for name, kind in _DECLARED_ORDERS + ) + ) + for name, sql in test_case.models: + (tmp_path / f"models/{name}.sql").write_text( + f"MODEL (description 'Test model {name}.', database warehouse, schema analytics);\n" + + sql + ) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=SnowflakeAdapter() + ) + _ = sql_analysis_models.compile_stage_work_info(reset=True) + + diagnostics: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( + project=project, + columns={ + "orders": tuple( + ColumnInfo(name, kind) for name, kind in (*_DECLARED_ORDERS, ("region", "VARCHAR")) + ) + }, + profile=ExpressionInferenceProfile(sql_analysis_dialect="snowflake"), + selected_keys=frozenset(model.key for model in project.models), + ) + + assert ( + sql_analysis_models.compile_stage_work_info().get("source_rebinding_analyses", 0) + == test_case.expected_reanalysed + ) + assert diagnostics == () + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index ad9eeb940c..f4656ffeb9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -104,7 +104,7 @@ def test_given_effective_target_when_binding_quotes_then_honours_session_setting ) physical: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, - columns={"orders": (ColumnInfo(name="customer", type="VARCHAR"),)}, + columns={"orders": (ColumnInfo(name="customer", type="VARCHAR(10)"),)}, profile=ExpressionInferenceProfile(sql_analysis_dialect="snowflake"), selected_keys=frozenset(model.key for model in project.models), ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py new file mode 100644 index 0000000000..ef20d12692 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_dependencies.py @@ -0,0 +1,214 @@ +"""Python nodes depending on a job or a tag get edges to its end nodes through the real pipeline.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, + PlannerInputError, + discover_project_inputs, + run_cli, + run_compile_pipeline, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.runtime.contracts.models import ConnectionHooks +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + SetDependencyErrorTestCase, + SetDependencyPlanTestCase, + WaitChangeTestCase, +) +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( + python_sql_deps, + set_dependency_project, +) + +_GROUP_FILES: tuple[tuple[str, str], ...] = ( + ( + "models/customer_orders.sql", + "MODEL (description 'Test model customer_orders.', materialized table);\n" + 'SELECT o.id FROM __ref("orders") AS o JOIN __ref("customers") AS c ON o.id = c.id\n', + ), + ( + "orchestration/jobs/wide_job.toml", + 'name = "wide_job"\ndescription = "Wide."\nselect = "orders customer_orders"\n', + ), + ( + "orchestration/jobs/reporting_job.toml", + 'name = "reporting_job"\ndescription = "Reporting."\nselect = "task:report customers"\n', + ), + ( + "orchestration/job_groups/daily_job_group.toml", + 'name = "daily_job_group"\ndescription = "Daily."\njobs = ["wide_job", "reporting_job"]\n', + ), +) + + +@pytest.mark.parametrize( + "test_case", + [ + SetDependencyPlanTestCase( + description="a job dependency reads the job's end nodes", + depends_on='job("nightly_job")', + expected_models=("order_items", "orders", "stg_orders"), + expected_sql_deps=("model:order_items", "model:orders"), + ), + SetDependencyPlanTestCase( + description="exclude removes nodes before reducing to end nodes", + depends_on='job("nightly_job", exclude="order_items")', + expected_models=("orders", "stg_orders"), + expected_sql_deps=("model:orders",), + ), + SetDependencyPlanTestCase( + description="a tag dependency reads its tagged end nodes", + depends_on='tag("finance", exclude="orders")', + expected_models=("order_items", "stg_orders"), + expected_sql_deps=("model:order_items",), + ), + SetDependencyPlanTestCase( + description="a model named and reached through a job is read once", + depends_on='(model("orders"), job("nightly_job"))', + expected_models=("order_items", "orders", "stg_orders"), + expected_sql_deps=("model:orders", "model:order_items"), + ), + ], + ids=lambda case: case.description, +) +def test_given_set_dependency_when_planning_the_tasks_upstream_then_selects_the_set( + test_case: SetDependencyPlanTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files(tmp_path, set_dependency_project(depends_on=test_case.depends_on)) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + + result: CompilePipelineResult = run_compile_pipeline( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + options=CompilePipelineOptions( + no_sql_validation=True, select=("+task:report",), resolve_python_run_selectors=True + ), + hooks=ConnectionHooks(), + ) + + assert ( + tuple(sorted(entry.name for entry in result.plan_output.model_entries)) + == test_case.expected_models + ) + assert python_sql_deps(discovered, "report") == test_case.expected_sql_deps + + +@pytest.mark.parametrize( + "test_case", + [ + SetDependencyErrorTestCase( + description="an unknown job", + depends_on='job("weekly_job")', + job_files=(), + expected_fragments=( + "task 'report' depends on job 'weekly_job', which the project does not declare", + ), + ), + SetDependencyErrorTestCase( + description="an unknown tag", + depends_on='tag("marketing")', + job_files=(), + expected_fragments=("task 'report' depends on tag 'marketing', which no node carries",), + ), + SetDependencyErrorTestCase( + description="a job its exclude empties", + depends_on='job("nightly_job", exclude="stg_orders orders order_items")', + job_files=(), + expected_fragments=("which leaves no model, seed, source, loader, task or asset",), + ), + SetDependencyErrorTestCase( + description="job group members that would wait for each other", + depends_on='job("wide_job")', + job_files=_GROUP_FILES, + expected_fragments=( + "a job dependency makes job group 'daily_job_group' wait in a loop", + "'wide_job''s model 'customer_orders' reads model 'customers'", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unresolvable_set_dependency_when_planning_then_refuses_naming_the_cause( + test_case: SetDependencyErrorTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files( + tmp_path, + set_dependency_project(depends_on=test_case.depends_on, job_files=test_case.job_files), + ) + + with pytest.raises(PlannerInputError) as raised: + _ = run_compile_pipeline( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + options=CompilePipelineOptions( + no_sql_validation=True, resolve_python_run_selectors=True + ), + hooks=ConnectionHooks(), + ) + + text: str = f"{raised.value} {getattr(raised.value, 'help', '')}" + assert all(fragment in text for fragment in test_case.expected_fragments), text + + +@pytest.mark.parametrize( + "test_case", + [ + WaitChangeTestCase( + description="a job gaining an end node", + edited_select="stg_orders orders order_items customers", + expected_changes=( + "report now also waits on model 'customers' through job 'nightly_job'", + ), + ), + WaitChangeTestCase( + description="a job losing an end node", + edited_select="stg_orders orders", + expected_changes=( + "report no longer waits on model 'order_items' through job 'nightly_job'", + ), + ), + WaitChangeTestCase( + description="an unchanged job", + edited_select="stg_orders orders order_items", + expected_changes=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_job_dependency_when_the_job_changes_then_the_plan_lists_wait_changes( + test_case: WaitChangeTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], + capsys: pytest.CaptureFixture[str], +) -> None: + files: dict[str, str] = set_dependency_project(depends_on='job("nightly_job")') + files["sqlbuild_project.toml"] = files["sqlbuild_project.toml"].replace( + ":memory:", str(tmp_path / "warehouse.duckdb") + ) + write_repo_files(tmp_path, files) + assert run_cli(["--project-dir", str(tmp_path), "--no-color", "build"]) == 0 + job: Path = tmp_path / "orchestration/jobs/nightly_job.toml" + _ = job.write_text( + job.read_text().replace("stg_orders orders order_items", test_case.edited_select) + ) + _ = capsys.readouterr() + + assert run_cli(["--project-dir", str(tmp_path), "--no-color", "plan", "--json"]) == 0 + + document: dict[str, Any] = json.loads(capsys.readouterr().out) + nodes: dict[str, dict[str, Any]] = {node["name"]: node for node in document["python_nodes"]} + assert tuple(nodes["report"].get("wait_changes", ())) == test_case.expected_changes diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 365c03d796..298c1326d7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -152,6 +152,8 @@ class ConcurrentCursorReadsTestCase: failing_relations: frozenset[str] expected_start_progress: str expected_unreadable: dict[str, tuple[str, ...]] + expected_statements: int + expected_max_active: int @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index c52642fb62..750a365eda 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -340,8 +340,10 @@ def _tracked(self, connection: Any, sql: str, eligible: bool) -> Any: self.cursor_sql.append(sql) try: time.sleep(0.01) - relation: str = sql.split(" FROM ", 1)[1].split(" ", 1)[0] - failing: bool = relation in self._failing[eligible] + relations: set[str] = { + part.split(None, 1)[0].rstrip(")") for part in sql.split(" FROM ")[1:] + } + failing: bool = not relations.isdisjoint(self._failing[eligible]) return _CURSOR_ANSWERS[(failing, self._fail_on_fetch)](connection, sql, self._lock) finally: with self._lock: diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py index ff7c1d48d2..98399cd1b4 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -39,20 +39,28 @@ "test_case", [ ConcurrentCursorReadsTestCase( - description="parallel reads stay within the bound", - relation_count=12, + description="batched reads run concurrently within the bound", + relation_count=120, concurrency=4, failing_relations=frozenset(), - expected_start_progress="Inspecting cursor bounds for 12 relations (4 concurrent)...", + expected_start_progress=( + "Inspecting cursor bounds for 120 relations in 3 statements (3 concurrent)..." + ), expected_unreadable={}, + expected_statements=3, + expected_max_active=3, ), ConcurrentCursorReadsTestCase( description="one failed relation stays unavailable and others succeed", relation_count=8, concurrency=4, failing_relations=frozenset({"staging.events_5"}), - expected_start_progress="Inspecting cursor bounds for 8 relations (4 concurrent)...", + expected_start_progress=( + "Inspecting cursor bounds for 8 relations in 1 statement (1 concurrent)..." + ), expected_unreadable={"daily_5": ("events_5.event_time",)}, + expected_statements=9, + expected_max_active=4, ), ], ids=lambda case: case.description, @@ -92,10 +100,11 @@ def test_given_concurrent_catalog_when_reading_cursor_bounds_then_matches_sequen reads: list[str] = cursor_progress(parallel_progress) assert parallel.cursor_snapshots == sequential.cursor_snapshots assert sequential_execute.max_active == 1 - assert 1 < parallel_execute.max_active <= test_case.concurrency + assert 1 < parallel_execute.max_active <= test_case.expected_max_active assert sorted(parallel_execute.cursor_sql) == sorted(sequential_execute.cursor_sql) + assert len(parallel_execute.cursor_sql) == test_case.expected_statements assert reads[0] == test_case.expected_start_progress - assert len(reads) == test_case.relation_count + 2 + assert len(reads) == test_case.expected_statements + 2 assert unreadable == test_case.expected_unreadable diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index 0ed6547cc9..1a69dee445 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -224,6 +224,30 @@ expected_column_table_names=frozenset(f"model_{index}" for index in range(251)), expected_fingerprint_names=frozenset(), ), + GatherWarehouseSnapshotTestCase( + description=( + "lists the whole schema beyond the name threshold but reads columns only for " + "the selection and what it reads" + ), + setup_sql=( + *(f"CREATE TABLE staging.model_{index} (id INTEGER)" for index in range(251)), + "CREATE TABLE staging.unselected (value VARCHAR)", + ), + model_locations={ + **{f"model_{index}": "staging" for index in range(METADATA_NAME_FILTER_LIMIT + 1)}, + "unselected": "staging", + }, + seed_locations={}, + selected_keys=frozenset( + CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=f"model_{index}") + for index in range(METADATA_NAME_FILTER_LIMIT + 1) + ), + expected_relation_names=frozenset( + {*(f"model_{index}" for index in range(251)), "unselected"} + ), + expected_column_table_names=frozenset(f"model_{index}" for index in range(251)), + expected_fingerprint_names=frozenset(), + ), ], ids=lambda case: case.description, ) @@ -361,7 +385,7 @@ def test_given_sources_when_gathering_source_columns_then_filters_metadata_names upstream_maxes=("2024-02-01 00:00:00",), ), }, - expected_progress_calls=5, + expected_progress_calls=3, ), GatherCursorSnapshotTestCase( description="derives highest eligible target max without removing future rows", @@ -393,7 +417,7 @@ def test_given_sources_when_gathering_source_columns_then_filters_metadata_names upstream_maxes=("2024-02-01 00:00:00",), ), }, - expected_progress_calls=5, + expected_progress_calls=3, ), GatherCursorSnapshotTestCase( description="gathers cursor bounds for first run with no target table", @@ -710,7 +734,7 @@ def test_given_timezone_cursor_text_when_gathering_snapshot_then_normalizes_ware description="explicit timestamp start skips eligible target query", expected_target_max=TimestampValue(value=datetime(2026, 9, 3)), expected_eligible_max=None, - expected_statement_count=2, + expected_statement_count=1, ) ], ids=lambda case: case.description, @@ -845,7 +869,7 @@ def _record_execute(*, connection: Any, sql: str) -> Any: cursor_snapshot: ModelCursorSnapshot = snapshot.cursor_snapshots["hourly_events"] assert cursor_snapshot.target_eligible_max == test_case.expected_eligible_max assert test_case.expected_eligible_sql in statements - assert "UNION ALL" not in statements[0] + assert statements[0].count("\nUNION ALL\n") == 1 @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index efc338a23f..c98ad03871 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -135,6 +135,7 @@ class SnowflakeManySchemasTestCase: description: str schema_count: int models_per_schema: int + schema_listing_beyond_32: bool expected_per_relation_column_reads: int expected_schema_column_reads: int @@ -173,3 +174,16 @@ class UnlistedMigrationOriginTestCase: select: tuple[str, ...] expected_warnings: tuple[str, ...] expected_unlisted_reads: int + + +@dataclass(frozen=True) +class PlanFromTestCase: + """Where a target and the command ask a plan to read from, and what the plan reports.""" + + description: str + target_plan_from: str | None + requested: str | None + store: bool + expected_report: str + expected_absent: tuple[str, ...] + expected_live_listing: bool diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index ecde09533b..eeb122945f 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -2,15 +2,20 @@ from __future__ import annotations +import json import re +import sqlite3 from collections import Counter from collections.abc import Callable +from contextlib import closing from dataclasses import fields, replace from datetime import UTC, datetime +from itertools import chain from pathlib import Path from tempfile import gettempdir from types import MappingProxyType from typing import Any, cast +from uuid import uuid4 from sqlbuild._native import ( CompiledFunction, @@ -51,7 +56,8 @@ run_compile_pipeline, write_fingerprint, ) -from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationReadProbe +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import ( CompiledFunctionView, @@ -73,6 +79,8 @@ RecordedQuery, RecordingSnowflakeWarehouse, SyntheticSnowflakeProject, + live_inspection_catalog, + write_synthetic_snowflake_project, ) @@ -485,7 +493,7 @@ def grouped(model: type) -> dict[str, Any]: ) _OFFLINE_INFORMATION_SCHEMA_KINDS: frozenset[str] = frozenset({"tables", "columns"}) _SHOW_SCHEMA_SCOPE: re.Pattern[str] = re.compile(r' IN SCHEMA (?:"[^"]*"\.)?"([^"]*)"') -_CURSOR_BOUND_RELATION: re.Pattern[str] = re.compile(r"\sFROM\s+(\S+)\s*$", re.IGNORECASE) +_CURSOR_BOUND_RELATION: re.Pattern[str] = re.compile(r"\sFROM\s+(\S+)\s*$", re.IGNORECASE | re.M) def plan_offline_snowflake_project( @@ -493,17 +501,20 @@ def plan_offline_snowflake_project( project: SyntheticSnowflakeProject, warehouse: RecordingSnowflakeWarehouse, no_cache: bool = True, + plan_from: str | None = None, + adapter_type: type[OfflineSnowflakeAdapter] = OfflineSnowflakeAdapter, ) -> CompilePipelineResult: """Compile and plan a synthetic Snowflake project against the recording warehouse.""" inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project.project_dir) return run_compile_pipeline( discovered_inputs=inputs, - adapter=OfflineSnowflakeAdapter(warehouse=warehouse), + adapter=adapter_type(warehouse=warehouse), options=CompilePipelineOptions( no_sql_validation=True, no_cache=no_cache, connection_config=dict(inputs.project_config.connection), + plan_from=plan_from, ), hooks=ConnectionHooks(), ) @@ -538,11 +549,13 @@ def offline_information_schema_queries( def offline_cursor_bound_relations(warehouse: RecordingSnowflakeWarehouse) -> tuple[str, ...]: - """Return the single relation each cursor-bound statement reads.""" + """Return every relation the cursor-bound statements read, in statement order.""" return tuple( - _CURSOR_BOUND_RELATION.findall(query.sql)[0] - for query in warehouse.queries_of_kind("cursor_bounds") + chain.from_iterable( + _CURSOR_BOUND_RELATION.findall(query.sql) + for query in warehouse.queries_of_kind("cursor_bounds") + ) ) @@ -680,3 +693,101 @@ def plan_source_reading_project( adapter=adapter, options=CompilePipelineOptions(no_sql_validation=True), ) + + +def write_targeted_snowflake_project( + *, project_dir: Path, target: str, plan_from: str | None +) -> SyntheticSnowflakeProject: + """A small synthetic project whose default target sets `plan_from`, or leaves it unset.""" + + project: SyntheticSnowflakeProject = write_synthetic_snowflake_project( + project_dir=project_dir, + models_per_schema=6, + incremental_every=3, + sources=4, + unmanaged_relations_per_schema=5, + ) + toml: Path = project_dir / "sqlbuild_project.toml" + setting: str = ("", f'plan_from = "{plan_from}"\n')[plan_from is not None] + toml.write_text( + toml.read_text().replace( + 'adapter = "snowflake"\n', f'adapter = "snowflake"\ndefault_target = "{target}"\n', 1 + ) + + f'\n[targets.{target}]\nschema = "preserve"\n{setting}', + encoding="utf-8", + ) + return project + + +def write_synced_control_store( + *, + project_dir: Path, + target: str, + warehouse: RecordingSnowflakeWarehouse, + schemas: tuple[str, ...], +) -> None: + """A control store whose development location mirrored `schemas` of the warehouse just now.""" + + catalog: InspectionCatalog = live_inspection_catalog(warehouse) + relations: tuple[RelationInfo, ...] = catalog.list_relations( + database="analytics", schemas=schemas + ) + columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( + catalog.get_columns_for_relations(relations=relations) + ) + location: str = str(uuid4()) + observed: str = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + (project_dir / ".sqlbuild").mkdir(parents=True) + with closing(sqlite3.connect(project_dir / ".sqlbuild/control.sqlite3")) as store: + _ = store.execute( + "CREATE TABLE sqb_development_locations (project_path TEXT, target TEXT, " + "location_id TEXT)" + ) + _ = store.execute( + "CREATE TABLE sqb_warehouse_catalog (event_id TEXT, location_id TEXT, target TEXT, " + "state_database TEXT, state_schema TEXT, observed_at TEXT, record_json JSON)" + ) + _ = store.execute( + "INSERT INTO sqb_development_locations VALUES (?, ?, ?)", + (str(project_dir), target, location), + ) + _ = store.executemany( + "INSERT INTO sqb_warehouse_catalog VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + _mirrored_row( + relation=relation, + columns=columns.get(relation.identity, ()), + scope=(location, target, observed), + ) + for relation in relations + ], + ) + store.commit() + + +def _mirrored_row( + *, relation: RelationInfo, columns: tuple[ColumnInfo, ...], scope: tuple[str, str, str] +) -> tuple[str, ...]: + location, target, observed = scope + record: dict[str, object] = { + "state_database": "ANALYTICS", + "state_schema": str(relation.schema).upper(), + "name": relation.name, + "relation_type": relation.relation_type, + "columns": [{"name": column.name, "data_type": column.type} for column in columns], + } + return ( + str(uuid4()), + location, + target, + "ANALYTICS", + str(relation.schema).upper(), + observed, + json.dumps(record), + ) + + +def planned_entries(result: CompilePipelineResult) -> list[tuple[str, str]]: + """Each planned model's name and reason, in name order.""" + + return sorted((entry.name, str(entry.reason)) for entry in result.plan_output.model_entries) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py new file mode 100644 index 0000000000..f92caf9741 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py @@ -0,0 +1,123 @@ +"""A target opts into planning from its synced copy of warehouse state; other plans read live.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import CompilePipelineResult +from tests.integration.src.sqlbuild.compiler.planner.main._test_types import PlanFromTestCase +from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( + plan_offline_snowflake_project, + planned_entries, + write_synced_control_store, + write_targeted_snowflake_project, +) +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + RecordingSnowflakeWarehouse, + SyntheticSnowflakeProject, +) + +_TARGET: str = "dev" +_MODEL_SCHEMAS: tuple[str, ...] = ("staging", "intermediate", "marts") +_SYNCED: str = "Planned from synced state: checked in " +_NO_STORE: str = "Planning live: the project has no control store" +_LIVE: str = "Planning live:" + + +@pytest.mark.parametrize( + "test_case", + [ + PlanFromTestCase( + description="a target without the setting plans live and says nothing", + target_plan_from=None, + requested=None, + store=True, + expected_report="", + expected_absent=(_LIVE, _SYNCED), + expected_live_listing=True, + ), + PlanFromTestCase( + description="the target's setting plans from the synced copy after one check", + target_plan_from="synced_state", + requested=None, + store=True, + expected_report=_SYNCED, + expected_absent=(_LIVE,), + expected_live_listing=False, + ), + PlanFromTestCase( + description="the flag plans from the synced copy for a target without the setting", + target_plan_from=None, + requested="synced_state", + store=True, + expected_report=_SYNCED, + expected_absent=(_LIVE,), + expected_live_listing=False, + ), + PlanFromTestCase( + description="the flag plans live for a target with the setting", + target_plan_from="synced_state", + requested="live", + store=True, + expected_report="", + expected_absent=(_LIVE, _SYNCED), + expected_live_listing=True, + ), + PlanFromTestCase( + description="a project without a control store plans live and says why", + target_plan_from="synced_state", + requested=None, + store=False, + expected_report=_NO_STORE, + expected_absent=(_SYNCED,), + expected_live_listing=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_plan_from_when_planning_then_reads_the_synced_copy_only_when_asked( + test_case: PlanFromTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_CONTROL_DATABASE_URL", raising=False) + project: SyntheticSnowflakeProject = write_targeted_snowflake_project( + project_dir=tmp_path / "orders_platform", + target=_TARGET, + plan_from=test_case.target_plan_from, + ) + live: CompilePipelineResult = plan_offline_snowflake_project( + project=project, + warehouse=RecordingSnowflakeWarehouse(relations=project.relations), + plan_from="live", + ) + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse( + relations=project.relations + ) + write_synced_control_store( + project_dir=(tmp_path / "no_store", project.project_dir)[test_case.store], + target=_TARGET, + warehouse=warehouse, + schemas=_MODEL_SCHEMAS, + ) + warehouse.queries.clear() + _ = capsys.readouterr() + + result: CompilePipelineResult = plan_offline_snowflake_project( + project=project, warehouse=warehouse, plan_from=test_case.requested + ) + + err: str = capsys.readouterr().err + assert planned_entries(result) == planned_entries(live) + assert ( + any( + query.kind == "show_tables" and "STAGING" in f"{query.sql} {query.params}" + for query in warehouse.queries + ) + == test_case.expected_live_listing + ) + assert test_case.expected_report in err + assert not any(text in err for text in test_case.expected_absent) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py index 67ce789fab..5934ad93d4 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py @@ -34,10 +34,12 @@ OfflineSnowflakeAdapter, RecordedQuery, RecordingSnowflakeWarehouse, + SchemaListingSnowflakeAdapter, SyntheticSnowflakeProject, write_synthetic_snowflake_project, ) +_CURSOR_BATCH_SIZE: int = 50 _ONE_READ_PER_SCHEMA: dict[str, int] = {"STAGING": 1, "INTERMEDIATE": 1, "MARTS": 1, "RAW": 1} _SOURCES: int = 260 _SHOW_READS_PER_SCHEMA: int = 3 @@ -81,7 +83,7 @@ def test_given_large_multi_schema_project_when_planning_then_metadata_reads_stay ) result: CompilePipelineResult = plan_offline_snowflake_project( - project=project, warehouse=warehouse + project=project, warehouse=warehouse, adapter_type=SchemaListingSnowflakeAdapter ) metadata: tuple[RecordedQuery, ...] = offline_metadata_queries(warehouse) @@ -112,14 +114,14 @@ def test_given_large_multi_schema_project_when_planning_then_metadata_reads_stay "test_case", [ SnowflakeCursorBoundsBudgetTestCase( - description="one statement per relation with bounded parallelism", + description="batched statements read each relation once with bounded parallelism", statement_latency_seconds=0.002, expected_max_concurrency=OfflineSnowflakeAdapter.metadata_inspection_concurrency, ) ], ids=lambda case: case.description, ) -def test_given_incremental_models_when_planning_then_cursor_bounds_run_one_per_relation_in_parallel( +def test_given_incremental_models_when_planning_then_cursor_bounds_run_in_parallel_batches( test_case: SnowflakeCursorBoundsBudgetTestCase, tmp_path: Path ) -> None: project: SyntheticSnowflakeProject = write_synthetic_snowflake_project( @@ -136,9 +138,9 @@ def test_given_incremental_models_when_planning_then_cursor_bounds_run_one_per_r bounds: tuple[RecordedQuery, ...] = warehouse.queries_of_kind("cursor_bounds") relations: tuple[str, ...] = offline_cursor_bound_relations(warehouse) - assert len(bounds) >= len(project.incremental_model_names) - assert all("UNION" not in query.sql.upper() for query in bounds) - assert len(relations) == len(bounds) == len(set(relations)) + assert len(relations) >= len(project.incremental_model_names) + assert len(relations) == len(set(relations)) + assert len(bounds) == -(-len(relations) // _CURSOR_BATCH_SIZE) assert 1 < warehouse.max_concurrent_cursor_bounds <= test_case.expected_max_concurrency @@ -178,9 +180,18 @@ def test_given_planned_project_when_planning_again_then_cursor_bounds_are_read_a description="many small schemas read model columns once per schema", schema_count=24, models_per_schema=2, + schema_listing_beyond_32=True, expected_per_relation_column_reads=30, expected_schema_column_reads=24, - ) + ), + SnowflakeManySchemasTestCase( + description="many small schemas read each relation's columns once in production", + schema_count=24, + models_per_schema=2, + schema_listing_beyond_32=False, + expected_per_relation_column_reads=78, + expected_schema_column_reads=0, + ), ], ids=lambda case: case.description, ) @@ -199,7 +210,11 @@ def test_given_many_small_schemas_when_planning_then_columns_are_read_per_schema ) result: CompilePipelineResult = plan_offline_snowflake_project( - project=project, warehouse=warehouse + project=project, + warehouse=warehouse, + adapter_type=(OfflineSnowflakeAdapter, SchemaListingSnowflakeAdapter)[ + test_case.schema_listing_beyond_32 + ], ) assert len(result.plan_output.model_entries) == len(project.model_schemas) diff --git a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py index e00348448c..578de6b0cc 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py +++ b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py @@ -21,6 +21,7 @@ "adapters/databricks/constants.py", "adapters/postgres/classes/postgres_adapter.py", "adapters/snowflake/_helpers/metadata_types.py", + "adapters/snowflake/_helpers/relation_changes.py", "adapters/snowflake/_helpers/show_metadata.py", "adapters/snowflake/classes/snowflake_adapter.py", "adapters/sqlserver/classes/sqlserver_adapter.py", diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py index 3eb644e84e..c95c119fb6 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/_test_types.py @@ -1,6 +1,10 @@ from __future__ import annotations from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import FakeRelation @dataclass(frozen=True) @@ -60,7 +64,7 @@ class SpeculativePrefetchTestCase: description: str failing_schema: str expected_error_fragment: str - expected_failed_reads: int + expected_failed_reads: tuple[int, int] @dataclass(frozen=True) @@ -214,3 +218,44 @@ class SessionDatabaseSwitchTestCase: switch_sql: str expected_scopes: tuple[str, ...] expected_session_reads: int + + +@dataclass(frozen=True) +class PartedListingTestCase: + """Schemas whose tables and views are listed by separate reads running together.""" + + description: str + schema_count: int + relations_per_schema: int + statement_latency_seconds: float + expected_max_concurrent: int + + +@dataclass(frozen=True) +class SyncedCatalogTestCase: + """A warehouse that moved on after the synced copy, and what a plan from the copy reads.""" + + description: str + changed: tuple[FakeRelation, ...] + dropped: tuple[str, ...] + expected_query_kinds: tuple[str, ...] + expected_refreshed: int + + +@dataclass(frozen=True) +class SyncedFallbackTestCase: + """A plan from synced state that cannot use the copy for a schema, and reads it live.""" + + description: str + empty_copy: bool + failing_check: bool + expected_message: str + + +@dataclass(frozen=True) +class ExactColumnThresholdTestCase: + """A column request of one schema's relations and how many reads of each kind serve it.""" + + description: str + relation_count: int + expected_query_kinds: dict[str, int] diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py index e049c88eac..70ab1ce838 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py @@ -16,12 +16,15 @@ from datetime import UTC, datetime from itertools import chain, compress from pathlib import Path -from typing import Any +from typing import Any, ClassVar import duckdb import pyarrow -from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog +from sqlbuild.adapter.relations.classes.synced_catalog import SyncedCatalog +from sqlbuild.adapter.relations.models import SyncedRelation from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection @@ -48,6 +51,12 @@ r"^SELECT\s+CAST\((?:MIN|MAX)\(.*\)\s+AS\s+VARCHAR\).*\s+FROM\s+(?P\S+)\s*$", re.IGNORECASE | re.DOTALL, ) +_CURSOR_BATCH_PATTERN: re.Pattern[str] = re.compile(r"^SELECT \d+ AS _read, ", re.IGNORECASE) +_CURSOR_BATCH_BRANCH: re.Pattern[str] = re.compile( + r"^SELECT (?P\d+) AS _read, (?P