diff --git a/Cargo.lock b/Cargo.lock index 636cc16748..544e9ef4cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4335,6 +4335,7 @@ dependencies = [ "sqlbuild-model-config", "sqlbuild-recall", "sqlbuild-state", + "sqlquack-fingerprint", "sqlx-core", "sqlx-postgres", "subtle", diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/models/fct_revenue.sql b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/fct_revenue.sql new file mode 100644 index 0000000000..85b8424713 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/fct_revenue.sql @@ -0,0 +1,2 @@ +MODEL (description "Order counts per customer, refreshed by the finance job.", materialized table); +SELECT customer_id, count(*) AS revenue FROM __ref("int_orders") GROUP BY customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/models/int_orders.sql b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/int_orders.sql new file mode 100644 index 0000000000..8563636afd --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/int_orders.sql @@ -0,0 +1,2 @@ +MODEL (description "Orders with their customer, refreshed by the staging job.", materialized table); +SELECT order_id, customer_id FROM __ref("stg_orders") diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/models/orders_export.sql b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/orders_export.sql new file mode 100644 index 0000000000..505c0ad53b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/orders_export.sql @@ -0,0 +1,2 @@ +MODEL (description "Orders exported by a job that nothing triggers.", materialized table); +SELECT order_id FROM __ref("stg_orders") diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/models/rpt_pnl.sql b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/rpt_pnl.sql new file mode 100644 index 0000000000..57a8e61f1f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/rpt_pnl.sql @@ -0,0 +1,2 @@ +MODEL (description "A report on revenue, refreshed by the finance job.", materialized table); +SELECT sum(revenue) AS total FROM __ref("fct_revenue") diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/models/summary.sql b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/summary.sql new file mode 100644 index 0000000000..fa2f40616b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/models/summary.sql @@ -0,0 +1,6 @@ +MODEL (description "Joins a branch the change reaches now with one that waits for the finance job.", materialized table); +SELECT c.customer_id, count(i.order_id) AS orders, max(f.revenue) AS revenue +FROM __ref("stg_customers") AS c +LEFT JOIN __ref("int_orders") AS i USING (customer_id) +LEFT JOIN __ref("fct_revenue") AS f USING (customer_id) +GROUP BY c.customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py new file mode 100644 index 0000000000..24b96260d5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py @@ -0,0 +1,15 @@ +"""Two scheduled jobs split the orders lineage and share int_orders; a third job has no trigger.""" + +from sqlbuild.orchestration import job, schedule + +daily = schedule(cron="0 2 * * *", timezone="UTC") +staging = job( + "staging_daily", description="Stage orders.", select="stg_orders int_orders", schedule=daily +) +finance = job( + "finance_daily", + description="Build revenue reports.", + select="int_orders fct_revenue rpt_pnl", + schedule=daily, +) +exports = job("adhoc_exports", description="Export orders on request.", select="orders_export") diff --git a/benchmarks/duckshop/edge_cases/changed_builds_two_up/models/customer_sales.sql b/benchmarks/duckshop/edge_cases/changed_builds_two_up/models/customer_sales.sql new file mode 100644 index 0000000000..c56d0b095c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_two_up/models/customer_sales.sql @@ -0,0 +1,2 @@ +MODEL (description "Sales per customer, two levels below changes held by different jobs.", materialized table); +SELECT customer_id, orders FROM __ref("customer_orders") WHERE orders > 0 diff --git a/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py new file mode 100644 index 0000000000..be61617149 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py @@ -0,0 +1,17 @@ +"""Two scheduled jobs each own one staging model; customer_orders reads both, in crm_daily.""" + +from sqlbuild.orchestration import job, schedule + +daily = schedule(cron="0 2 * * *", timezone="UTC") +crm = job( + "crm_daily", + description="Refresh customers.", + select="stg_customers customer_orders", + schedule=daily, +) +sales = job( + "sales_daily", + description="Refresh sales.", + select="stg_orders customer_sales", + schedule=daily, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py new file mode 100644 index 0000000000..377130a8b6 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py @@ -0,0 +1,17 @@ +"""A sensor on new source data starts a job that builds only what changed, source data included.""" + +from sqlbuild.orchestration import job, sensor, source_new_data + +orders_refresh = job( + "orders_refresh", + description="Refresh orders when the source has new data.", + select="orders", + settings={"changed": True}, +) +on_source_orders = sensor( + "on_source_orders", + description="Refresh orders for each changed source version.", + when=source_new_data(select="source:raw_orders"), + run=orders_refresh, + minimum_interval_seconds=1, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/fingerprint_scheme_change.json b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_scheme_change.json new file mode 100644 index 0000000000..897fde10e4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_scheme_change.json @@ -0,0 +1,13 @@ +{ + "description": "A model built before a fingerprint scheme change keeps its old recorded hash; the node is unchanged until its SQL changes.", + "dialect": "duckdb", + "definition": "select order_id,\n amount -- gross\nfrom orders", + "edited_definition": "select order_id, amount * 2 as amount from orders", + "old_scheme_hash": "scheme-orders-hash-before-upgrade", + "metadata": { + "model_name": "scheme_orders", + "config": {"materialized": "table"}, + "execution_signature": {}, + "local_function_hashes": {} + } +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs index 3011d88df7..6c3d893c95 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs @@ -1,12 +1,4 @@ -//! Fingerprint dialect aliases and the function language fingerprinted like a query. +//! The function language fingerprinted like a query. -/// The fingerprint dialect when a project names none. -pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; -/// Python's `NATIVE_DIALECT_ALIASES`. -pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 3] = [ - ("postgres", "postgresql"), - ("motherduck", "duckdb"), - ("sqlserver", "tsql"), -]; /// The function language whose definitions are fingerprinted like queries. pub(crate) const SQL_FUNCTION_LANGUAGE: &str = "sql"; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/compiler_query_fingerprint.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/compiler_query_fingerprint.rs index 946c2aab59..99d5571db1 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/compiler_query_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/compiler_query_fingerprint.rs @@ -1,16 +1,10 @@ //! Python's `compute_query_hash`: a query fingerprint under the project's dialect alias. -use crate::sql_tokens::constants::{FINGERPRINT_DIALECT_ALIASES, GENERIC_FINGERPRINT_DIALECT}; +use sqlbuild_core::fingerprint_identity::main::fingerprint_dialect::fingerprint_dialect; + use crate::sql_tokens::main::query_fingerprint::query_fingerprint; /// The fingerprint of `sql` under `dialect` (aliased, or generic when empty or absent). pub fn compiler_query_fingerprint(sql: &str, dialect: Option<&str>) -> Result { - let dialect = match dialect.filter(|name| !name.is_empty()) { - None => GENERIC_FINGERPRINT_DIALECT, - Some(name) => FINGERPRINT_DIALECT_ALIASES - .iter() - .find(|(alias, _)| *alias == name) - .map_or(name, |(_, target)| target), - }; - query_fingerprint(sql, dialect) + query_fingerprint(sql, fingerprint_dialect(dialect)) } diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index e6208078b5..93c34ee625 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -13,8 +13,9 @@ use sqlbuild_orchestration::runs::models::{CommandPlan, PlannerResource}; use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; use crate::adhoc::constants::{ - ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, COMMAND_DEST, FRESHNESS_VERB, INTEGER_CONVERTERS, - LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, + ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST, + FRESHNESS_VERB, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, + REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, }; use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::job_settings_arguments::job_settings_arguments; @@ -51,6 +52,13 @@ impl CommandPlanner for CliPlanner { selectors.insert(selector, texts(value)); continue; } + if dest == CHANGED_COUNTS_DEST { + settings.insert(CHANGED_DEST.to_owned(), json!(texts(value))); + continue; + } + if dest == CHANGED_DEST && settings.contains_key(CHANGED_DEST) { + continue; + } settings.insert(dest.clone(), setting(dest, value)?); } if verb == BUILD_VERB { diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index 9490e619b9..c4fc1ca671 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -15,6 +15,12 @@ pub(crate) const FRESHNESS_VERB: &str = "freshness"; /// The parser destination holding the verb itself, which is not a setting. pub(crate) const COMMAND_DEST: &str = "command"; +/// The destination holding the kinds `--changed=` counts; the job setting carries them as `changed`. +pub(crate) const CHANGED_COUNTS_DEST: &str = "changed_counts"; + +/// The job setting `--changed` becomes. +pub(crate) const CHANGED_DEST: &str = "changed"; + /// The destination `--load` and `--no-load` share; only `--no-load` is accepted. pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources"; @@ -22,8 +28,10 @@ pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources"; pub(crate) const SELECTOR_DESTS: [&str; 3] = ["select", "exclude", "positional_select"]; /// Destinations an ad-hoc run may set; the control plane gates history rewrites and warehouses. -pub(crate) const ALLOWED_DESTS: [&str; 27] = [ +pub(crate) const ALLOWED_DESTS: [&str; 29] = [ "max_microbatches", + "changed", + "reach", "run_tests", "run_audits", "include_python", diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 370c7bd492..8631f17666 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -95,6 +95,26 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select json!({"fail_fast": true}), )), }, + AdhocPlanTestCase { + description: "--changed with its counts and reach is the job setting a schedule would use", + command: &[ + "build", + "-s", + "orders", + "--changed=source_data,definition", + "--reach", + "all", + ], + expected_plan: Ok(( + &["model:orders"], + json!({"changed": ["definition", "source_data"], "reach": "all"}), + )), + }, + AdhocPlanTestCase { + description: "a bare --changed uses the project's [changes] defaults", + command: &["build", "-s", "orders", "--changed"], + expected_plan: Ok((&["model:orders"], json!({"changed": true}))), + }, AdhocPlanTestCase { description: "an abbreviated flag is the same flag, so it is still detected", command: &["build", "-s", "orders", "--full-ref"], @@ -144,11 +164,6 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select command: &["build", "--vars", "{\"day\": 1}"], expected_plan: Err("--vars is not available for ad-hoc runs"), }, - AdhocPlanTestCase { - description: "a flag outside the allowlist is refused by the name people typed", - command: &["build", "--changed"], - expected_plan: Err("--changed is not available for ad-hoc runs"), - }, AdhocPlanTestCase { description: "a selector file is refused", command: &["build", "--select-file", "selection.txt"], diff --git a/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs b/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs index fc5e1b4606..03c2ac7cd4 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs @@ -229,6 +229,7 @@ fn format_args(action: &ActionSpec, default: &str) -> String { match action.nargs { Nargs::Single => name, Nargs::Optional => format!("[{name}]"), + Nargs::Attached => format!("[={name}]"), Nargs::ZeroOrMore => format!("[{name} ...]"), Nargs::OneOrMore => format!("{name} [{name} ...]"), Nargs::Parser => format!("{name} ..."), @@ -252,6 +253,12 @@ fn invocation(action: &ActionSpec) -> String { if action.nargs == Nargs::Zero { return options; } + if action.nargs == Nargs::Attached { + return format!( + "{options}{}", + format_args(action, &dest(action).to_uppercase()) + ); + } format!( "{options} {}", format_args(action, &dest(action).to_uppercase()) @@ -438,6 +445,11 @@ fn usage_part(action: &ActionSpec, grouped: bool) -> Option { let option: &str = &action.option_strings[0]; let part: String = if action.nargs == Nargs::Zero { option.to_owned() + } else if action.nargs == Nargs::Attached { + format!( + "{option}{}", + format_args(action, &dest(action).to_uppercase()) + ) } else { format!( "{option} {}", diff --git a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs index 19bb1563e3..d867bc0438 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs @@ -6,7 +6,7 @@ use serde_json::Value as Json; use crate::argv::_helpers::table::root_parser; use crate::argv::constants::CONCURRENCY_DEST; -use crate::argv::models::{ActionKind, ActionSpec, Default, ParserSpec, Value}; +use crate::argv::models::{ActionKind, ActionSpec, Default, Nargs, ParserSpec, Value}; pub(crate) fn setting_argument( parser: &ParserSpec, @@ -45,6 +45,9 @@ pub(crate) fn setting_argument( return boolean_argument(name, value, &actions); } let option = &action.option_strings[0]; + if action.nargs == Nargs::Attached { + return attached_argument(name, value, option); + } let text = setting_text(name, value, action)?; if action .choices @@ -83,6 +86,25 @@ fn boolean_argument( Ok(None) } +/// `true` as the bare option, `false` as nothing, and a list of strings as `--option=a,b`. +fn attached_argument(name: &str, value: &Json, option: &str) -> Result, String> { + match value { + Json::Bool(true) => Ok(Some(option.to_owned())), + Json::Bool(false) => Ok(None), + Json::Array(items) if !items.is_empty() => { + let texts: Vec<&str> = items + .iter() + .map(Json::as_str) + .collect::>() + .ok_or_else(|| format!("job setting {name} must be true or a list of strings"))?; + Ok(Some(format!("{option}={}", texts.join(",")))) + } + _ => Err(format!( + "job setting {name} must be true or a list of strings" + )), + } +} + fn setting_text(name: &str, value: &Json, action: &ActionSpec) -> Result { match action.converter.as_deref() { Some("int" | "_positive_integer") => { diff --git a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs index b53bf9bf04..0d08aca26e 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs @@ -2,11 +2,11 @@ use crate::argv::_helpers::parsing::{Exit, KnownArgs, Run}; use crate::argv::constants::{ - COMMAND_DEST, CONCURRENCY_COMMANDS, CONCURRENCY_ENV_VAR, CURSOR_COMMANDS, DBT_COMMAND, - DBT_DEBUG_COMMAND, DBT_PASSTHROUGH_COMMANDS, DBT_PASSTHROUGH_SEPARATOR, DBT_VERBOSE_OPTION, - DBT_VERBOSE_OPTIONS, DEBUG_OPTION, INT_CONVERTER, JANITOR_COMMAND, NO_COLOR_OPTION, - POSITIONAL_SELECT_COMMANDS, RENAME_COMMAND, SCOPE_COMMAND, SCOPE_DEFAULT_PAGE_SIZE, - SCOPE_GLOBAL_SUMMARY, SCOPE_KIND_DEST, TEST_COMMAND, + CHANGE_COUNTS, COMMAND_DEST, CONCURRENCY_COMMANDS, CONCURRENCY_ENV_VAR, CURSOR_COMMANDS, + DBT_COMMAND, DBT_DEBUG_COMMAND, DBT_PASSTHROUGH_COMMANDS, DBT_PASSTHROUGH_SEPARATOR, + DBT_VERBOSE_OPTION, DBT_VERBOSE_OPTIONS, DEBUG_OPTION, INT_CONVERTER, JANITOR_COMMAND, + NO_COLOR_OPTION, POSITIONAL_SELECT_COMMANDS, RENAME_COMMAND, SCOPE_COMMAND, + SCOPE_DEFAULT_PAGE_SIZE, SCOPE_GLOBAL_SUMMARY, SCOPE_KIND_DEST, TEST_COMMAND, }; use crate::argv::models::{Namespace, ParserSpec, Value}; @@ -102,6 +102,8 @@ impl Run<'_> { "test --inspect cannot be combined with --json or --json-output", )); } + namespace = + with_changed_counts(namespace).map_err(|message| self.error(parser, &message))?; if let Some(message) = since_error(&namespace) { return Err(self.error(parser, message)); } @@ -298,3 +300,31 @@ fn subcommand_parser<'a>(parser: &'a ParserSpec, name: &str) -> Option<&'a Parse .find(|(command, _)| command == name) .map(|(_, spec)| spec) } + +/// `namespace` with `--changed=COUNTS` split into `changed_counts`; refuses `--reach` alone. +fn with_changed_counts(mut namespace: Namespace) -> Result { + if let Value::Str(text) = namespace.value("changed") { + let counts: Vec<&str> = text.split(',').map(str::trim).collect(); + if counts.iter().any(|count| !CHANGE_COUNTS.contains(count)) { + return Err(format!( + "--changed lists what counts as a change: one or more of {}, separated by commas, \ + not {text:?}", + CHANGE_COUNTS.join(", ") + )); + } + let mut listed: Vec = Vec::new(); + for count in CHANGE_COUNTS { + if counts.contains(&count) { + listed.push(Value::Str(count.to_owned())); + } + } + namespace.set("changed", Value::Bool(true)); + namespace.set("changed_counts", Value::List(listed)); + } else if truthy(&namespace.value("changed")) { + namespace.set("changed_counts", Value::None); + } + if namespace.value("reach") != Value::None && !truthy(&namespace.value("changed")) { + return Err("--reach needs --changed".to_owned()); + } + Ok(namespace) +} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs index 6516242c26..cb7c78461e 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs @@ -292,7 +292,11 @@ impl<'p, 'r, 'h> Level<'p, 'r, 'h> { action_tuples.push((index, self.arg_strings[first..stop].to_vec())); break; }; - let count: usize = match_argument(spec, b"A")?; + let count: usize = if spec.nargs == Nargs::Attached { + 1 + } else { + match_argument(spec, b"A")? + }; let second: char = option_string.chars().nth(1).unwrap_or_default(); if count == 1 { stop = start + 1; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/patterns.rs b/crates/sqlbuild-cli/src/argv/_helpers/patterns.rs index 59205784ea..1d700b2fc4 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/patterns.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/patterns.rs @@ -54,7 +54,8 @@ fn nargs_pattern(action: &ActionSpec) -> Vec { element("A", 1, 1), element("-AO", 0, UNBOUNDED), ], - (Nargs::Zero, true) => Vec::new(), + (Nargs::Zero | Nargs::Attached, true) => Vec::new(), + (Nargs::Attached, false) => vec![element("-", 0, UNBOUNDED)], (Nargs::Zero, false) => vec![element("-", 0, UNBOUNDED)], } } diff --git a/crates/sqlbuild-cli/src/argv/_helpers/table.rs b/crates/sqlbuild-cli/src/argv/_helpers/table.rs index c094f1210f..a319f7ba27 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/table.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/table.rs @@ -174,6 +174,7 @@ fn nargs(json: &Json) -> Nargs { Json::Number(_) => Nargs::Zero, Json::String(text) => match text.as_str() { "?" => Nargs::Optional, + "=?" => Nargs::Attached, "*" => Nargs::ZeroOrMore, "+" => Nargs::OneOrMore, _ => Nargs::Parser, diff --git a/crates/sqlbuild-cli/src/argv/_helpers/values.rs b/crates/sqlbuild-cli/src/argv/_helpers/values.rs index 79602bc483..de33818b52 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/values.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/values.rs @@ -40,7 +40,7 @@ impl Run<'_> { args: &[String], ) -> Result, String> { let positional: bool = action.option_strings.is_empty(); - if args.is_empty() && action.nargs == Nargs::Optional { + if args.is_empty() && matches!(action.nargs, Nargs::Optional | Nargs::Attached) { let value: Value = if positional { match &action.default { Default::Suppress => return Ok(None), @@ -61,7 +61,12 @@ impl Run<'_> { Default::Value(value) => Some(value.clone()), }); } - if args.len() == 1 && matches!(action.nargs, Nargs::Single | Nargs::Optional) { + if args.len() == 1 + && matches!( + action.nargs, + Nargs::Single | Nargs::Optional | Nargs::Attached + ) + { let value: Value = self.converted_value(action, &args[0])?; self.check_value(action, &value)?; return Ok(Some(value)); diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 97cddf35e6..56091bba6e 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1752,6 +1752,33 @@ "default": false, "group": 1 }, + { + "kind": "store", + "options": [ + "--changed" + ], + "dest": "changed", + "nargs": "=?", + "const": true, + "default": false, + "metavar": "COUNTS", + "help": "run only what changed, and what it reaches downstream; COUNTS lists what counts as a change, such as --changed=definition,source_data (default: [changes] counts, else definition)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reach" + ], + "dest": "reach", + "choices": [ + "job", + "all" + ], + "default": null, + "help": "with --changed: job stops at nodes another triggered job refreshes; all runs everything downstream (default: [changes] reach, else job)", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -2309,14 +2336,30 @@ "group": 1 }, { - "kind": "store_true", + "kind": "store", "options": [ "--changed" ], "dest": "changed", - "nargs": 0, + "nargs": "=?", "const": true, "default": false, + "metavar": "COUNTS", + "help": "run only what changed, and what it reaches downstream; COUNTS lists what counts as a change, such as --changed=definition,source_data (default: [changes] counts, else definition)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reach" + ], + "dest": "reach", + "choices": [ + "job", + "all" + ], + "default": null, + "help": "with --changed: job stops at nodes another triggered job refreshes; all runs everything downstream (default: [changes] reach, else job)", "group": 1 }, { diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index e4a1b23bfc..5d238abd5b 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -2,6 +2,8 @@ use crate::argv::models::NamespaceDefault; +/// What `--changed=COUNTS` may list. +pub(crate) const CHANGE_COUNTS: [&str; 2] = ["definition", "source_data"]; /// The exported `sqb` parser tree: every parser, group and action in `add_argument` order. pub(crate) const COMMAND_TABLE: &str = include_str!("command_table.json"); /// The exported `sqb dbt plan|run|build` execution parser, its command name a placeholder. @@ -71,7 +73,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 185] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 187] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -127,6 +129,8 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 185] = [ ("fail_fast", NamespaceDefault::Bool(false)), ("full_refresh", NamespaceDefault::Bool(false)), ("changed", NamespaceDefault::Bool(false)), + ("changed_counts", NamespaceDefault::None), + ("reach", NamespaceDefault::None), ("as_target", NamespaceDefault::None), ("include_python", NamespaceDefault::Bool(true)), ("load_sources", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/argv/models.rs b/crates/sqlbuild-cli/src/argv/models.rs index 44d88ac46a..35da9d6276 100644 --- a/crates/sqlbuild-cli/src/argv/models.rs +++ b/crates/sqlbuild-cli/src/argv/models.rs @@ -13,6 +13,8 @@ pub enum Nargs { Zero, /// `nargs='?'`. Optional, + /// `nargs='=?'`: an option whose value is given only as `--option=value`, never as the next string. + Attached, /// `nargs='*'`. ZeroOrMore, /// `nargs='+'`. diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index 2524c91185..191a753c36 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -19,6 +19,36 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( attribute: "select", expected_value: strings(&["orders", "customers", "products"]), }, + ParsedValueTestCase { + description: "a bare --changed leaves the next string as a selector", + argv: &["build", "--changed", "definition"], + attribute: "select", + expected_value: strings(&["definition"]), + }, + ParsedValueTestCase { + description: "a bare --changed uses the configured counts", + argv: &["build", "--changed", "tag:finance"], + attribute: "changed_counts", + expected_value: Value::None, + }, + ParsedValueTestCase { + description: "--changed=COUNTS turns --changed on", + argv: &["build", "--changed=source_data"], + attribute: "changed", + expected_value: Value::Bool(true), + }, + ParsedValueTestCase { + description: "--changed=COUNTS lists the counts in canonical order", + argv: &["plan", "--changed=source_data,definition"], + attribute: "changed_counts", + expected_value: strings(&["definition", "source_data"]), + }, + ParsedValueTestCase { + description: "--reach takes its value as the next string", + argv: &["build", "--changed", "--reach", "all", "orders"], + attribute: "reach", + expected_value: Value::Str("all".to_owned()), + }, ParsedValueTestCase { description: "an unset concurrency stays None", argv: &["build"], @@ -131,6 +161,24 @@ fn given_help_version_or_rejected_arguments_when_parsing_then_exits_after_writin expected_exit: 2, expected_last_write: "sqb: error[C900]: --concurrency must be >= 1\n\n", }, + ExitTestCase { + description: "--reach without --changed is refused", + argv: &["build", "--reach", "all"], + expected_exit: 2, + expected_last_write: "sqb: error[C900]: --reach needs --changed\n\n", + }, + ExitTestCase { + description: "--changed lists only known kinds of change", + argv: &["build", "--changed=definition,schedule"], + expected_exit: 2, + expected_last_write: "sqb: error[C900]: --changed lists what counts as a change: one or more of definition, source_data, separated by commas, not \"definition,schedule\"\n\n", + }, + ExitTestCase { + description: "--reach accepts job or all", + argv: &["build", "--changed", "--reach", "group"], + expected_exit: 2, + expected_last_write: "sqb build: error[C900]: argument --reach: invalid choice: 'group' (choose from 'job', 'all')\n\n", + }, ExitTestCase { description: "a missing required sub-command is named", argv: &["rules"], @@ -245,6 +293,28 @@ fn given_job_settings_when_validating_then_values_survive_the_build_parser() { ], expected_empty: false, }, + JobSettingsTestCase { + description: "changed as true or as a list of counts, with a reach", + settings: BTreeMap::from([ + ("changed".into(), json!(["source_data", "definition"])), + ("reach".into(), json!("all")), + ]), + expected_values: vec![ + ("changed", Value::Bool(true)), + ("changed_counts", strings(&["definition", "source_data"])), + ("reach", Value::Str("all".into())), + ], + expected_empty: false, + }, + JobSettingsTestCase { + description: "changed as true uses the configured counts", + settings: BTreeMap::from([("changed".into(), json!(true))]), + expected_values: vec![ + ("changed", Value::Bool(true)), + ("changed_counts", Value::None), + ], + expected_empty: false, + }, JobSettingsTestCase { description: "default-valued flags do not become explicit options", settings: BTreeMap::from([ diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 9e50dee08d..43487a3698 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -110,7 +110,7 @@ pub(crate) const DAG_ROUTE: Route = routed( }, ); -const PLAN_FIELDS: [Field; 19] = [ +const PLAN_FIELDS: [Field; 22] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -121,6 +121,9 @@ const PLAN_FIELDS: [Field; 19] = [ CURSOR_OVERRIDES, JSON_OUTPUT, same("full_refresh"), + same("changed"), + same("changed_counts"), + same("reach"), same("load_sources"), same("include_python"), NO_COLOR, @@ -139,7 +142,7 @@ pub(crate) const PLAN_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const BUILD_FIELDS: [Field; 36] = [ +const BUILD_FIELDS: [Field; 38] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -152,6 +155,8 @@ const BUILD_FIELDS: [Field; 36] = [ same("fail_fast"), same("full_refresh"), same("changed"), + same("changed_counts"), + same("reach"), same("load_sources"), attr("reload_sources", "reload"), same("include_python"), diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 0521263158..09e0abbf49 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -14,8 +14,8 @@ use crate::project::_helpers::help::{ }; use crate::project::_helpers::keys::Place; use crate::project::_helpers::sections::{ - load_constants, load_control, load_cost, load_cursors, load_dbt, load_diff, load_git, - load_janitor, load_local_dbt, load_microbatches, load_migrations, load_policy_flag, + load_changes, load_constants, load_control, load_cost, load_cursors, load_dbt, load_diff, + load_git, load_janitor, load_local_dbt, load_microbatches, load_migrations, load_policy_flag, load_review, load_scenario, load_settings, load_snapshots, setting_overrides, }; use crate::project::_helpers::sinks::load_sinks; @@ -28,7 +28,9 @@ use crate::project::constants::{ LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; -use crate::project::models::{Built, ConfigFile, ConfigPaths, ControlConfig, ReviewConfig}; +use crate::project::models::{ + Built, ChangesConfig, ConfigFile, ConfigPaths, ControlConfig, ReviewConfig, +}; use crate::project::types::{Entries, Fields, Loaded}; use crate::recall::constants::RECALL_SECTION; use crate::recall::main::recall_section::recall_section; @@ -279,6 +281,7 @@ fn project_fields(payload: &Entries) -> Loaded { load_git(get(payload, "git"), PROJECT_FILE)?; load_review(get(payload, "review"), PROJECT_FILE)?; load_control(get(payload, "control"), PROJECT_FILE)?; + load_changes(get(payload, "changes"), PROJECT_FILE)?; let mut fields = identity(payload)?; fields.extend(sections(payload)?); fields.push(("targets", targets(payload)?)); @@ -449,6 +452,11 @@ pub(crate) fn control_config(paths: &ConfigPaths) -> Result Result { + project_section(paths, "changes", load_changes) +} + /// `[review]` of `sqlbuild_project.toml`. pub(crate) fn review_config(paths: &ConfigPaths) -> Result { project_section(paths, "review", load_review) diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 2bceff46dc..32947c975f 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -23,14 +23,15 @@ use crate::project::constants::{ USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ - CONTROL_KEYS, MIN_SYNC_INTERVAL_SECONDS, SYNC_INTERVAL_HELP, SYNC_INTERVAL_OFF, + CHANGE_COUNTS, CHANGE_REACHES, CHANGES_KEYS, CONTROL_KEYS, MIN_SYNC_INTERVAL_SECONDS, + SYNC_INTERVAL_HELP, SYNC_INTERVAL_OFF, }; use crate::project::constants::{ DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, }; use crate::project::errors::ConfigFailure; -use crate::project::models::{Built, ConfigFile, ControlConfig, ReviewConfig}; +use crate::project::models::{Built, ChangesConfig, ConfigFile, ControlConfig, ReviewConfig}; use crate::project::types::{Entries, Fields, Loaded}; use sqlbuild_core::constants::ZERO_DAY_DURATION; @@ -624,6 +625,51 @@ pub(crate) fn load_control( }) } +/// `[changes]`: what `--changed` counts as a change and how far a change reaches downstream. +pub(crate) fn load_changes( + payload: Option<&ConfigValue>, + file: ConfigFile, +) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "changes"))?; + validate_allowed_keys(mapping, &CHANGES_KEYS, place(file, "changes"))?; + let counts: Option> = match get(mapping, "counts") { + None => None, + payload => { + let counts: Vec = string_sequence(payload, place(file, "changes.counts"))?; + if counts.is_empty() + || counts + .iter() + .any(|count| !CHANGE_COUNTS.contains(&count.as_str())) + { + return Err(ConfigFailure::at( + file, + format!( + "changes.counts must list one or more of: {}", + CHANGE_COUNTS.join(", ") + ), + )); + } + Some(counts) + } + }; + let reach: Option = match get(mapping, "reach") { + None => None, + Some(ConfigValue::String(text)) if CHANGE_REACHES.contains(&text.as_str()) => { + Some(text.clone()) + } + Some(_) => { + return Err(ConfigFailure::at( + file, + format!( + "changes.reach must be one of: {}", + CHANGE_REACHES.join(", ") + ), + )); + } + }; + Ok(ChangesConfig { counts, reach }) +} + /// `[review]`: the defaults of `sqb review`, validated so a typo fails like any other section. pub(crate) fn load_review(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { let mapping = coerce_mapping(payload, place(file, "review"))?; diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 987a3563ea..5bc122f08b 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -201,6 +201,11 @@ pub(crate) const TARGET_KEYS: [&str; 21] = [ "concurrency", ]; pub(crate) const CONTROL_KEYS: [&str; 3] = ["location", "server", "sync_at_least_every"]; +pub(crate) const CHANGES_KEYS: [&str; 2] = ["counts", "reach"]; +/// What `[changes] counts` may list, in the order `--changed` prints them. +pub(crate) const CHANGE_COUNTS: [&str; 2] = ["definition", "source_data"]; +/// How far `[changes] reach` lets a change run downstream. +pub(crate) const CHANGE_REACHES: [&str; 2] = ["job", "all"]; /// Shortest safety sync interval, so a typo cannot bring back aggressive warehouse polling. pub(crate) const MIN_SYNC_INTERVAL_SECONDS: u64 = 60; pub(crate) const SYNC_INTERVAL_OFF: [&str; 3] = ["0", "0s", "off"]; diff --git a/crates/sqlbuild-config/src/project/main/load_changes_config.rs b/crates/sqlbuild-config/src/project/main/load_changes_config.rs new file mode 100644 index 0000000000..10261eb27b --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/load_changes_config.rs @@ -0,0 +1,9 @@ +//! Read `[changes]` from `sqlbuild_project.toml`. + +use crate::project::errors::LoadFailure; +use crate::project::models::{ChangesConfig, ConfigPaths}; + +/// What `--changed` counts and how far a change reaches; each field is `None` when unset. +pub fn load_changes_config(paths: &ConfigPaths) -> Result { + crate::project::_helpers::documents::changes_config(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 7d1a244037..d44f5e229b 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -1,5 +1,6 @@ pub mod deprecated_setting_warnings; pub mod explicit_references_help; +pub mod load_changes_config; pub mod load_control_config; pub mod load_git_base_ref; pub mod load_local_config; diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index 3c9b9b461e..ef8016f60e 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -67,6 +67,15 @@ pub struct ControlConfig { pub sync_at_least_every_seconds: Option, } +/// `[changes]` of `sqlbuild_project.toml`: what `--changed` counts and how far a change reaches. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ChangesConfig { + /// Non-empty, each one of `CHANGE_COUNTS`. + pub counts: Option>, + /// One of `CHANGE_REACHES`. + pub reach: Option, +} + /// `[review]` of `sqlbuild_project.toml`: the defaults `sqb review` uses when flags are omitted. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ReviewConfig { diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 7b57c16140..0fb8d81691 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,13 +1,14 @@ use crate::project::errors::ConfigFailure; +use crate::project::main::load_changes_config::load_changes_config; use crate::project::main::load_control_config::load_control_config; use crate::project::main::load_git_base_ref::load_git_base_ref; use crate::project::main::load_review_config::load_review_config; use crate::project::models::ConfigFile; -use crate::project::models::{ControlConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, ReviewConfig}; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; use crate::project::tests::test_types::{ - ControlConfigTestCase, GitBaseRefTestCase, ProjectFailureTestCase, ReviewConfigErrorTestCase, - ReviewConfigTestCase, + ChangesConfigTestCase, ControlConfigTestCase, GitBaseRefTestCase, ProjectFailureTestCase, + ReviewConfigErrorTestCase, ReviewConfigTestCase, }; const PROJECT: &str = "name = 'orders'\nadapter = 'duckdb'\n"; @@ -374,3 +375,93 @@ fn given_invalid_control_section_when_loading_then_project_loading_rejects_it() } } } + +#[test] +fn given_changes_section_when_reading_then_returns_counts_and_reach() { + let test_cases = [ + ChangesConfigTestCase { + description: "both kinds of change and every node downstream", + project: "name = 'orders'\nadapter = 'duckdb'\n[changes]\n\ + counts = ['definition', 'source_data']\nreach = 'all'\n", + expected_config: ChangesConfig { + counts: Some(vec!["definition".to_owned(), "source_data".to_owned()]), + reach: Some("all".to_owned()), + }, + }, + ChangesConfigTestCase { + description: "only new source data, reach left to the default", + project: "name = 'orders'\nadapter = 'duckdb'\n[changes]\ncounts = ['source_data']\n", + expected_config: ChangesConfig { + counts: Some(vec!["source_data".to_owned()]), + reach: None, + }, + }, + ChangesConfigTestCase { + description: "no changes section", + project: PROJECT, + expected_config: ChangesConfig::default(), + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let config: ChangesConfig = load_changes_config(&paths).expect(test_case.description); + + assert_eq!( + config, test_case.expected_config, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_changes_section_when_loading_then_project_loading_rejects_it() { + let counts = "changes.counts must list one or more of: definition, source_data"; + let test_cases = [ + ReviewConfigErrorTestCase { + description: "an empty counts list", + project: "name = 'orders'\nadapter = 'duckdb'\n[changes]\ncounts = []\n", + expected_fragment: counts, + }, + ReviewConfigErrorTestCase { + description: "an unknown kind of change", + project: "name = 'orders'\nadapter = 'duckdb'\n[changes]\ncounts = ['schedule']\n", + expected_fragment: counts, + }, + ReviewConfigErrorTestCase { + description: "counts as one string", + project: "name = 'orders'\nadapter = 'duckdb'\n[changes]\ncounts = 'definition'\n", + expected_fragment: "changes.counts must be a list of strings", + }, + ReviewConfigErrorTestCase { + description: "an unknown reach", + project: "name = 'orders'\nadapter = 'duckdb'\n[changes]\nreach = 'group'\n", + expected_fragment: "changes.reach must be one of: job, all", + }, + ReviewConfigErrorTestCase { + description: "a misspelt changes key", + project: "name = 'orders'\nadapter = 'duckdb'\n[changes]\ncount = ['definition']\n", + expected_fragment: "changes contains unknown key(s): count", + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let failures: [String; 2] = [ + format!( + "{:?}", + load_changes_config(&paths).expect_err(test_case.description) + ), + format!("{:?}", first_failure(&paths).expect(test_case.description)), + ]; + + for failure in failures { + assert!( + failure.contains(test_case.expected_fragment), + "{}: {failure}", + test_case.description + ); + } + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index 66f541daa4..3f25b8d921 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::project::errors::ConfigFailure; -use crate::project::models::{ControlConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, ReviewConfig}; pub(super) struct ProjectFailureTestCase { pub(super) description: &'static str, @@ -21,6 +21,12 @@ pub(crate) struct GitBaseRefTestCase { pub(crate) expected_base_ref: Option<&'static str>, } +pub(crate) struct ChangesConfigTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) expected_config: ChangesConfig, +} + pub(crate) struct ControlConfigTestCase { pub(crate) description: &'static str, pub(crate) project: &'static str, diff --git a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs index 7b752a338a..c32ef975c0 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs @@ -1,6 +1,14 @@ -//! Non-definition metadata excluded by the planner's version identity. +//! Non-definition metadata excluded by the planner's version identity, and fingerprint dialect names. pub const AUDIT_GATE_METADATA_KEY: &str = "audit_gate"; pub const MIGRATION_FINGERPRINT_METADATA_KEY: &str = "migration_fingerprint"; pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; +/// The fingerprint dialect when a project names none. +pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; +/// Python's `NATIVE_DIALECT_ALIASES`. +pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 3] = [ + ("postgres", "postgresql"), + ("motherduck", "duckdb"), + ("sqlserver", "tsql"), +]; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/fingerprint_dialect.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/fingerprint_dialect.rs new file mode 100644 index 0000000000..377194898a --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/fingerprint_dialect.rs @@ -0,0 +1,17 @@ +//! The fingerprint dialect a project's SQL dialect name maps to. + +use crate::fingerprint_identity::constants::{ + FINGERPRINT_DIALECT_ALIASES, GENERIC_FINGERPRINT_DIALECT, +}; + +/// Python's `compute_query_hash` dialect: the aliased name, or generic when empty or absent. +#[must_use] +pub fn fingerprint_dialect(dialect: Option<&str>) -> &str { + match dialect.filter(|name| !name.is_empty()) { + None => GENERIC_FINGERPRINT_DIALECT, + Some(name) => FINGERPRINT_DIALECT_ALIASES + .iter() + .find(|(alias, _)| *alias == name) + .map_or(name, |(_, target)| target), + } +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs index 914d2d4cf4..86131ac87e 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs @@ -1,5 +1,6 @@ pub mod canonical_text; pub mod decoded_object; +pub mod fingerprint_dialect; pub mod model_local_identity_hash; pub mod normalize_version_identity_metadata; pub mod python_equal; diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index edbf035ab1..ce822cc0b6 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -29,6 +29,7 @@ sqlbuild-cursor.workspace = true sqlbuild-model-config.workspace = true sqlbuild-recall.workspace = true sqlbuild-state.workspace = true +sqlquack-fingerprint.workspace = true sqlx-core.workspace = true sqlx-postgres.workspace = true subtle.workspace = true diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs index 8b77655a04..06e05befb0 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs @@ -1,16 +1,19 @@ //! Indexed exact-node/destination reads, shared by evidence, definition comparison and the catalog. +use std::collections::{BTreeSet, HashMap}; + use uuid::Uuid; use crate::store::errors::ControlError; -use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::main::{decode_row::decode_row, decompress::decompress, query::query}; use crate::store::models::{ControlTransaction, Parameter}; use crate::warehouse_metadata::models::{FingerprintObservation, FingerprintQuery}; /// Nodes read per statement; well under both stores' bound-parameter limits. const NODES_PER_STATEMENT: usize = 100; -type LatestBuild = Option<(FingerprintObservation, String)>; +/// A node's newest build, its recorded metadata JSON and the hash of its recorded definition text. +type LatestBuild = Option<(FingerprintObservation, String, String)>; pub(crate) async fn latest_fingerprint( transaction: ControlTransaction, @@ -94,7 +97,8 @@ pub(crate) async fn latest_fingerprints( (transaction, found) = query(transaction, &format!("SELECT w.column1 AS position,f.event_id,f.observed_at,f.source_event_id,f.record_json ->> 'run_id' AS run_id, \ f.record_json ->> 'definition_hash' AS definition_hash,f.record_json ->> 'version_hash' AS version_hash, \ - f.record_json ->> 'schema_fingerprint' AS schema_fingerprint,f.record_json ->> 'metadata_json' AS metadata_json \ + f.record_json ->> 'schema_fingerprint' AS schema_fingerprint,f.record_json ->> 'metadata_json' AS metadata_json, \ + f.definition_text_hash \ FROM (VALUES {}) AS w JOIN sqb_warehouse_fingerprints f ON f.event_id=(SELECT g.event_id FROM sqb_warehouse_fingerprints g \ WHERE g.location_id=$1 AND g.target=$2 AND g.node_type=w.column2 AND g.node_name=w.column3 AND {} \ ORDER BY g.target_database,g.target_schema,g.target_name,g.observed_at DESC,g.event_id DESC LIMIT 1)", @@ -107,9 +111,50 @@ pub(crate) async fn latest_fingerprints( .remove("metadata_json") .and_then(|value| value.as_str().map(str::to_owned)) .unwrap_or_default(); - builds[position] = Some((decode_row(row)?, metadata)); + let text_hash = row + .remove("definition_text_hash") + .and_then(|value| value.as_str().map(str::to_owned)) + .unwrap_or_default(); + builds[position] = Some((decode_row(row)?, metadata, text_hash)); } } } Ok((transaction, builds)) } + +/// The recorded definition texts with these hashes; a hash with no stored text is left out. +pub(crate) async fn definition_texts( + mut transaction: ControlTransaction, + hashes: &BTreeSet, +) -> Result<(ControlTransaction, HashMap), ControlError> { + let hashes: Vec<&String> = hashes.iter().collect(); + let mut texts: HashMap = HashMap::new(); + for chunk in hashes.chunks(NODES_PER_STATEMENT) { + let placeholders: Vec = + (1..=chunk.len()).map(|index| format!("${index}")).collect(); + let found; + (transaction, found) = query( + transaction, + &format!( + "SELECT text_hash, body FROM sqb_source_texts WHERE text_hash IN ({})", + placeholders.join(",") + ), + chunk + .iter() + .map(|hash| Parameter::Text((*hash).clone())) + .collect(), + ) + .await?; + for mut row in found { + let hash = row + .remove("text_hash") + .and_then(|value| value.as_str().map(str::to_owned)) + .unwrap_or_default(); + let body = decompress(&row.remove("body").unwrap_or_default())?; + if let Ok(text) = String::from_utf8(body) { + texts.insert(hash, text); + } + } + } + Ok((transaction, texts)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs index 57a1ede606..43f3352405 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs @@ -1,12 +1,18 @@ //! Physical observations and planner-local definitions remain independent evidence dimensions. +use std::collections::{BTreeSet, HashMap}; + +use sqlbuild_core::fingerprint_identity::main::fingerprint_dialect::fingerprint_dialect; use sqlbuild_core::fingerprint_identity::main::recorded_model_local_hash::recorded_model_local_hash; +use sqlquack_fingerprint::{FingerprintDialect, FingerprintError}; use uuid::Uuid; +use crate::snapshots::constants::MODEL_NODE_KIND; use crate::snapshots::models::SnapshotNode; use crate::store::errors::ControlError; use crate::store::main::{decode_row::decode_row, query::query}; use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::_helpers::fingerprint_reads::definition_texts; use crate::warehouse_metadata::_helpers::ingestion::scoped_event_id; use crate::warehouse_metadata::models::{ DefinitionEvidence, FingerprintObservation, FingerprintQuery, PhysicalEvidence, @@ -23,13 +29,87 @@ pub(crate) fn destination(node: &SnapshotNode) -> Option { }) } +/// A node's newest build: its record, recorded metadata JSON and recorded definition text hash. +type RecordedBuild = (FingerprintObservation, String, String); + +/// Each node's definition evidence, reading the recorded texts its comparison re-fingerprints. +pub(crate) async fn definition_evidences( + transaction: ControlTransaction, + builds: &[(&SnapshotNode, Option<&RecordedBuild>)], +) -> Result<(ControlTransaction, Vec), ControlError> { + let hashes: BTreeSet = builds + .iter() + .filter_map(|(node, build)| rehash_text(node, *build)) + .map(str::to_owned) + .collect(); + let (transaction, texts): (ControlTransaction, HashMap) = + definition_texts(transaction, &hashes).await?; + let evidences: Vec = builds + .iter() + .map(|(node, build)| { + let recorded_text: Option<&str> = rehash_text(node, *build) + .and_then(|hash| texts.get(hash)) + .map(String::as_str); + definition_evidence(node, *build, recorded_text) + }) + .collect(); + Ok((transaction, evidences)) +} + +/// One node's definition evidence, reading the recorded text its comparison re-fingerprints. +pub(crate) async fn node_definition_evidence( + transaction: ControlTransaction, + node: &SnapshotNode, + build: Option<&RecordedBuild>, +) -> Result<(ControlTransaction, DefinitionEvidence), ControlError> { + let (transaction, evidences) = definition_evidences(transaction, &[(node, build)]).await?; + let evidence: DefinitionEvidence = + evidences + .into_iter() + .next() + .unwrap_or(DefinitionEvidence::Unavailable { + reason: "no definition evidence was read".into(), + current_local_hash: None, + }); + Ok((transaction, evidence)) +} + +/// The recorded text hash to re-fingerprint, for a model that differs and has a snapshot dialect. +fn rehash_text<'a>(node: &SnapshotNode, build: Option<&'a RecordedBuild>) -> Option<&'a str> { + let (_, _, text_hash) = build?; + (node.kind == MODEL_NODE_KIND + && node.definition["planner_identity"].get("dialect").is_some() + && matches!( + definition_evidence(node, build, None), + DefinitionEvidence::Changed { .. } + )) + .then_some(text_hash.as_str()) +} + +/// The planner's query fingerprint of `sql` under `dialect`, or `recorded` when it does not tokenize. +fn query_hash(sql: &str, dialect: Option<&str>, recorded: &str) -> String { + let dialect = FingerprintDialect::from_name(fingerprint_dialect(dialect)) + .unwrap_or(FingerprintDialect::Generic); + sqlquack_fingerprint::query_fingerprint(sql, dialect) + .unwrap_or_else(|_: FingerprintError| recorded.to_owned()) +} + +/// Compare planner-local identities, re-fingerprinting `recorded_text` under the snapshot's dialect. pub(crate) fn definition_evidence( node: &SnapshotNode, - build: Option<&(FingerprintObservation, String)>, + build: Option<&RecordedBuild>, + recorded_text: Option<&str>, ) -> DefinitionEvidence { - let current = node.definition["planner_identity"]["local_hash"].as_str(); - let recorded = build.and_then(|(record, metadata)| match node.kind.as_str() { - "model" => recorded_model_local_hash(&record.definition_hash, metadata), + let identity = &node.definition["planner_identity"]; + let current = identity["local_hash"].as_str(); + let recorded = build.and_then(|(record, metadata, _)| match node.kind.as_str() { + "model" => { + let definition_hash: String = recorded_text.zip(identity.get("dialect")).map_or_else( + || record.definition_hash.clone(), + |(text, dialect)| query_hash(text, dialect.as_str(), &record.definition_hash), + ); + recorded_model_local_hash(&definition_hash, metadata) + } _ => None, }); match (current, recorded) { diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs index 28eb6a962d..90b60ff129 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs @@ -14,7 +14,7 @@ use crate::warehouse_metadata::_helpers::catalog_nodes::{ catalog_node, last_outcome, latest_attempts, }; use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprints; -use crate::warehouse_metadata::_helpers::node_status::{definition_evidence, destination}; +use crate::warehouse_metadata::_helpers::node_status::{definition_evidences, destination}; use crate::warehouse_metadata::models::{ CatalogNodeBuild, CatalogNodeRecord, DefinitionEvidence, FingerprintQuery, }; @@ -52,14 +52,20 @@ pub async fn list_person_catalog_nodes( let (transaction, mut attempts) = latest_attempts(transaction, (location_id, target)).await?; let (transaction, mut checks) = read_node_check_counts(transaction, (location_id, target)).await?; + let (transaction, evidences) = definition_evidences( + transaction, + &positions + .iter() + .zip(&found) + .map(|(position, build)| (nodes[*position].1, build.as_ref())) + .collect::>(), + ) + .await?; let mut builds: Vec> = vec![None; nodes.len()]; let mut changed: Vec = vec![false; nodes.len()]; - for (position, build) in positions.iter().zip(found) { - changed[*position] = matches!( - definition_evidence(nodes[*position].1, build.as_ref()), - DefinitionEvidence::Changed { .. } - ); - builds[*position] = build.map(|(record, _)| CatalogNodeBuild { + for ((position, build), evidence) in positions.iter().zip(found).zip(evidences) { + changed[*position] = matches!(evidence, DefinitionEvidence::Changed { .. }); + builds[*position] = build.map(|(record, _, _)| CatalogNodeBuild { observed_at: record.observed_at, run_id: record.run_id, version_hash: record.version_hash, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_fingerprint.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_fingerprint.rs index 195e811a72..284e1cd0a5 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_fingerprint.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_fingerprint.rs @@ -31,5 +31,5 @@ pub async fn read_person_fingerprint( let (transaction, evidence) = latest_fingerprint(transaction, (location_id, target, options)).await?; commit(transaction).await?; - Ok(evidence.map(|(record, _)| record)) + Ok(evidence.map(|(record, _, _)| record)) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs index 95bb96b049..737f00bb45 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs @@ -11,10 +11,10 @@ use crate::store::main::commit::commit; use crate::store::models::ControlStore; use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprint; use crate::warehouse_metadata::_helpers::node_status::{ - definition_evidence, destination, physical_evidence, + destination, node_definition_evidence, physical_evidence, }; use crate::warehouse_metadata::models::{ - NodeStatus, NodeStatusQuery, PhysicalEvidence, SourceDataStatus, + DefinitionEvidence, NodeStatus, NodeStatusQuery, PhysicalEvidence, SourceDataStatus, }; pub async fn read_person_node_status( @@ -75,12 +75,15 @@ pub async fn read_person_node_status( reason: "node has no physical relation destination".into(), } }; + let definition: DefinitionEvidence; + (transaction, definition) = + node_definition_evidence(transaction, &node, build.as_ref()).await?; let status = NodeStatus { snapshot_id: snapshot.snapshot_id, node_key: options.node_key.clone(), physical, - definition: definition_evidence(&node, build.as_ref()), - build: build.map(|(record, _)| record), + definition, + build: build.map(|(record, _, _)| record), source_data: SourceDataStatus::Unknown { reason: "consumed-input provenance is not recorded".into(), }, diff --git a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs index cba2166f6f..00b594f9d2 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs @@ -4,6 +4,8 @@ use serde_json::{Value, json}; use std::cell::Cell; use uuid::Uuid; +use sqlbuild_core::fingerprint_identity::main::recorded_model_local_hash::recorded_model_local_hash; + use crate::credentials::models::{AuditContext, DefaultRole}; use crate::runs::tests::helpers::{request, request_run, snapshot}; use crate::snapshots::main::publish_snapshot::publish_snapshot; @@ -270,6 +272,8 @@ pub(crate) async fn node_status_reads( (location, input.clone()), ) .await; + fingerprint_scheme_change_reads(store, credentials, (location, session_id), input.clone()) + .await; assert!(matches!( read_person_node_status( store, @@ -612,3 +616,129 @@ async fn list_wide_target( .await; (timings[1], statements) } + +/// A build recorded under an earlier fingerprint scheme reads as unchanged until its SQL changes. +async fn fingerprint_scheme_change_reads( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location, session_id): (Uuid, Uuid), + input: SnapshotInput, +) { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration"); + let data: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("fingerprint_scheme_change.json")).expect("fixture"), + ) + .expect("scheme change inputs"); + let text = |key: &str| data[key].as_str().expect(key).to_owned(); + let metadata: String = data["metadata"].to_string(); + let current_hash: String = recorded_model_local_hash( + &sqlquack_fingerprint::query_fingerprint( + &text("definition"), + sqlquack_fingerprint::FingerprintDialect::DuckDB, + ) + .expect("current fingerprint"), + &metadata, + ) + .expect("current local hash"); + let with_dialect: Value = json!({"local_hash": current_hash, "dialect": data["dialect"]}); + let without_dialect: Value = json!({"local_hash": current_hash}); + let publish = |identity: &Value, generation: i64| { + let mut input = input.clone(); + input.generation = generation; + input.nodes.insert("model:scheme_orders".into(), SnapshotNode { + kind: "model".into(), + definition: json!({"name":"scheme_orders","destination":{"database":null,"schema":"main","name":"scheme_orders"}, + "planner_identity":identity,"deps":[]}), + raw_sql: None, compiled_sql: None, python_source: None, + }); + input + }; + publish_snapshot(store, &publish(&with_dialect, 50)) + .await + .expect("snapshot with dialect"); + let build = |generation: i64, event: &str, definition: &str, ts: &str| -> MetadataBatch { + serde_json::from_value(json!({"kind":"fingerprints","session_id":session_id,"generation":generation,"records":[{ + "event_id": event, "state_database": null, "state_schema": "build_history", "node_type": "model", + "node_name": "scheme_orders", "target_database": null, "target_schema": "main", "target_name": "scheme_orders", + "run_id": event, "definition_hash": data["old_scheme_hash"], "version_hash": "scheme-version", + "schema_fingerprint": "scheme-schema", "definition": definition, "metadata_json": metadata, "ts": ts}]})) + .expect("scheme build") + }; + ingest_fingerprints( + store, + (location, "development"), + &build( + 50, + "scheme-built", + &text("definition"), + "2026-01-06T10:00:00Z", + ), + ) + .await + .expect("pre-upgrade build"); + let options = NodeStatusQuery { + node_key: "model:scheme_orders".into(), + }; + let status = || async { + let status = json!( + read_person_node_status( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("scheme status") + ); + let listed = list_person_catalog_nodes( + store, + credentials, + (location, "development"), + DefaultRole::None, + ) + .await + .expect("scheme listing") + .into_iter() + .map(|node| json!(node)) + .find(|node| node["node_key"] == "model:scheme_orders") + .expect("scheme node listed"); + ( + status["definition"]["status"].clone(), + listed["definition_changed"].clone(), + ) + }; + assert_eq!( + status().await, + (json!("matched"), json!(false)), + "the recorded SQL fingerprints like the current SQL under the snapshot's dialect" + ); + publish_snapshot(store, &publish(&without_dialect, 51)) + .await + .expect("snapshot without dialect"); + assert_eq!( + status().await, + (json!("changed"), json!(true)), + "without a dialect the recorded hash is compared as stored" + ); + publish_snapshot(store, &publish(&with_dialect, 52)) + .await + .expect("snapshot with dialect again"); + ingest_fingerprints( + store, + (location, "development"), + &build( + 52, + "scheme-edited", + &text("edited_definition"), + "2026-01-07T10:00:00Z", + ), + ) + .await + .expect("build of edited SQL"); + assert_eq!( + status().await, + (json!("changed"), json!(true)), + "a recorded build of different SQL still reads as changed" + ); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 82ce827200..3e5a9123ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -31,8 +31,9 @@ use crate::bindings::_helpers::check::check_core::{ use crate::bindings::_helpers::check::check_results::write_check_results; use crate::bindings::_helpers::command_runtime::warehouses::describe_connection_warehouse; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; +use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; use crate::bindings::_helpers::consumption::recorder::{ - ConsumptionRecorder, Prepared, prepare as prepare_consumption, run_reason_text, + ConsumptionRecorder, Prepared, held_back_text, prepare as prepare_consumption, run_reason_text, }; use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; @@ -49,6 +50,7 @@ use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; use crate::bindings::_helpers::sql_test_command::timeouts::{TimeoutKind, resolve_run_timeout}; +use sqlbuild_state::consumption::models::ChangedPolicy; const EXECUTION_COMMAND: &str = "sqb build"; const NODE_START: [&str; 2] = ["name", "resource_kind"]; @@ -375,13 +377,15 @@ pub(crate) fn execute_build_plan<'py>( ), ], )?; + let policy: Option = changed_policy( + &context.get("effective_project_dir")?, + &context.request, + &project, + )?; let prepared: Prepared<'py> = prepare_consumption( (&plan, &project), (&adapter, &connection_config), - ( - context.request_flag("changed")?, - context.request_flag("full_refresh")?, - ), + (policy.as_ref(), context.request_flag("full_refresh")?), ( native_field(&project, "run_id")?.str()?.extract()?, native_field(&preparation.lifecycle, "on_node_complete")?, @@ -516,14 +520,29 @@ fn report_changed_decision(context: &BuildContext<'_>, prepared: &Prepared<'_>) .map(|key| format!("{}:{}", key.kind, key.name)) .collect(); worker.unchanged(context.py(), &skipped)?; + let held_back: Vec<(String, String)> = decision + .held_back + .iter() + .map(|(key, why)| (format!("{}:{}", key.kind, key.name), held_back_text(why))) + .collect(); + worker.held_back(context.py(), &held_back)?; } + let total: usize = decision.run + decision.skipped.len() + decision.held_back.len(); + let left: String = if decision.held_back.is_empty() { + format!("{} unchanged are skipped", decision.skipped.len()) + } else { + format!( + "{} unchanged and {} held back are skipped", + decision.skipped.len(), + decision.held_back.len() + ) + }; emit( &context.stream, &format!( - "Changed: {} of {} selected models and seeds run; {} unchanged are skipped.\n", - decision.run, - decision.run + decision.skipped.len(), - decision.skipped.len() + "{}\nChanged: {} of {total} selected models and seeds run; {left}.\n", + policy_line(&decision.policy), + decision.run ), )?; if context.request_flag("verbose")? { @@ -538,6 +557,17 @@ fn report_changed_decision(context: &BuildContext<'_>, prepared: &Prepared<'_>) ), )?; } + for (key, why) in &decision.held_back { + emit( + &context.stream, + &format!( + " held back {} {}: {}\n", + key.kind, + key.name, + held_back_text(why) + ), + )?; + } } Ok(()) } @@ -558,8 +588,7 @@ fn with_skipped_keys<'py>( let entry: Bound<'py, PyAny> = entry?; let name: String = native_field(&entry, "name")?.extract()?; if decision - .skipped - .iter() + .not_run() .any(|key| key.kind == kind && key.name == name) { keys.push(native_field(&entry, "key")?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs index 8b4a8dedbf..30ef82fae9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -5,3 +5,7 @@ pub(crate) const NO_CHANGE_REASON: &str = "no_change"; pub(crate) const REPORT_ATTEMPTS: usize = 3; /// The HTTP status a control server answers while its store is busy; reports retry it. pub(crate) const STORE_BUSY_STATUS: u16 = 503; +/// Selector kinds that can name a job: `job:` and a bare name. +pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; +/// Separates the parts of an intersection selector. +pub(crate) const SELECTOR_INTERSECTION: char = ','; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs index 3cea848026..f5122b631c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -1,25 +1,35 @@ -//! Jobs for the command's target, as publication keeps them, loaded only when a selector names one. +//! The command target's jobs, loaded when a selector names one or `--changed` needs triggered jobs. use std::borrow::Cow; use std::cell::RefCell; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; -use sqlbuild_analysis::graph::models::{JobSelection, ProjectGraph}; +use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector, ProjectGraph}; use sqlbuild_cli::native_commands::main::effective_target::effective_target; +use crate::bindings::_helpers::compiled_project::constants::{ + JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, +}; use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; -use crate::bindings::_helpers::native_host::orchestration_declarations::job_selections; +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + job_selections, triggered_job_names, +}; + +/// Each job and group's selection, and the names of those a schedule or sensor starts. +type Catalog = (BTreeMap, BTreeSet); /// The running command's project directory, its `--target` and, once loaded, its jobs. struct JobScope { project_dir: PathBuf, selected_target: Option, - catalog: Option>, + catalog: Option, } thread_local! { @@ -55,7 +65,7 @@ pub(crate) fn graph_with_jobs<'g>( return Ok(Cow::Borrowed(graph)); } Ok(match job_catalog(py)? { - Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), + Some((catalog, _)) => Cow::Owned(attach_jobs(graph.clone(), catalog)), None => Cow::Borrowed(graph), }) } @@ -63,11 +73,63 @@ pub(crate) fn graph_with_jobs<'g>( /// The command's job and job group names, loaded on first use; empty outside a command. pub(crate) fn job_names(py: Python<'_>) -> PyResult> { Ok(job_catalog(py)? - .map(|catalog| catalog.into_keys().collect()) + .map(|(catalog, _)| catalog.into_keys().collect()) .unwrap_or_default()) } -fn job_catalog(py: Python<'_>) -> PyResult>> { +/// Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. +pub(crate) fn triggered_jobs( + py: Python<'_>, + graph: &ProjectGraph, + selectors: &[String], +) -> PyResult)>> { + let Some((catalog, triggered)) = job_catalog(py)? else { + return Ok(Vec::new()); + }; + if triggered.is_empty() { + return Ok(Vec::new()); + } + let running: BTreeSet = selected_jobs(selectors, &catalog); + let mut graph: ProjectGraph = attach_jobs(graph.clone(), catalog); + let mut resolved = graph.jobs.take().unwrap_or_default(); + let mut running_nodes: BTreeSet = BTreeSet::new(); + for name in &running { + if let Some(Ok(keys)) = resolved.get(name) { + running_nodes.extend(keys.iter().cloned()); + } + } + Ok(triggered + .into_iter() + .filter(|name| !running.contains(name)) + .filter_map(|name| match resolved.remove(&name) { + Some(Ok(keys)) => Some((name, &keys - &running_nodes)), + _ => None, + }) + .collect()) +} + +/// The jobs `selectors` name, as `job:` or as a bare job name. +fn selected_jobs( + selectors: &[String], + catalog: &BTreeMap, +) -> BTreeSet { + let mut jobs: BTreeSet = BTreeSet::new(); + let parts = selectors + .iter() + .flat_map(|selector| selector.split(char::is_whitespace)) + .flat_map(|token| token.split(SELECTOR_INTERSECTION)); + for part in parts { + let Ok(ParsedSelector::Kind { kind, value, .. }) = parse_selector(part) else { + continue; + }; + if JOB_SELECTOR_KINDS.contains(&kind.as_str()) && catalog.contains_key(&value) { + jobs.insert(value); + } + } + jobs +} + +fn job_catalog(py: Python<'_>) -> PyResult> { let Some(scope) = JOB_SCOPE.with(|scope| { scope.borrow().as_ref().map(|scope| match &scope.catalog { Some(catalog) => Ok(catalog.clone()), @@ -83,7 +145,10 @@ fn job_catalog(py: Python<'_>) -> PyResult let target: Option = effective_target(&project_dir, selected_target).map_err(PyValueError::new_err)?; let roots: Vec<(Bound<'_, PyAny>, String)> = discover_declarations(py, &project_dir)?; - let catalog: BTreeMap = job_selections(py, &roots, target.as_deref())?; + let catalog: Catalog = ( + job_selections(py, &roots, target.as_deref())?, + triggered_job_names(py, &roots, target.as_deref())?, + ); JOB_SCOPE.with(|scope| { if let Some(scope) = scope.borrow_mut().as_mut() { scope.catalog = Some(catalog.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 45463f934b..509d5b7530 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -158,6 +158,22 @@ impl WorkerBuild { Ok(()) } + /// Nodes `build --changed` holds back wait for the job that refreshes them, with why. + pub(crate) fn held_back(&self, py: Python<'_>, nodes: &[(String, String)]) -> PyResult<()> { + for (key, why) in nodes { + self.complete( + py, + key, + ( + NodeOutcome::Skipped, + None, + (None, Some(format!("held back: {why}"))), + ), + )?; + } + Ok(()) + } + fn complete( &self, py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs index 3902571ece..fd894e652d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs @@ -214,7 +214,7 @@ fn entry_forced(entry: &Bound<'_, PyAny>, full_refresh: bool) -> PyResult, &Bound<'_, PyAny>), (changed, full_refresh): (bool, bool), @@ -271,10 +271,6 @@ pub(crate) fn build_facts( } } let sources: HashMap> = source_versions(plan)?; - let existing: BTreeSet = native_field(plan, "existing_model_names")? - .try_iter()? - .map(|name| name?.extract()) - .collect::>()?; let dialect: Option = native_field(project, "sql_analysis_dialect")?.extract()?; let recorded: Bound<'_, PyAny> = native_field(plan, "recorded_fingerprints")?; let recorded_models: Bound<'_, PyAny> = if recorded.is_none() { @@ -342,13 +338,7 @@ pub(crate) fn build_facts( pending.push_back(upstream.clone()); } } - let mut forced: Vec = match selected.get(&key) { - Some(forced) => forced.clone(), - None if key.kind == MODEL && !existing.contains(&key.name) => { - vec![ForcedRun::RelationMissing] - } - None => Vec::new(), - }; + let mut forced: Vec = selected.get(&key).cloned().unwrap_or_default(); let destination: Bound<'_, PyAny> = node.destination()?; let relation: Option = (!native_field(&destination, "schema")?.is_none()) .then(|| relation(&destination)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs index 1e68918e23..35b4948cbe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod facts; +pub(crate) mod policy; pub(crate) mod recorder; pub(crate) mod relation_reads; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs new file mode 100644 index 0000000000..fc6d3bfe13 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs @@ -0,0 +1,129 @@ +//! What `--changed` counts and how far a change reaches: the command line, else `[changes]`. + +use std::collections::BTreeSet; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_state::consumption::models::{ + ChangeKind, ChangedPolicy, NodeKey, Reach, TriggeredJob, +}; + +use crate::bindings::_helpers::compiled_project::job_catalog::triggered_jobs; +use crate::bindings::_helpers::discovery::project_config::configured_changes; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::records::fields::field as native_field; + +const DEFINITION: &str = "definition"; +const SOURCE_DATA: &str = "source_data"; +const REACH_JOB: &str = "job"; +const REACH_ALL: &str = "all"; + +fn change_kind(name: &str) -> PyResult { + match name { + DEFINITION => Ok(ChangeKind::Definition), + SOURCE_DATA => Ok(ChangeKind::SourceData), + other => Err(PyValueError::new_err(format!( + "unknown change kind '{other}': expected {DEFINITION} or {SOURCE_DATA}" + ))), + } +} + +/// A change kind as `[changes] counts` and `--changed=` spell it. +pub(crate) const fn kind_value(kind: ChangeKind) -> &'static str { + match kind { + ChangeKind::Definition => DEFINITION, + ChangeKind::SourceData => SOURCE_DATA, + } +} + +/// The request's `--select` selectors. +fn selectors(request: &Bound<'_, PyAny>) -> PyResult> { + Ok(native_field(request, "select")? + .extract::>>()? + .unwrap_or_default()) +} + +/// A triggered job with its selected nodes as consumption keys. +fn triggered_job((name, keys): (String, BTreeSet)) -> TriggeredJob { + TriggeredJob { + name, + nodes: keys + .into_iter() + .map(|(kind, name)| NodeKey { kind, name }) + .collect(), + } +} + +/// A reach as `[changes] reach` and `--reach` spell it. +pub(crate) const fn reach_value(reach: &Reach) -> &'static str { + match reach { + Reach::Job(_) => REACH_JOB, + Reach::All => REACH_ALL, + } +} + +/// The policy of a `--changed` request, or None without it; triggered jobs resolve on `project`. +pub(crate) fn changed_policy( + project_dir: &Bound<'_, PyAny>, + request: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult> { + if !native_field(request, "changed")?.is_truthy()? { + return Ok(None); + } + let configured = configured_changes(project_dir)?; + let counts: Vec = match native_field(request, "changed_counts")?.extract()? { + Some(counts) => counts, + None => configured + .counts + .unwrap_or_else(|| vec![DEFINITION.to_owned()]), + }; + let reach: String = match native_field(request, "reach")?.extract()? { + Some(reach) => reach, + None => configured.reach.unwrap_or_else(|| REACH_JOB.to_owned()), + }; + let counts: BTreeSet = counts + .iter() + .map(|name| change_kind(name)) + .collect::>()?; + let reach: Reach = match reach.as_str() { + REACH_ALL => Reach::All, + REACH_JOB => { + let graph: PyRef<'_, NativeProjectGraph> = + native_field(project, "lineage_graph")?.extract()?; + Reach::Job( + triggered_jobs(project.py(), graph.graph(), &selectors(request)?)? + .into_iter() + .map(triggered_job) + .collect(), + ) + } + other => { + return Err(PyValueError::new_err(format!( + "unknown reach '{other}': expected {REACH_JOB} or {REACH_ALL}" + ))); + } + }; + Ok(Some(ChangedPolicy { counts, reach })) +} + +/// One line saying what the policy counts and how far it reaches, as build and plan print it. +pub(crate) fn policy_line(policy: &ChangedPolicy) -> String { + let label = |kind: &ChangeKind| match kind { + ChangeKind::Definition => "definition", + ChangeKind::SourceData => "source data", + }; + let counted: Vec<&str> = policy.counts.iter().map(label).collect(); + let uncounted: Vec = [ChangeKind::Definition, ChangeKind::SourceData] + .iter() + .filter(|kind| !policy.counts.contains(kind)) + .map(|kind| format!(" ({} not counted)", label(kind))) + .collect(); + format!( + "Counting: {}{} · reach: {}", + counted.join(", "), + uncounted.concat(), + reach_value(&policy.reach) + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs index 30eb68ad72..d110fe8344 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs @@ -11,11 +11,12 @@ use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; use sqlbuild_state::consumption::main::changed_run_set::changed_run_set; use sqlbuild_state::consumption::main::consumed_inputs::consumed_inputs; use sqlbuild_state::consumption::main::consumption_history::consumption_history; +use sqlbuild_state::consumption::main::read_back_history::read_back_history; use sqlbuild_state::consumption::main::started_append::started_append; use sqlbuild_state::consumption::main::succeeded_appends::succeeded_appends; use sqlbuild_state::consumption::models::{ - ConsumptionAppend, ConsumptionHistory, ConsumptionRow, ForcedRun, InputFact, NodeFacts, - NodeKey, RunReason, + ChangedOutcome, ChangedPolicy, ConsumptionAppend, ConsumptionHistory, ConsumptionRow, + ForcedRun, HeldBack, InputFact, NodeFacts, NodeKey, Reach, RelationKey, RunReason, }; use uuid::Uuid; @@ -74,11 +75,24 @@ pub(crate) struct ConsumptionRecorder { /// What `--changed` decided, for the build's report. pub(crate) struct ChangedDecision { + pub(crate) policy: ChangedPolicy, pub(crate) run: usize, + /// Unchanged nodes, which are up to date. pub(crate) skipped: Vec, + /// Nodes downstream of a change that wait for a later run. + pub(crate) held_back: Vec<(NodeKey, HeldBack)>, pub(crate) reasons: Vec<(NodeKey, RunReason)>, } +impl ChangedDecision { + /// Every selected node `--changed` leaves unbuilt: unchanged and held back. + pub(crate) fn not_run(&self) -> impl Iterator { + self.skipped + .iter() + .chain(self.held_back.iter().map(|(key, _)| key)) + } +} + fn kind_of(result: &Bound<'_, PyAny>) -> PyResult> { for (field, kind) in [(MODEL_RESULT, MODEL), (SEED_RESULT, SEED)] { if let Some(name) = result.getattr_opt(field)? { @@ -264,50 +278,21 @@ pub(crate) struct Prepared<'py> { pub(crate) fn prepare<'py>( (plan, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (changed, full_refresh): (bool, bool), + (policy, full_refresh): (Option<&ChangedPolicy>, bool), (run_id, base_on_node_complete): (String, Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = plan.py(); - let facts: BuildFacts = build_facts((plan, project), (changed, full_refresh))?; + let facts: BuildFacts = build_facts((plan, project), (policy.is_some(), full_refresh))?; let connection = ConsumptionConnection::new(adapter, connection_config); - let mut warnings: Vec = Vec::new(); - let mut rows: Vec = Vec::new(); - for location in read_locations(&facts, changed) { - match connection.read_recent(py, &location) { - Ok(read) => rows.extend(read), - Err(error) => warnings.push(format!( - "could not read consumed inputs in schema {}: {error}; its models and seeds count as changed", - location.schema - )), - } - } - let history: ConsumptionHistory = consumption_history(&rows, RECENT_EXECUTIONS); - let mut skipped: BTreeSet = BTreeSet::new(); - let decision: Option = if changed { - let mut reasons: Vec<(NodeKey, RunReason)> = Vec::new(); - for (key, reason) in changed_run_set(&facts.nodes, &history) { - if !facts.runnable.contains(&key) { - continue; - } - match reason { - Some(reason) => reasons.push((key, reason)), - None => { - skipped.insert(key); - } - } - } - Some(ChangedDecision { - run: reasons.len(), - skipped: skipped.iter().cloned().collect(), - reasons, - }) - } else { - None - }; + let (history, warnings): (ConsumptionHistory, Vec) = + read_history(py, &connection, (&facts, policy)); + let decision: Option = + policy.map(|policy| changed_decision(&facts, &history, policy)); + let not_run: BTreeSet<&NodeKey> = decision.iter().flat_map(ChangedDecision::not_run).collect(); let executions: HashMap = facts .runnable .iter() - .filter(|key| !skipped.contains(*key)) + .filter(|key| !not_run.contains(key)) .map(|key| (key.clone(), Uuid::now_v7().to_string())) .collect(); let nodes: HashMap = facts @@ -363,6 +348,135 @@ pub(crate) fn prepare<'py>( }) } +/// What `plan --changed` previews: what a build would run now, read without recording anything. +pub(crate) fn preview_changed( + (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (policy, full_refresh): (&ChangedPolicy, bool), +) -> PyResult<(ChangedDecision, Vec)> { + let py: Python<'_> = plan.py(); + let facts: BuildFacts = build_facts((plan, project), (true, full_refresh))?; + let connection = ConsumptionConnection::new(adapter, connection_config); + let (history, warnings): (ConsumptionHistory, Vec) = + read_history(py, &connection, (&facts, Some(policy))); + connection.close(py)?; + Ok((changed_decision(&facts, &history, policy), warnings)) +} + +/// Recent executions the build reads, with older ones read back when deciding needs them. +fn read_history( + py: Python<'_>, + connection: &ConsumptionConnection, + (facts, policy): (&BuildFacts, Option<&ChangedPolicy>), +) -> (ConsumptionHistory, Vec) { + let mut warnings: Vec = Vec::new(); + let mut rows: Vec = Vec::new(); + for location in read_locations(facts, policy.is_some()) { + match connection.read_recent(py, &location) { + Ok(read) => rows.extend(read), + Err(error) => warnings.push(format!( + "could not read consumed inputs in schema {}: {error}; its models and seeds count as changed", + location.schema + )), + } + } + let history: ConsumptionHistory = consumption_history(&rows, RECENT_EXECUTIONS); + if policy.is_some_and(needs_read_back) { + let (history, read_back): (ConsumptionHistory, Vec) = + with_read_back(py, connection, history, &facts.nodes); + warnings.extend(read_back); + return (history, warnings); + } + (history, warnings) +} + +/// What `--changed` does with each runnable node. +fn changed_decision( + facts: &BuildFacts, + history: &ConsumptionHistory, + policy: &ChangedPolicy, +) -> ChangedDecision { + let mut reasons: Vec<(NodeKey, RunReason)> = Vec::new(); + let mut skipped: Vec = Vec::new(); + let mut held_back: Vec<(NodeKey, HeldBack)> = Vec::new(); + for (key, outcome) in changed_run_set(&facts.nodes, history, policy) { + if !facts.runnable.contains(&key) { + continue; + } + match outcome { + ChangedOutcome::Runs(reason) => reasons.push((key, reason)), + ChangedOutcome::Unchanged => skipped.push(key), + ChangedOutcome::HeldBack(why) => held_back.push((key, why)), + } + } + ChangedDecision { + policy: policy.clone(), + run: reasons.len(), + skipped, + held_back, + reasons, + } +} + +/// Whether deciding needs older executions: to tell definition from data, or to attribute a change. +fn needs_read_back(policy: &ChangedPolicy) -> bool { + policy.counts.len() == 1 || matches!(policy.reach, Reach::Job(_)) +} + +/// `history` plus every older upstream execution its builds read, followed back until none is left. +fn with_read_back( + py: Python<'_>, + connection: &ConsumptionConnection, + history: ConsumptionHistory, + nodes: &[NodeFacts], +) -> (ConsumptionHistory, Vec) { + let mut warnings: BTreeSet = BTreeSet::new(); + let history: ConsumptionHistory = read_back_history(history, nodes, |references| { + let (rows, failures) = read_references(py, connection, references); + warnings.extend(failures); + rows + }); + (history, warnings.into_iter().collect()) +} + +/// The rows of these executions; a schema that cannot be read leaves its rebuilds counted. +fn read_references( + py: Python<'_>, + connection: &ConsumptionConnection, + references: &[(RelationKey, String)], +) -> (Vec, Vec) { + let mut by_location: BTreeMap> = BTreeMap::new(); + for (relation, id) in references { + by_location + .entry(Location::of(relation)) + .or_default() + .push(id.as_str()); + } + let mut rows: Vec = Vec::new(); + let mut warnings: Vec = Vec::new(); + for (location, ids) in by_location { + match connection.read_executions(py, &location, &ids) { + Ok(read) => rows.extend(read), + Err(error) => warnings.push(format!( + "could not read earlier executions in schema {}: {error}; rebuilt upstreams there count as changed", + location.schema + )), + } + } + (rows, warnings) +} + +/// Why `--changed` holds a node back, as one line for verbose output. +pub(crate) fn held_back_text(why: &HeldBack) -> String { + match why { + HeldBack::RefreshedBy(jobs) => format!( + "job {} refreshes it, and the change is not in that job", + jobs.join(", ") + ), + HeldBack::ReadsHeldBack(key) => format!("reads held-back {} {}", key.kind, key.name), + } +} + /// Why `--changed` runs a node, as one line for verbose output. pub(crate) fn run_reason_text(reason: &RunReason) -> String { match reason { @@ -379,6 +493,9 @@ pub(crate) fn run_reason_text(reason: &RunReason) -> String { RunReason::DefinitionChanged => "definition changed".to_owned(), RunReason::InputsChanged => "inputs changed".to_owned(), RunReason::InputChanged(input) => format!("input {input} changed"), + RunReason::UpstreamDefinitionChanged(input) => { + format!("upstream {input} was rebuilt with a definition change") + } RunReason::InputUnknown(input) => format!("input {input} has no known version"), RunReason::UpstreamRuns(key) => format!("upstream {} {} runs", key.kind, key.name), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs index 536de4c6f2..14bf776fb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs @@ -10,13 +10,16 @@ use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_config::project::errors::{ConfigFailure, FailureClass, LoadFailure}; use sqlbuild_config::project::main::deprecated_setting_warnings::deprecated_setting_warnings; +use sqlbuild_config::project::main::load_changes_config::load_changes_config; use sqlbuild_config::project::main::load_git_base_ref::load_git_base_ref; use sqlbuild_config::project::main::load_local_config::load_local_config; use sqlbuild_config::project::main::load_project_config::load_project_config; use sqlbuild_config::project::main::load_review_config::load_review_config; use sqlbuild_config::project::main::local_sql_analysis_failure::local_sql_analysis_failure; use sqlbuild_config::project::main::resolve_config_file::resolve_config_file; -use sqlbuild_config::project::models::{Built, ConfigFile, ConfigPaths, ReviewConfig}; +use sqlbuild_config::project::models::{ + Built, ChangesConfig, ConfigFile, ConfigPaths, ReviewConfig, +}; use crate::bindings::_helpers::boundary::config_values::config_value_to_python; use crate::bindings::_helpers::native_host::host::write_console; @@ -193,16 +196,26 @@ pub(crate) fn load_project_config_file( /// `[git] base_ref` of the project at `project_dir`, which `--since @base` names. pub(crate) fn configured_git_base_ref(project_dir: &Bound<'_, PyAny>) -> PyResult> { - let files = files(project_dir)?; - load_git_base_ref(&files.native) - .or_else(|failure| Err(load_error(project_dir.py(), &failure, &files)?)) + configured(project_dir, load_git_base_ref) } /// `[review]` of the project at `project_dir`: the defaults `sqb review` falls back to. pub(crate) fn configured_review(project_dir: &Bound<'_, PyAny>) -> PyResult { - let files = files(project_dir)?; - load_review_config(&files.native) - .or_else(|failure| Err(load_error(project_dir.py(), &failure, &files)?)) + configured(project_dir, load_review_config) +} + +/// `[changes]` of the project at `project_dir`: what `--changed` counts and reaches by default. +pub(crate) fn configured_changes(project_dir: &Bound<'_, PyAny>) -> PyResult { + configured(project_dir, load_changes_config) +} + +/// One configuration section of the project at `project_dir`, with load failures raised as Python's. +fn configured( + project_dir: &Bound<'_, PyAny>, + load: fn(&ConfigPaths) -> Result, +) -> PyResult { + let files: Files<'_> = files(project_dir)?; + load(&files.native).or_else(|failure| Err(load_error(project_dir.py(), &failure, &files)?)) } /// `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index ef6bb7fe9f..ad40b99ed6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -17,3 +17,11 @@ pub(crate) const COMMAND_IMPLEMENTATIONS: [(RunCommand, &str); 5] = [ /// DuckDB's message when another process, such as a running build, holds the database file. pub(crate) const DUCKDB_LOCK_CONFLICT: &str = "Conflicting lock is held"; + +/// The build settings a job run's `--changed` reach reads, and the reach every job run uses. +pub(crate) const CHANGED_SETTING: &str = "changed"; +pub(crate) const REACH_SETTING: &str = "reach"; +/// A job run reaches its whole selection: `reach` only stops builds that are not a job's run. +pub(crate) const JOB_RUN_REACH: &str = "all"; +/// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. +pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index e98184f7c9..40f4066522 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -155,9 +155,10 @@ fn identity_nodes( let options = PyDict::new(py); options.set_item("project", project)?; let scope = build_planner_scope(py, Some(options))?; + let dialect: Option = field(project, "sql_analysis_dialect")?.extract()?; let identities = direct_model_version_identities( (&field(project, "functions")?, &scope), - field(project, "sql_analysis_dialect")?.extract()?, + dialect.clone(), ( &field(project, "seeds")?, &py.None().into_bound(py), @@ -166,15 +167,19 @@ fn identity_nodes( &py.None().into_bound(py), )?; for node in nodes.values_mut() { - let hashes = match node.kind.as_str() { - "model" => "model_local_hashes", - "udf" | "table_fn" => "function_local_hashes", - "seed" => "seed_version_hashes", + let (hashes, with_dialect) = match node.kind.as_str() { + "model" => ("model_local_hashes", true), + "udf" | "table_fn" => ("function_local_hashes", false), + "seed" => ("seed_version_hashes", false), _ => continue, }; let name = node.definition["name"].as_str().unwrap_or_default(); let hash: String = field(&identities, hashes)?.get_item(name)?.extract()?; - node.definition["planner_identity"] = json!({"local_hash": hash}); + node.definition["planner_identity"] = if with_dialect { + json!({"local_hash": hash, "dialect": dialect}) + } else { + json!({"local_hash": hash}) + }; } Ok(nodes) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 934cb34d5d..4d28d969ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -29,8 +29,11 @@ use crate::bindings::_helpers::compile_attachment::effective_config::build_effec use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; -use crate::bindings::_helpers::location_command::constants::COMMAND_IMPLEMENTATIONS; +use crate::bindings::_helpers::location_command::constants::{ + CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, +}; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::session::InvocationRecall; @@ -70,6 +73,27 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap, + mut settings: BTreeMap, +) -> PyResult> { + let Some(changed) = settings.get(CHANGED_SETTING).cloned() else { + return Ok(settings); + }; + if changed == Value::Bool(false) { + return Ok(settings); + } + settings.insert(REACH_SETTING.to_owned(), Value::from(JOB_RUN_REACH)); + if changed == Value::Bool(true) && configured_changes(project_dir)?.counts.is_none() { + settings.insert( + CHANGED_SETTING.to_owned(), + Value::from(JOB_RUN_COUNTS.to_vec()), + ); + } + Ok(settings) +} + /// The frozen command's request and warehouse scope; build carries the verified inputs in its request. fn command_request<'py>( py: Python<'py>, @@ -99,6 +123,9 @@ fn command_request<'py>( settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); } let command: RunCommand = frozen.command; + if frozen.job_key.is_some() { + settings = job_run_settings(&arguments.value("project_dir")?, settings)?; + } let flags: Vec = if command == RunCommand::Build { job_settings_arguments(&settings) } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index 3677e129f8..be69f7d915 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -11,6 +11,7 @@ use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; use sqlbuild_state::consumption::constants::{ CONSUMPTION_COLUMN_TYPES, INTEGER_STATE_VALUE_TYPE, NULLABLE_COLUMNS, RECENT_EXECUTIONS, }; +use sqlbuild_state::consumption::main::executions_by_id_sql::executions_by_id_sql; use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql; use sqlbuild_state::consumption::models::{ @@ -26,6 +27,8 @@ use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; /// Rows per `INSERT`; SQL Server accepts at most 1,000 rows in one `VALUES` list. pub(crate) const ROWS_PER_STATEMENT: usize = 500; +/// Execution IDs per `IN` list when reading executions back by ID. +const EXECUTION_ID_BATCH: usize = 500; /// Warehouse timestamp texts with a UTC offset, as drivers render aware values. const OFFSET_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f%:z", "%Y-%m-%d %H:%M:%S%.f%:z"]; /// Warehouse timestamp texts without an offset, which the consumption table stores as UTC. @@ -250,6 +253,30 @@ fn read_recent( .collect() } +/// Rows of the named executions in `location`, read in batches; none when the table is missing. +fn read_executions( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, + execution_ids: &[&str], +) -> PyResult> { + let Some(table) = table_of(adapter, location)? else { + return Ok(Vec::new()); + }; + if execution_ids.is_empty() || !table_exists(adapter, connection, location)? { + return Ok(Vec::new()); + } + let mut rows: Vec = Vec::new(); + for batch in execution_ids.chunks(EXECUTION_ID_BATCH) { + let result: Bound<'_, PyAny> = + execute(adapter, connection, &executions_by_id_sql(&table, batch))?; + for row in call_with(&native_field(&result, "fetchall")?, &[])?.try_iter()? { + rows.push(read_row(&row?)?); + } + } + Ok(rows) +} + pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); @@ -428,6 +455,20 @@ impl ConsumptionConnection { read_recent(self.adapter.bind(py), &self.connected(py)?, location) } + pub(crate) fn read_executions( + &self, + py: Python<'_>, + location: &Location, + execution_ids: &[&str], + ) -> PyResult> { + read_executions( + self.adapter.bind(py), + &self.connected(py)?, + location, + execution_ids, + ) + } + pub(crate) fn append( &self, py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 1c4dda6f85..42adf451a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,6 +1,6 @@ //! The public authoring records enter native compilation here, never scheduler execution. -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::path::{Component, Path}; use pyo3::exceptions::{PyTypeError, PyValueError}; @@ -385,10 +385,7 @@ pub(crate) fn job_selections( if kind != JOB_LABEL && kind != GROUP_LABEL { continue; } - let targets: Vec = declaration.getattr("targets")?.extract()?; - if !targets.is_empty() - && !target.is_some_and(|target| targets.iter().any(|name| name == target)) - { + if !declared_for(&declaration, target)? { continue; } let tokens = |field: &str| -> PyResult> { @@ -422,3 +419,38 @@ pub(crate) fn job_selections( } Ok(catalog) } + +/// Whether a declaration applies to `target`: it names no targets, or names this one. +fn declared_for(declaration: &Bound<'_, PyAny>, target: Option<&str>) -> PyResult { + let targets: Vec = declaration.getattr("targets")?.extract()?; + Ok( + targets.is_empty() + || target.is_some_and(|target| targets.iter().any(|name| name == target)), + ) +} + +/// The jobs and groups that a schedule, calendar schedule or sensor declared for `target` starts. +pub(crate) fn triggered_job_names( + py: Python<'_>, + roots: &[(Bound<'_, PyAny>, String)], + target: Option<&str>, +) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for (declaration, kind, _) in reachable_declarations(py, roots)? { + if !declared_for(&declaration, target)? { + continue; + } + let started: Option> = match kind { + JOB_LABEL | GROUP_LABEL => { + (!declaration.getattr("schedule")?.is_none()).then(|| declaration.clone()) + } + CALENDAR_LABEL => Some(declaration.getattr("job")?), + SENSOR_LABEL => Some(declaration.getattr("run")?), + _ => None, + }; + if let Some(job) = started { + names.insert(job.getattr("name")?.extract()?); + } + } + Ok(names) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs new file mode 100644 index 0000000000..aafe27a81f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs @@ -0,0 +1,125 @@ +//! What `plan --changed` previews: the nodes a build would run, hold back and skip, and why. + +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; +use sqlbuild_core::cli_text::main::format_tree_connector::format_tree_connector; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_state::consumption::models::{HeldBack, NodeKey, RunReason}; + +use crate::bindings::_helpers::consumption::policy::{kind_value, reach_value}; +use crate::bindings::_helpers::consumption::recorder::{ChangedDecision, run_reason_text}; + +fn node(key: &NodeKey) -> String { + format!("{}:{}", key.kind, key.name) +} + +/// The upstream a node runs for, when it runs only because an upstream runs. +fn downstream_of(reason: &RunReason) -> Option<&NodeKey> { + match reason { + RunReason::UpstreamRuns(upstream) => Some(upstream), + _ => None, + } +} + +fn held_back_detail(why: &HeldBack) -> String { + match why { + HeldBack::RefreshedBy(jobs) => format!("job {}", jobs.join(", ")), + HeldBack::ReadsHeldBack(key) => format!("reads {}, which is held back", key.name), + } +} + +fn section(style: CliStyle, title: &str, rows: &[(String, String)]) -> Vec { + let width: usize = rows + .iter() + .map(|(name, _)| name.chars().count()) + .max() + .unwrap_or(0); + let mut lines: Vec = vec![style.plan_section(&format!("{title} ({})", rows.len()))]; + for (index, (name, detail)) in rows.iter().enumerate() { + lines.push(format!( + "{} {name: String { + let style = CliStyle { use_color }; + let mut changed: Vec<(String, String)> = Vec::new(); + let mut downstream: Vec<(String, String)> = Vec::new(); + for (key, reason) in &decision.reasons { + match downstream_of(reason) { + Some(upstream) => { + downstream.push((key.name.clone(), format!("reads {}", upstream.name))) + } + None => changed.push((key.name.clone(), run_reason_text(reason))), + } + } + let held_back: Vec<(String, String)> = decision + .held_back + .iter() + .map(|(key, why)| (key.name.clone(), held_back_detail(why))) + .collect(); + let mut blocks: Vec = Vec::new(); + for (title, rows) in [ + ("Changed", &changed), + ("Downstream of changes", &downstream), + ("Held back", &held_back), + ] { + if !rows.is_empty() { + blocks.push(section(style, title, rows).join("\n")); + } + } + if !decision.skipped.is_empty() { + blocks.push(style.muted(&format!( + "Unchanged ({}): skipped by --changed", + decision.skipped.len() + ))); + } + if blocks.is_empty() { + return style.muted("Nothing selected runs under --changed."); + } + blocks.join("\n\n") +} + +/// The plan JSON's `changed` object: the policy, and each selected node's outcome. +pub(crate) fn changed_json<'py>( + py: Python<'py>, + decision: &ChangedDecision, +) -> PyResult> { + let result: Bound<'py, PyDict> = PyDict::new(py); + let counts: Vec<&str> = decision + .policy + .counts + .iter() + .map(|kind| kind_value(*kind)) + .collect(); + result.set_item("counts", counts)?; + result.set_item("reach", reach_value(&decision.policy.reach))?; + let runs: Bound<'py, PyList> = PyList::empty(py); + for (key, reason) in &decision.reasons { + let entry: Bound<'py, PyDict> = PyDict::new(py); + entry.set_item("node", node(key))?; + entry.set_item("reason", run_reason_text(reason))?; + entry.set_item("downstream_of", downstream_of(reason).map(node))?; + runs.append(entry)?; + } + result.set_item("runs", runs)?; + let held_back: Bound<'py, PyList> = PyList::empty(py); + for (key, why) in &decision.held_back { + let entry: Bound<'py, PyDict> = PyDict::new(py); + entry.set_item("node", node(key))?; + match why { + HeldBack::RefreshedBy(jobs) => entry.set_item("refreshed_by", jobs)?, + HeldBack::ReadsHeldBack(upstream) => entry.set_item("reads", node(upstream))?, + } + held_back.append(entry)?; + } + result.set_item("held_back", held_back)?; + let unchanged: Vec = decision.skipped.iter().map(node).collect(); + result.set_item("unchanged", unchanged)?; + Ok(result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs index 5e6119a24d..b6cbd6def1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod changed; pub(crate) mod plan; pub(crate) mod registration; pub(crate) mod rules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 8baa0f2968..9ceefd6297 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -24,10 +24,13 @@ use crate::bindings::_helpers::command_runtime::runtime_errors::raised; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; +use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; +use crate::bindings::_helpers::consumption::recorder::{ChangedDecision, preview_changed}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; -use crate::bindings::_helpers::plan_output::plan_json::format_plan_json; +use crate::bindings::_helpers::plan_command::changed::{changed_json, changed_text}; +use crate::bindings::_helpers::plan_output::plan_json::{CHANGED_KEY, format_plan_json}; use crate::bindings::_helpers::plan_output::plan_text::format_plan; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::new_record; @@ -63,6 +66,13 @@ fn print_line(py: Python<'_>, text: &str) -> PyResult<()> { Ok(()) } +/// Write `text` to the progress stream and flush it, so it shows before slow work starts. +fn write_progress(stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) +} + /// The discovered project, adapter context, output stream and progress reporters of one plan. struct PlanInvocation<'py> { project_dir: Bound<'py, PyAny>, @@ -227,14 +237,13 @@ fn compile_pipeline<'py>( )?; let as_target: Bound<'py, PyAny> = native_field(request, "as_target")?; if !as_target.is_none() { - invocation.stream.call_method1( - "write", - (format!( + write_progress( + &invocation.stream, + &format!( "Previewing plan as target '{}' through the active target's connection (inspection only).\n", as_target.str()? - ),), + ), )?; - invocation.stream.call_method0("flush")?; } let options: Bound<'py, PyDict> = PyDict::new(py); options.set_item( @@ -281,10 +290,39 @@ fn compile_pipeline<'py>( ) } +/// What `--changed` would run now, or None without `--changed`; read warnings go to the stream. +fn changed_preview( + request: &Bound<'_, PyAny>, + invocation: &PlanInvocation<'_>, + pipeline: &Bound<'_, PyAny>, +) -> PyResult> { + let project: Bound<'_, PyAny> = native_field(pipeline, "project")?; + let Some(policy) = changed_policy(&invocation.project_dir, request, &project)? else { + return Ok(None); + }; + let adapter: Bound<'_, PyAny> = match &invocation.synced_copy { + Some(facade) => facade.clone().into_any(), + None => native_field(&invocation.context, "adapter")?, + }; + let (decision, warnings) = preview_changed( + (&native_field(pipeline, "plan_output")?, &project), + ( + &adapter, + &native_field(&invocation.context, "connection_config")?, + ), + (&policy, native_field(request, "full_refresh")?.is_truthy()?), + )?; + for warning in warnings { + write_progress(&invocation.stream, &format!("warning: {warning}\n"))?; + } + Ok(Some(decision)) +} + fn write_output( request: &Bound<'_, PyAny>, invocation: &PlanInvocation<'_>, pipeline: &Bound<'_, PyAny>, + changed: Option<&ChangedDecision>, ) -> PyResult<()> { let py: Python<'_> = request.py(); let plan: Bound<'_, PyAny> = native_field(pipeline, "plan_output")?; @@ -298,6 +336,9 @@ fn write_output( synced_copy_json(py, facade.get())?.into_any(), )); } + if let Some(decision) = changed { + options.push((CHANGED_KEY, changed_json(py, decision)?.into_any())); + } let text: String = format_plan_json(py, Some(keywords(py, &options)?))?; imported(py, "builtins", "print")?.call1((text,))?; return Ok(()); @@ -311,24 +352,27 @@ fn write_output( Some(DEFAULT_SECTION_ENTRIES) }, )?; - let text: String = format_plan( - py, - Some(keywords( - py, - &[ - ("plan", plan), - ("full_refresh", native_field(request, "full_refresh")?), - ("use_color", flag(py, invocation.use_color)), - ( - "display_options", - new_record(py, RecordId::DisplayOptions, (), Some(&display))?, - ), - ("python_plan_entries", python_entries), - ("include_direct_freshness_diagnostics", flag(py, false)), - ("include_completion", flag(py, true)), - ], - )?), - )?; + let mut options: Vec<(&str, Bound<'_, PyAny>)> = vec![ + ("plan", plan), + ("full_refresh", native_field(request, "full_refresh")?), + ("use_color", flag(py, invocation.use_color)), + ( + "display_options", + new_record(py, RecordId::DisplayOptions, (), Some(&display))?, + ), + ("python_plan_entries", python_entries), + ("include_direct_freshness_diagnostics", flag(py, false)), + ("include_completion", flag(py, true)), + ]; + if let Some(decision) = changed { + print_line(py, &policy_line(&decision.policy))?; + options.push(( + "changed_sections", + pyo3::types::PyString::new(py, &changed_text(decision, invocation.use_color)) + .into_any(), + )); + } + let text: String = format_plan(py, Some(keywords(py, &options)?))?; print_line(py, &format!("\n{text}"))?; if let Some(facade) = &invocation.synced_copy { print_line(py, &synced_copy_text(facade.get()))?; @@ -383,13 +427,13 @@ pub(crate) fn run_plan_command(request: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = request.py(); let invocation: PlanInvocation<'_> = resolve_invocation(request)?; if !native_field(request, "json_output")?.is_truthy()? { - invocation.stream.call_method1("write", ("\n",))?; - invocation.stream.call_method0("flush")?; + write_progress(&invocation.stream, "\n")?; } let verbose: bool = native_field(request, "verbose")?.is_truthy()?; let pipeline: Bound<'_, PyAny> = with_verbose_inspection(py, verbose, || compile_pipeline(request, &invocation))?; - write_output(request, &invocation, &pipeline)?; + let changed: Option = changed_preview(request, &invocation, &pipeline)?; + write_output(request, &invocation, &pipeline, changed.as_ref())?; Ok(i32::from(plan_has_errors(&native_field( &pipeline, "plan_output", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 551852218b..2664374219 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -21,6 +21,8 @@ use crate::bindings::_helpers::plan_output::values::{ }; use crate::bindings::_helpers::records::fields::field as native_field; +/// The plan JSON key of what `--changed` would run, hold back and skip. +pub(crate) const CHANGED_KEY: &str = "changed"; const MICROBATCH_MODE: &str = "microbatch"; const RENAMED_REASON: &str = "renamed"; const RENAMED_DECISION: &str = "renamed"; @@ -749,7 +751,10 @@ pub(crate) fn format_plan_json( py, "format_plan_json", options, - (&["plan"], &["python_plan_entries", SYNCED_COPY_KEY]), + ( + &["plan"], + &["python_plan_entries", SYNCED_COPY_KEY, CHANGED_KEY], + ), )?; let plan: Bound<'_, PyAny> = arguments.value("plan")?; let python_entries: Vec> = if arguments.passed("python_plan_entries")? { @@ -825,6 +830,9 @@ pub(crate) fn format_plan_json( if !synced_copy.is_none() { result.set_item(SYNCED_COPY_KEY, synced_copy)?; } + if arguments.passed(CHANGED_KEY)? { + result.set_item(CHANGED_KEY, arguments.value(CHANGED_KEY)?)?; + } indented_json(result.as_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs index 568bf822dc..0a5c86190e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -47,6 +47,7 @@ const PLAN_OPTIONS: (&[&str], &[&str]) = ( "python_plan_entries", "include_direct_freshness_diagnostics", "include_completion", + "changed_sections", ], ); @@ -524,7 +525,11 @@ pub(crate) fn format_plan(py: Python<'_>, options: Option>) -> let arguments: KeywordArguments<'_> = KeywordArguments::read(py, "format_plan", options, PLAN_OPTIONS)?; let report: PlanTextReport = report(&arguments)?; - let text: String = format_plan_text(&report); + let mut text: String = format_plan_text(&report); + if arguments.passed("changed_sections")? { + let sections: String = arguments.value("changed_sections")?.extract()?; + text = format!("{text}\n\n{sections}"); + } if !arguments.flag("include_completion")? { return Ok(text); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index c34a7e8402..a5441c3aa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -244,6 +244,8 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("fail_fast", FieldDefault::False), FieldSpec::new("full_refresh", FieldDefault::False), FieldSpec::new("changed", FieldDefault::False), + FieldSpec::new("changed_counts", FieldDefault::None), + FieldSpec::new("reach", FieldDefault::None), FieldSpec::new("load_sources", FieldDefault::None), FieldSpec::new("reload_sources", FieldDefault::False), FieldSpec::new("include_python", FieldDefault::True), @@ -561,6 +563,9 @@ const PLAN_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("cursor_overrides", FieldDefault::None), FieldSpec::new("json_output", FieldDefault::False), FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("changed", FieldDefault::False), + FieldSpec::new("changed_counts", FieldDefault::None), + FieldSpec::new("reach", FieldDefault::None), FieldSpec::new("load_sources", FieldDefault::None), FieldSpec::new("include_python", FieldDefault::True), FieldSpec::new("no_color", FieldDefault::False), diff --git a/crates/sqlbuild-state/src/consumption/_helpers/decision.rs b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs index 41d173f41e..af9ed420a8 100644 --- a/crates/sqlbuild-state/src/consumption/_helpers/decision.rs +++ b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs @@ -1,10 +1,16 @@ -//! Decide which selected nodes `--changed` runs, from their facts and their recent executions. +//! Decide what `--changed` does with each selected node, from its facts, its recent executions and the policy. +use std::cell::RefCell; use std::collections::{BTreeSet, HashMap}; use crate::consumption::_helpers::history::Window; -use crate::consumption::constants::{NODE_INPUT, OPAQUE_INPUT, SOURCE_INPUT}; -use crate::consumption::models::{ConsumptionHistory, InputFact, NodeFacts, NodeKey, RunReason}; +use crate::consumption::constants::{ + DEFINITION_PLANNER_CHANGES, NODE_INPUT, OPAQUE_INPUT, SOURCE_INPUT, +}; +use crate::consumption::models::{ + ChangeKind, ChangedOutcome, ChangedPolicy, ConsumptionHistory, ForcedRun, HeldBack, InputFact, + NodeFacts, NodeKey, Reach, RelationKey, RunReason, +}; /// The state table key of an input: its kind and name. pub(crate) fn input_key(input: &InputFact) -> (&'static str, String) { @@ -20,41 +26,380 @@ pub(crate) fn node_input_name(key: &NodeKey) -> String { format!("{}:{}", key.kind, key.name) } -/// Each selected node in input order with the reason it runs, or None when it is skipped. +/// One reason a node's own records give for running, its kind (None always counts) and its origin. +struct Change { + reason: RunReason, + kind: Option, + origin: Option, +} + +/// Where a node stands after its own records and its upstreams are considered. +enum Status<'a> { + Unaffected, + /// It runs; `groups` holds, for each change that reaches it, the triggered jobs holding that change. + Runs { + reason: RunReason, + groups: BTreeSet>, + }, + Held { + why: HeldBack, + groups: BTreeSet>, + }, +} + +/// What the decision reads besides each node's own facts. +struct Context<'a> { + by_key: HashMap<&'a NodeKey, &'a NodeFacts>, + by_input_name: HashMap, + triggered: HashMap<&'a NodeKey, Vec<&'a str>>, + history: &'a ConsumptionHistory, + counts: &'a BTreeSet, + by_relation: HashMap<&'a RelationKey, &'a NodeKey>, + moved: RefCell>, +} + +/// Every relation where a rebuild's definition changed (empty: none did; None: unknown). +type Moved = Option>; + +/// Each selected node in input order with what `--changed` does with it. pub(crate) fn decide( nodes: &[NodeFacts], history: &ConsumptionHistory, -) -> Vec<(NodeKey, Option)> { - let by_key: HashMap<&NodeKey, &NodeFacts> = - nodes.iter().map(|node| (&node.key, node)).collect(); - let mut stale: HashMap<&NodeKey, Option> = HashMap::new(); - for node in topological(nodes, &by_key) { - let reason: Option = match own_reason(node, &by_key, history) { - Some(reason) => Some(reason), - None => stale_upstream(node, &stale), - }; - stale.insert(&node.key, reason); + policy: &ChangedPolicy, +) -> Vec<(NodeKey, ChangedOutcome)> { + let mut triggered: HashMap<&NodeKey, Vec<&str>> = HashMap::new(); + if let Reach::Job(jobs) = &policy.reach { + for job in jobs { + for node in &job.nodes { + triggered.entry(node).or_default().push(job.name.as_str()); + } + } + for names in triggered.values_mut() { + names.sort_unstable(); + names.dedup(); + } + } + let context: Context<'_> = Context { + by_key: nodes.iter().map(|node| (&node.key, node)).collect(), + by_input_name: nodes + .iter() + .map(|node| (node_input_name(&node.key), node)) + .collect(), + triggered, + history, + counts: &policy.counts, + by_relation: nodes + .iter() + .map(|node| (&node.relation, &node.key)) + .collect(), + moved: RefCell::new(HashMap::new()), + }; + let mut state: HashMap<&NodeKey, Status<'_>> = HashMap::new(); + for node in topological(nodes, &context.by_key) { + let status: Status<'_> = node_status(node, &context, &state); + state.insert(&node.key, status); } nodes .iter() .filter(|node| node.selected) - .map(|node| (node.key.clone(), stale.remove(&node.key).flatten())) + .map(|node| { + let outcome: ChangedOutcome = match state.remove(&node.key) { + Some(Status::Runs { reason, .. }) => ChangedOutcome::Runs(reason), + Some(Status::Held { why, .. }) => ChangedOutcome::HeldBack(why), + Some(Status::Unaffected) | None => ChangedOutcome::Unchanged, + }; + (node.key.clone(), outcome) + }) .collect() } -/// The first upstream of `node` that runs or is stale. -fn stale_upstream( - node: &NodeFacts, - stale: &HashMap<&NodeKey, Option>, -) -> Option { +/// Runs when it or an upstream changed, unless held back by its jobs or a held-back upstream. +fn node_status<'a>( + node: &'a NodeFacts, + context: &Context<'a>, + state: &HashMap<&'a NodeKey, Status<'a>>, +) -> Status<'a> { + let own: Vec = own_changes(node, context); + let mut groups: BTreeSet> = BTreeSet::new(); + let mut running: Option<&NodeKey> = None; + let mut held: Option<&NodeKey> = None; for input in &node.inputs { - if let InputFact::Node(upstream) = input - && matches!(stale.get(upstream), Some(Some(_))) - { - return Some(RunReason::UpstreamRuns(upstream.clone())); + let InputFact::Node(upstream) = input else { + continue; + }; + match state.get(upstream) { + Some(Status::Runs { + groups: carried, .. + }) => { + running.get_or_insert(upstream); + groups.extend(carried.iter().cloned()); + } + Some(Status::Held { + groups: carried, .. + }) => { + held.get_or_insert(upstream); + groups.extend(carried.iter().cloned()); + } + Some(Status::Unaffected) | None => {} } } - None + for change in &own { + groups.insert(holding_jobs(node, change, context)); + } + let reason: Option = match (own.into_iter().next(), running) { + (Some(change), _) => Some(change.reason), + (None, Some(upstream)) => Some(RunReason::UpstreamRuns(upstream.clone())), + (None, None) if held.is_some() => None, + (None, None) => return Status::Unaffected, + }; + if let Some(jobs) = context.triggered.get(&node.key) + && groups.iter().all(|holding| outside(holding, jobs)) + { + return Status::Held { + why: HeldBack::RefreshedBy(jobs.iter().map(|job| (*job).to_owned()).collect()), + groups, + }; + } + match (held, reason) { + (Some(upstream), _) => Status::Held { + why: HeldBack::ReadsHeldBack(upstream.clone()), + groups, + }, + (None, Some(reason)) => Status::Runs { reason, groups }, + (None, None) => Status::Unaffected, + } +} + +/// The triggered jobs of the node a change came from, which hold that change. +fn holding_jobs<'a>(node: &'a NodeFacts, change: &Change, context: &Context<'a>) -> Vec<&'a str> { + let origin: &NodeKey = change + .origin + .as_ref() + .and_then(|key| context.by_key.get(key).map(|facts| &facts.key)) + .unwrap_or(&node.key); + context.triggered.get(origin).cloned().unwrap_or_default() +} + +/// Every reason the node's own facts or records give that always counts or has a counted kind. +fn own_changes(node: &NodeFacts, context: &Context<'_>) -> Vec { + let mut changes: Vec = node + .forced + .iter() + .map(|forced| Change { + reason: RunReason::Forced(forced.clone()), + kind: forced_kind(forced), + origin: None, + }) + .collect(); + let history: &ConsumptionHistory = context.history; + match history.window(&node.relation) { + None => changes.push(always(RunReason::NoRecord)), + Some(window) if window.ambiguous => changes.push(always(RunReason::AmbiguousOrder)), + Some(window) => changes.extend(execution_changes(node, context, &window)), + } + changes.retain(|change| counted(change.kind, context.counts)); + changes +} + +/// A planner change of the definition itself counts as Definition; every other forced run always counts. +fn forced_kind(forced: &ForcedRun) -> Option { + match forced { + ForcedRun::PlannerChange(kind) if DEFINITION_PLANNER_CHANGES.contains(&kind.as_str()) => { + Some(ChangeKind::Definition) + } + _ => None, + } +} + +/// True when none of the jobs holding a change refresh the node. +fn outside(holding: &[&str], jobs: &[&str]) -> bool { + holding.iter().all(|job| !jobs.contains(job)) +} + +fn always(reason: RunReason) -> Change { + Change { + reason, + kind: None, + origin: None, + } +} + +fn counted(kind: Option, counts: &BTreeSet) -> bool { + kind.is_none_or(|kind| counts.contains(&kind)) +} + +/// Every difference between what the node reads now and what each execution in the window read. +fn execution_changes(node: &NodeFacts, context: &Context<'_>, window: &Window<'_>) -> Vec { + let history: &ConsumptionHistory = context.history; + let expected: BTreeSet<(&str, String)> = node.inputs.iter().map(input_key).collect(); + let mut changes: Vec = Vec::new(); + for execution in window.executions { + if !execution.complete { + return vec![always(RunReason::Unfinished)]; + } + if execution.version_hash.as_deref() != Some(node.version_hash.as_str()) { + changes.push(definition(RunReason::DefinitionChanged)); + } + let recorded: BTreeSet<(&str, String)> = execution + .inputs + .iter() + .map(|input| (input.kind.as_str(), input.name.clone())) + .collect(); + if recorded != expected { + changes.push(definition(RunReason::InputsChanged)); + continue; + } + for input in &node.inputs { + let (kind, name) = input_key(input); + let read: Option<&str> = execution + .inputs + .iter() + .find(|recorded| recorded.kind == kind && recorded.name == name) + .map(|recorded| recorded.version.as_str()); + match input { + InputFact::Opaque(_) => return vec![always(RunReason::InputUnknown(name))], + InputFact::Source { version: None, .. } => changes.push(Change { + reason: RunReason::InputUnknown(name), + kind: Some(ChangeKind::SourceData), + origin: None, + }), + InputFact::Source { + version: Some(current), + .. + } => { + if read != Some(current.as_str()) { + changes.push(Change { + reason: RunReason::InputChanged(name), + kind: Some(ChangeKind::SourceData), + origin: None, + }); + } + } + InputFact::Node(upstream) => { + let Some(facts) = context.by_key.get(upstream).copied() else { + changes.push(always(RunReason::InputUnknown(name))); + continue; + }; + let Some(current) = history.latest_complete(&facts.relation) else { + changes.push(Change { + reason: RunReason::InputUnknown(name), + kind: None, + origin: Some(upstream.clone()), + }); + continue; + }; + let Some(read) = read.filter(|read| *read != current) else { + continue; + }; + match definition_moved(context, &facts.relation, (read, current)) { + Some(moved) if !moved.is_empty() => { + changes.extend(moved.iter().map(|changed| { + Change { + reason: RunReason::UpstreamDefinitionChanged(name.clone()), + kind: Some(ChangeKind::Definition), + origin: Some( + context + .by_relation + .get(changed) + .map_or_else(|| upstream.clone(), |key| (*key).clone()), + ), + } + })); + } + moved => changes.push(Change { + reason: RunReason::InputChanged(name), + kind: moved.map(|_| ChangeKind::SourceData), + origin: Some(upstream.clone()), + }), + } + } + } + } + } + changes +} + +fn definition(reason: RunReason) -> Change { + Change { + reason, + kind: Some(ChangeKind::Definition), + origin: None, + } +} + +/// Where execution `new` of `relation` or its reads changed definition since execution `old`. +fn definition_moved( + context: &Context<'_>, + relation: &RelationKey, + (old, new): (&str, &str), +) -> Moved { + if old == new { + return Some(BTreeSet::new()); + } + let memo_key: (RelationKey, String, String) = + (relation.clone(), old.to_owned(), new.to_owned()); + let known: Option = context.moved.borrow().get(&memo_key).cloned(); + if let Some(known) = known { + return known; + } + context.moved.borrow_mut().insert(memo_key.clone(), None); + let moved: Moved = compare_executions(context, relation, (old, new)); + context.moved.borrow_mut().insert(memo_key, moved.clone()); + moved +} + +fn compare_executions( + context: &Context<'_>, + relation: &RelationKey, + (old, new): (&str, &str), +) -> Moved { + let history: &ConsumptionHistory = context.history; + let before = history.execution(relation, old)?; + let after = history.execution(relation, new)?; + if !before.complete || !after.complete { + return None; + } + let mut changed: BTreeSet = BTreeSet::new(); + if before.version_hash != after.version_hash { + changed.insert(relation.clone()); + } + if after.inputs.iter().any(|input| input.kind == OPAQUE_INPUT) { + return (!changed.is_empty()).then_some(changed); + } + let mut unknown: bool = false; + let before_nodes: usize = before + .inputs + .iter() + .filter(|input| input.kind == NODE_INPUT) + .count(); + let mut after_nodes: usize = 0; + for input in after.inputs.iter().filter(|input| input.kind == NODE_INPUT) { + after_nodes += 1; + let Some(read_before) = before + .inputs + .iter() + .find(|earlier| earlier.kind == NODE_INPUT && earlier.name == input.name) + else { + changed.insert(relation.clone()); + continue; + }; + let Some(upstream) = context.by_input_name.get(&input.name).copied() else { + unknown = true; + continue; + }; + match definition_moved( + context, + &upstream.relation, + (&read_before.version, &input.version), + ) { + Some(moved) => changed.extend(moved), + None => unknown = true, + } + } + if after_nodes != before_nodes { + changed.insert(relation.clone()); + } + (!changed.is_empty() || !unknown).then_some(changed) } /// Nodes with every upstream that has facts before it. @@ -94,67 +439,3 @@ fn topological<'a>( } ordered } - -/// Why the node's own facts or records make it run, ignoring its upstreams' staleness. -fn own_reason( - node: &NodeFacts, - by_key: &HashMap<&NodeKey, &NodeFacts>, - history: &ConsumptionHistory, -) -> Option { - if let Some(forced) = node.forced.first() { - return Some(RunReason::Forced(forced.clone())); - } - let Some(window) = history.window(&node.relation) else { - return Some(RunReason::NoRecord); - }; - if window.ambiguous { - return Some(RunReason::AmbiguousOrder); - } - execution_reason(node, (by_key, history), &window) -} - -fn execution_reason( - node: &NodeFacts, - (by_key, history): (&HashMap<&NodeKey, &NodeFacts>, &ConsumptionHistory), - window: &Window<'_>, -) -> Option { - let expected: BTreeSet<(&str, String)> = node.inputs.iter().map(input_key).collect(); - for execution in window.executions { - if !execution.complete { - return Some(RunReason::Unfinished); - } - if execution.version_hash.as_deref() != Some(node.version_hash.as_str()) { - return Some(RunReason::DefinitionChanged); - } - let recorded: BTreeSet<(&str, String)> = execution - .inputs - .iter() - .map(|input| (input.kind.as_str(), input.name.clone())) - .collect(); - if recorded != expected { - return Some(RunReason::InputsChanged); - } - for input in &node.inputs { - let (kind, name) = input_key(input); - let current: Option<&str> = match input { - InputFact::Source { version, .. } => version.as_deref(), - InputFact::Node(upstream) => by_key - .get(upstream) - .and_then(|facts| history.latest_complete(&facts.relation)), - InputFact::Opaque(_) => None, - }; - let Some(current) = current else { - return Some(RunReason::InputUnknown(name)); - }; - let read: Option<&str> = execution - .inputs - .iter() - .find(|recorded| recorded.kind == kind && recorded.name == name) - .map(|recorded| recorded.version.as_str()); - if read != Some(current) { - return Some(RunReason::InputChanged(name)); - } - } - } - None -} diff --git a/crates/sqlbuild-state/src/consumption/_helpers/history.rs b/crates/sqlbuild-state/src/consumption/_helpers/history.rs index a0534d26bd..e749ba91c3 100644 --- a/crates/sqlbuild-state/src/consumption/_helpers/history.rs +++ b/crates/sqlbuild-state/src/consumption/_helpers/history.rs @@ -20,6 +20,28 @@ type RowIdentity<'a> = (&'a str, &'a str, Option<(&'a str, &'a str, &'a str)>); /// Recent executions per relation, from rows read newest first with at most `limit` per relation. pub(crate) fn build_history(rows: &[ConsumptionRow], limit: usize) -> ConsumptionHistory { + let mut relations: HashMap = HashMap::new(); + for (relation, execution) in executions_of(rows) { + relations + .entry(relation) + .or_default() + .executions + .push(execution); + } + for history in relations.values_mut() { + history.executions.sort_by(|left, right| { + (right.started_at, &right.execution_id).cmp(&(left.started_at, &left.execution_id)) + }); + history.truncated = history.executions.len() >= limit; + } + ConsumptionHistory { + relations, + referenced: HashMap::new(), + } +} + +/// Each execution in `rows` with its relation, its duplicated rows counted once. +pub(crate) fn executions_of(rows: &[ConsumptionRow]) -> Vec<(RelationKey, Execution)> { let mut events: HashMap, (&ConsumptionRow, i64, i64)> = HashMap::new(); for row in rows { let input = row.row.input.as_ref(); @@ -54,21 +76,12 @@ pub(crate) fn build_history(rows: &[ConsumptionRow], limit: usize) -> Consumptio execution.latest = execution.latest.max(latest); execution.rows.push(row); } - let mut relations: HashMap = HashMap::new(); - for ((relation, execution_id), execution_rows) in grouped { - relations - .entry(relation.clone()) - .or_default() - .executions - .push(execution(execution_id, &execution_rows)); - } - for history in relations.values_mut() { - history.executions.sort_by(|left, right| { - (right.started_at, &right.execution_id).cmp(&(left.started_at, &left.execution_id)) - }); - history.truncated = history.executions.len() >= limit; - } - ConsumptionHistory { relations } + grouped + .into_iter() + .map(|((relation, execution_id), execution_rows)| { + (relation.clone(), execution(execution_id, &execution_rows)) + }) + .collect() } fn execution(execution_id: &str, rows: &ExecutionRows<'_>) -> Execution { @@ -142,6 +155,34 @@ pub(crate) struct Window<'a> { } impl ConsumptionHistory { + /// The execution `execution_id` of `relation`, recent or read back because a build read it. + pub(crate) fn execution( + &self, + relation: &RelationKey, + execution_id: &str, + ) -> Option<&Execution> { + self.relations + .get(relation) + .and_then(|history| { + history + .executions + .iter() + .find(|execution| execution.execution_id == execution_id) + }) + .or_else(|| { + self.referenced + .get(&(relation.clone(), execution_id.to_owned())) + }) + } + + /// Every execution read, recent or read back. + pub(crate) fn all_executions(&self) -> impl Iterator { + self.relations + .values() + .flat_map(|history| history.executions.iter()) + .chain(self.referenced.values()) + } + /// The newest executions of `relation` within the skew, or None when it has none. pub(crate) fn window(&self, relation: &RelationKey) -> Option> { let history: &RelationHistory = self.relations.get(relation)?; diff --git a/crates/sqlbuild-state/src/consumption/_helpers/mod.rs b/crates/sqlbuild-state/src/consumption/_helpers/mod.rs index f6c27c7cb2..a7264e1091 100644 --- a/crates/sqlbuild-state/src/consumption/_helpers/mod.rs +++ b/crates/sqlbuild-state/src/consumption/_helpers/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod decision; pub(crate) mod history; pub(crate) mod ranked_executions; +pub(crate) mod unread_references; +pub(crate) mod with_referenced; diff --git a/crates/sqlbuild-state/src/consumption/_helpers/unread_references.rs b/crates/sqlbuild-state/src/consumption/_helpers/unread_references.rs new file mode 100644 index 0000000000..952a5c7a7a --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/unread_references.rs @@ -0,0 +1,37 @@ +//! Older executions that recent builds read, so a rebuilt upstream's definitions can be compared. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::consumption::_helpers::decision::node_input_name; +use crate::consumption::constants::NODE_INPUT; +use crate::consumption::models::{ConsumptionHistory, NodeFacts, RelationKey}; + +/// Upstream executions that builds in `history` read but `history` lacks, by the upstream's relation. +pub(crate) fn unread_references( + history: &ConsumptionHistory, + nodes: &[NodeFacts], +) -> BTreeMap> { + let relation_of: HashMap = nodes + .iter() + .map(|node| (node_input_name(&node.key), &node.relation)) + .collect(); + let mut unread: BTreeMap> = BTreeMap::new(); + for execution in history.all_executions() { + for input in execution + .inputs + .iter() + .filter(|input| input.kind == NODE_INPUT) + { + let Some(relation) = relation_of.get(&input.name) else { + continue; + }; + if history.execution(relation, &input.version).is_none() { + unread + .entry((*relation).clone()) + .or_default() + .insert(input.version.clone()); + } + } + } + unread +} diff --git a/crates/sqlbuild-state/src/consumption/_helpers/with_referenced.rs b/crates/sqlbuild-state/src/consumption/_helpers/with_referenced.rs new file mode 100644 index 0000000000..6afcbe4fda --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/with_referenced.rs @@ -0,0 +1,22 @@ +//! Add executions read back by ID to the history `--changed` decides from. + +use crate::consumption::_helpers::history::executions_of; +use crate::consumption::models::{ConsumptionHistory, ConsumptionRow}; + +/// `history` with the executions in `rows` added as read-back executions. +pub(crate) fn with_referenced( + mut history: ConsumptionHistory, + rows: &[ConsumptionRow], +) -> ConsumptionHistory { + for (relation, execution) in executions_of(rows) { + if history + .execution(&relation, &execution.execution_id) + .is_none() + { + history + .referenced + .insert((relation, execution.execution_id.clone()), execution); + } + } + history +} diff --git a/crates/sqlbuild-state/src/consumption/constants.rs b/crates/sqlbuild-state/src/consumption/constants.rs index 2237518864..427d51b454 100644 --- a/crates/sqlbuild-state/src/consumption/constants.rs +++ b/crates/sqlbuild-state/src/consumption/constants.rs @@ -40,6 +40,9 @@ pub const SOURCE_INPUT: &str = "source"; pub const NODE_INPUT: &str = "node"; /// Input kind of anything whose version cannot be known. pub const OPAQUE_INPUT: &str = "opaque"; +/// Planner change kinds that are edits of the node's own definition. +pub const DEFINITION_PLANNER_CHANGES: [&str; 3] = + ["query_changed", "function_changed", "config_changed"]; /// The version recorded for an input whose current version is unknown. pub const UNKNOWN_VERSION: &str = "unknown"; /// Executions read per relation, newest first. diff --git a/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs b/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs index ed3bab7959..14d2bc817f 100644 --- a/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs +++ b/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs @@ -1,12 +1,15 @@ -//! Which selected nodes `--changed` runs. +//! What `--changed` does with each selected node. use crate::consumption::_helpers::decision::decide; -use crate::consumption::models::{ConsumptionHistory, NodeFacts, NodeKey, RunReason}; +use crate::consumption::models::{ + ChangedOutcome, ChangedPolicy, ConsumptionHistory, NodeFacts, NodeKey, +}; -/// Each selected node in order with why it runs, or None when nothing it or an upstream read changed. +/// Each selected node in order: why it runs, that it is unchanged, or why it is held back. pub fn changed_run_set( nodes: &[NodeFacts], history: &ConsumptionHistory, -) -> Vec<(NodeKey, Option)> { - decide(nodes, history) + policy: &ChangedPolicy, +) -> Vec<(NodeKey, ChangedOutcome)> { + decide(nodes, history, policy) } diff --git a/crates/sqlbuild-state/src/consumption/main/executions_by_id_sql.rs b/crates/sqlbuild-state/src/consumption/main/executions_by_id_sql.rs new file mode 100644 index 0000000000..66461a7749 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/executions_by_id_sql.rs @@ -0,0 +1,20 @@ +//! Read named executions from one consumption table. + +use crate::consumption::constants::CONSUMPTION_COLUMN_TYPES; + +/// Every row of the executions in `execution_ids`, which must be non-empty. +pub fn executions_by_id_sql(qualified_name: &str, execution_ids: &[&str]) -> String { + let columns: Vec<&str> = CONSUMPTION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + let ids: Vec = execution_ids + .iter() + .map(|id| format!("'{}'", id.replace('\'', "''"))) + .collect(); + format!( + "SELECT {columns} FROM {qualified_name} WHERE execution_id IN ({ids})", + columns = columns.join(", "), + ids = ids.join(", "), + ) +} diff --git a/crates/sqlbuild-state/src/consumption/main/mod.rs b/crates/sqlbuild-state/src/consumption/main/mod.rs index 8f097fabfd..31accb8109 100644 --- a/crates/sqlbuild-state/src/consumption/main/mod.rs +++ b/crates/sqlbuild-state/src/consumption/main/mod.rs @@ -1,7 +1,9 @@ pub mod changed_run_set; pub mod consumed_inputs; pub mod consumption_history; +pub mod executions_by_id_sql; pub mod prune_executions_sql; +pub mod read_back_history; pub mod recent_executions_sql; pub mod started_append; pub mod succeeded_appends; diff --git a/crates/sqlbuild-state/src/consumption/main/read_back_history.rs b/crates/sqlbuild-state/src/consumption/main/read_back_history.rs new file mode 100644 index 0000000000..566d25716f --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/read_back_history.rs @@ -0,0 +1,34 @@ +//! Follow the executions recent builds read back until every reachable one is in the history. + +use std::collections::BTreeSet; + +use crate::consumption::_helpers::unread_references::unread_references; +use crate::consumption::_helpers::with_referenced::with_referenced; +use crate::consumption::models::{ConsumptionHistory, ConsumptionRow, NodeFacts, RelationKey}; + +/// `history` plus every older execution its builds read, asking `read` for each reference once. +pub fn read_back_history( + history: ConsumptionHistory, + nodes: &[NodeFacts], + mut read: impl FnMut(&[(RelationKey, String)]) -> Vec, +) -> ConsumptionHistory { + let mut history: ConsumptionHistory = history; + let mut attempted: BTreeSet<(RelationKey, String)> = BTreeSet::new(); + loop { + let mut unread: Vec<(RelationKey, String)> = Vec::new(); + for (relation, ids) in unread_references(&history, nodes) { + for id in ids { + let reference: (RelationKey, String) = (relation.clone(), id); + if !attempted.contains(&reference) { + unread.push(reference); + } + } + } + if unread.is_empty() { + return history; + } + let rows: Vec = read(&unread); + attempted.extend(unread); + history = with_referenced(history, &rows); + } +} diff --git a/crates/sqlbuild-state/src/consumption/models.rs b/crates/sqlbuild-state/src/consumption/models.rs index b4624353d7..a5b8ce8560 100644 --- a/crates/sqlbuild-state/src/consumption/models.rs +++ b/crates/sqlbuild-state/src/consumption/models.rs @@ -90,10 +90,60 @@ pub enum RunReason { DefinitionChanged, InputsChanged, InputChanged(String), + /// An upstream was rebuilt with a definition change this node has not read yet. + UpstreamDefinitionChanged(String), InputUnknown(String), UpstreamRuns(NodeKey), } +/// What a change is, for `[changes] counts`; reasons with no kind always count. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum ChangeKind { + /// SQL, config, hooks, contract or a called function changed, here or in a rebuilt upstream. + Definition, + /// New data: a source's freshness value moved, or an upstream was rebuilt with the same definitions. + SourceData, +} + +/// A job or group that a schedule or sensor starts, with every node a run of it builds. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TriggeredJob { + pub name: String, + pub nodes: std::collections::BTreeSet, +} + +/// How far a change reaches downstream. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Reach { + /// Stop at nodes that a triggered job without the change refreshes. + Job(Vec), + All, +} + +/// What `--changed` counts and how far it reaches. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChangedPolicy { + pub counts: std::collections::BTreeSet, + pub reach: Reach, +} + +/// Why a node downstream of a change waits for a later run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum HeldBack { + /// These triggered jobs refresh it, and none of them holds the change. + RefreshedBy(Vec), + /// It reads this node, which is held back. + ReadsHeldBack(NodeKey), +} + +/// What `--changed` does with one selected node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ChangedOutcome { + Runs(RunReason), + Unchanged, + HeldBack(HeldBack), +} + /// One execution of a relation rebuilt from its deduplicated rows. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Execution { @@ -111,6 +161,8 @@ pub struct Execution { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ConsumptionHistory { pub(crate) relations: std::collections::HashMap, + /// Older executions read back because a newer build read them, by relation and execution ID. + pub(crate) referenced: std::collections::HashMap<(RelationKey, String), Execution>, } /// One relation's executions, newest first, and whether older ones were left unread. diff --git a/crates/sqlbuild-state/src/consumption/tests.rs b/crates/sqlbuild-state/src/consumption/tests.rs index 5ba8b6e3a0..6027e17bc4 100644 --- a/crates/sqlbuild-state/src/consumption/tests.rs +++ b/crates/sqlbuild-state/src/consumption/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_decide.rs"] +mod test_decide; #[path = "tests/test_simulation.rs"] mod test_simulation; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-state/src/consumption/tests/helpers.rs b/crates/sqlbuild-state/src/consumption/tests/helpers.rs index fc5737ee65..1ca2aac9c0 100644 --- a/crates/sqlbuild-state/src/consumption/tests/helpers.rs +++ b/crates/sqlbuild-state/src/consumption/tests/helpers.rs @@ -5,15 +5,20 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use crate::consumption::_helpers::decision::decide; use crate::consumption::_helpers::history::build_history; use crate::consumption::constants::RECENT_EXECUTIONS; +use crate::consumption::main::changed_run_set::changed_run_set; use crate::consumption::main::consumed_inputs::consumed_inputs; +use crate::consumption::main::consumption_history::consumption_history; +use crate::consumption::main::read_back_history::read_back_history; use crate::consumption::main::started_append::started_append; use crate::consumption::main::succeeded_appends::succeeded_appends; +use crate::consumption::models::TriggeredJob; use crate::consumption::models::{ - ConsumptionAppend, ConsumptionHistory, ConsumptionRow, ForcedRun, InputFact, NodeFacts, - NodeKey, RelationKey, RunReason, + ChangeKind, ChangedOutcome, ChangedPolicy, ConsumptionAppend, ConsumptionHistory, + ConsumptionRow, ForcedRun, HeldBack, InputFact, NodeFacts, NodeKey, Reach, RelationKey, + RunReason, }; use crate::consumption::tests::test_types::{ - Content, Derivation, Input, Miss, Node, Random, Source, World, Writer, + Content, Derivation, Input, Miss, Node, Random, Scenario, Source, World, Writer, }; impl Random { @@ -39,6 +44,8 @@ impl Random { } const SECOND: i64 = 1_000_000; +/// Between scenario builds, well past the order skew. +const HOUR: i64 = 3_600 * SECOND; const SIMULATED_CASES: u64 = 400; const SIMULATED_BUILDS: usize = 14; @@ -306,14 +313,18 @@ impl World { .collect(); let facts: Vec = self.facts(&selected, &observed, full_refresh); let history: ConsumptionHistory = build_history(&self.read_rows(), RECENT_EXECUTIONS); - let decided: Vec<(NodeKey, Option)> = decide(&facts, &history); + let policy = ChangedPolicy { + counts: BTreeSet::from([ChangeKind::Definition, ChangeKind::SourceData]), + reach: Reach::All, + }; + let decided: Vec<(NodeKey, ChangedOutcome)> = decide(&facts, &history, &policy); let run: Vec = (0..self.nodes.len()) .filter(|index| selected.contains(index)) .filter(|index| { !changed - || decided - .iter() - .any(|(key, reason)| *key == Self::key(*index) && reason.is_some()) + || decided.iter().any(|(key, outcome)| { + *key == Self::key(*index) && matches!(outcome, ChangedOutcome::Runs(_)) + }) }) .collect(); self.trace.push(format!( @@ -539,3 +550,241 @@ pub(crate) fn found(writer: Writer) -> Option<(u64, String, Vec)> { ) }) } + +impl Scenario { + /// Models named with their direct inputs; an input starting with `src_` is a source. + pub(crate) fn new(models: &[(&str, &[&str])]) -> Self { + let mut sources: BTreeMap = BTreeMap::new(); + let nodes: Vec = models + .iter() + .map(|(name, inputs)| NodeFacts { + key: scenario_key(name), + relation: RelationKey { + database: None, + schema: "analytics".to_owned(), + name: (*name).to_owned(), + }, + version_hash: "v1".to_owned(), + selected: true, + forced: Vec::new(), + inputs: inputs + .iter() + .map(|input| { + if input.starts_with("src_") { + sources.insert((*input).to_owned(), "d1".to_owned()); + InputFact::Source { + name: (*input).to_owned(), + version: None, + } + } else { + InputFact::Node(scenario_key(input)) + } + }) + .collect(), + }) + .collect(); + let mut scenario = Self { + nodes, + sources, + rows: Vec::new(), + clock: 1_000 * SECOND, + }; + scenario.refresh_sources(); + scenario + } + + fn refresh_sources(&mut self) { + for node in &mut self.nodes { + for input in &mut node.inputs { + if let InputFact::Source { name, version } = input { + *version = self.sources.get(name).cloned(); + } + } + } + } + + /// New data arrives in a source. + pub(crate) fn new_data(&mut self, source: &str, version: &str) { + self.sources.insert(source.to_owned(), version.to_owned()); + self.refresh_sources(); + } + + /// A deploy changes a model's definition. + pub(crate) fn redefine(&mut self, name: &str, version_hash: &str) { + for node in &mut self.nodes { + if node.key.name == name { + version_hash.clone_into(&mut node.version_hash); + } + } + } + + /// Build the named models in order, recording what each read. + pub(crate) fn build(&mut self, names: &[&str]) { + let by_key: HashMap = self + .nodes + .iter() + .map(|node| (node.key.clone(), node.clone())) + .collect(); + let mut this_build: HashMap = HashMap::new(); + self.clock += HOUR; + for name in names { + let history: ConsumptionHistory = consumption_history(&self.rows, RECENT_EXECUTIONS); + let node: &NodeFacts = &by_key[&scenario_key(name)]; + self.clock += SECOND; + let execution_id: String = format!("{name}@{}", self.clock); + let inputs = consumed_inputs(node, (&by_key, &history), &this_build); + let mut appends: Vec = + vec![started_append(node, &execution_id, "run")]; + appends.extend(succeeded_appends(node, (&execution_id, "run"), &inputs)); + for row in appends { + self.clock += 1; + self.rows.push(ConsumptionRow { + row, + recorded_at: self.clock, + }); + } + this_build.insert(node.key.clone(), execution_id); + } + } + + /// Forget the executions of a model older than its latest, as retention pruning does. + pub(crate) fn prune(&mut self, name: &str) { + let latest: Option = self + .rows + .iter() + .rev() + .find(|row| row.row.node.name == name) + .map(|row| row.row.execution_id.clone()); + self.rows.retain(|row| { + row.row.node.name != name || Some(&row.row.execution_id) == latest.as_ref() + }); + } + + /// The rows of each relation's newest executions, as the recent-executions read returns them. + fn recent_rows(&self) -> Vec { + let mut recent: HashMap<&RelationKey, Vec<&str>> = HashMap::new(); + for row in self.rows.iter().rev() { + let ids: &mut Vec<&str> = recent.entry(&row.row.relation).or_default(); + if !ids.contains(&row.row.execution_id.as_str()) && ids.len() < RECENT_EXECUTIONS { + ids.push(&row.row.execution_id); + } + } + self.rows + .iter() + .filter(|row| { + recent + .get(&row.row.relation) + .is_some_and(|ids| ids.contains(&row.row.execution_id.as_str())) + }) + .cloned() + .collect() + } + + /// The recorded rows of these executions. + fn rows_of(&self, references: &[(RelationKey, String)]) -> Vec { + self.rows + .iter() + .filter(|row| { + references.contains(&(row.row.relation.clone(), row.row.execution_id.clone())) + }) + .cloned() + .collect() + } + + /// Run a model regardless of its records, as the planner does for `forced`. + pub(crate) fn force(&mut self, name: &str, forced: ForcedRun) { + for node in &mut self.nodes { + if node.key.name == name { + node.forced.push(forced.clone()); + } + } + } + + /// What `--changed` does, reading recent rows and then the older executions they reference. + pub(crate) fn decide( + &self, + counts: &[ChangeKind], + jobs: Option<&[(&str, &[&str])]>, + ) -> Vec<(String, ChangedOutcome)> { + let history: ConsumptionHistory = read_back_history( + consumption_history(&self.recent_rows(), RECENT_EXECUTIONS), + &self.nodes, + |references| self.rows_of(references), + ); + let reach: Reach = match jobs { + None => Reach::All, + Some(jobs) => Reach::Job( + jobs.iter() + .map(|(name, nodes)| TriggeredJob { + name: (*name).to_owned(), + nodes: nodes.iter().map(|node| scenario_key(node)).collect(), + }) + .collect(), + ), + }; + let policy = ChangedPolicy { + counts: counts.iter().copied().collect(), + reach, + }; + changed_run_set(&self.nodes, &history, &policy) + .into_iter() + .map(|(key, outcome)| (key.name, outcome)) + .collect() + } +} + +pub(crate) fn scenario_key(name: &str) -> NodeKey { + NodeKey { + kind: "model".to_owned(), + name: name.to_owned(), + } +} + +pub(crate) fn runs(reason: RunReason) -> ChangedOutcome { + ChangedOutcome::Runs(reason) +} + +pub(crate) fn after(name: &str) -> ChangedOutcome { + runs(RunReason::UpstreamRuns(scenario_key(name))) +} + +pub(crate) fn refreshed_by(job: &str) -> ChangedOutcome { + ChangedOutcome::HeldBack(HeldBack::RefreshedBy(vec![job.to_owned()])) +} + +pub(crate) fn reads_held(name: &str) -> ChangedOutcome { + ChangedOutcome::HeldBack(HeldBack::ReadsHeldBack(scenario_key(name))) +} + +/// A streaming source feeding a staging model and a fact built on it. +pub(crate) fn streaming() -> Scenario { + let mut scenario = Scenario::new(&[ + ("stg_events", &["src_events"]), + ("fct_events", &["stg_events"]), + ]); + scenario.build(&["stg_events", "fct_events"]); + scenario +} + +/// The ticket's two-job project, built once: job_a holds staging, job_b the finance models. +pub(crate) fn two_jobs() -> Scenario { + let mut scenario = Scenario::new(&[ + ("stg_orders", &["src_orders"]), + ("int_orders", &["stg_orders"]), + ("fct_revenue", &["int_orders"]), + ("rpt_pnl", &["fct_revenue"]), + ("orders_export", &["stg_orders"]), + ]); + scenario.build(&[ + "stg_orders", + "int_orders", + "fct_revenue", + "rpt_pnl", + "orders_export", + ]); + scenario.redefine("stg_orders", "v2"); + scenario +} + +pub(crate) const JOB_A: (&str, &[&str]) = ("job_a", &["stg_orders", "int_orders"]); +pub(crate) const JOB_B: (&str, &[&str]) = ("job_b", &["fct_revenue", "rpt_pnl"]); diff --git a/crates/sqlbuild-state/src/consumption/tests/test_decide.rs b/crates/sqlbuild-state/src/consumption/tests/test_decide.rs new file mode 100644 index 0000000000..1d43a54395 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests/test_decide.rs @@ -0,0 +1,299 @@ +use crate::consumption::constants::RECENT_EXECUTIONS; +use crate::consumption::models::ChangeKind::{Definition, SourceData}; +use crate::consumption::models::{ChangedOutcome, ForcedRun, RunReason}; +use crate::consumption::tests::helpers::{ + JOB_A, JOB_B, after, reads_held, refreshed_by, runs, streaming, two_jobs, +}; +use crate::consumption::tests::test_types::{DecideTestCase, Scenario}; + +#[test] +fn given_recorded_builds_when_deciding_changed_then_counts_and_reach_pick_what_runs() { + let mut new_data_rebuilt = streaming(); + new_data_rebuilt.new_data("src_events", "d2"); + new_data_rebuilt.build(&["stg_events"]); + let mut redefined_rebuilt = streaming(); + redefined_rebuilt.redefine("stg_events", "v2"); + redefined_rebuilt.build(&["stg_events"]); + let mut new_data = streaming(); + new_data.new_data("src_events", "d2"); + let mut pruned = streaming(); + pruned.new_data("src_events", "d2"); + pruned.build(&["stg_events"]); + pruned.prune("stg_events"); + let mut converging = Scenario::new(&[ + ("helper", &[]), + ("clean_orders", &["helper"]), + ("fct_revenue", &["helper"]), + ("dim_dates", &["src_dates"]), + ("summary", &["clean_orders", "fct_revenue", "dim_dates"]), + ]); + converging.build(&[ + "helper", + "clean_orders", + "fct_revenue", + "dim_dates", + "summary", + ]); + converging.redefine("helper", "v2"); + let mut propagated = two_jobs(); + propagated.build(&["stg_orders", "int_orders"]); + let two_jobs = two_jobs(); + let mut both_redefined = Scenario::new(&[ + ("stg_a", &["src_a"]), + ("stg_b", &["src_b"]), + ("joined", &["stg_b", "stg_a"]), + ]); + both_redefined.build(&["stg_a", "stg_b", "joined"]); + both_redefined.redefine("stg_a", "v2"); + both_redefined.redefine("stg_b", "v2"); + both_redefined.build(&["stg_a", "stg_b"]); + let mut redefined_two_up = Scenario::new(&[ + ("stg_a", &["src_a"]), + ("stg_b", &["src_b"]), + ("int_joined", &["stg_a", "stg_b"]), + ("mart", &["int_joined"]), + ]); + redefined_two_up.build(&["stg_a", "stg_b", "int_joined", "mart"]); + redefined_two_up.redefine("stg_a", "v2"); + redefined_two_up.redefine("stg_b", "v2"); + redefined_two_up.build(&["stg_a", "stg_b", "int_joined"]); + let mut renamed = streaming(); + renamed.force("fct_events", ForcedRun::PlannerChange("renamed".to_owned())); + let mut deep = Scenario::new(&[ + ("raw_copy", &["src_events"]), + ("stg_events", &["raw_copy"]), + ("daily", &["stg_events"]), + ]); + deep.build(&["raw_copy", "stg_events", "daily"]); + for batch in 0..=RECENT_EXECUTIONS { + deep.new_data("src_events", &format!("d{batch}")); + deep.build(&["raw_copy", "stg_events"]); + } + let test_cases = [ + DecideTestCase { + description: "an upstream rebuilt with only new data stays out of --changed", + outcomes: new_data_rebuilt.decide(&[Definition], None), + expected_outcomes: vec![ + ("stg_events", ChangedOutcome::Unchanged), + ("fct_events", ChangedOutcome::Unchanged), + ], + }, + DecideTestCase { + description: "counting source data, the same rebuilt upstream pulls its downstream in", + outcomes: new_data_rebuilt.decide(&[Definition, SourceData], None), + expected_outcomes: vec![ + ("stg_events", ChangedOutcome::Unchanged), + ( + "fct_events", + runs(RunReason::InputChanged("model:stg_events".to_owned())), + ), + ], + }, + DecideTestCase { + description: "an upstream rebuilt with a definition change pulls its downstream in", + outcomes: redefined_rebuilt.decide(&[Definition], None), + expected_outcomes: vec![ + ("stg_events", ChangedOutcome::Unchanged), + ( + "fct_events", + runs(RunReason::UpstreamDefinitionChanged( + "model:stg_events".to_owned(), + )), + ), + ], + }, + DecideTestCase { + description: "new source data is not a change by default", + outcomes: new_data.decide(&[Definition], None), + expected_outcomes: vec![ + ("stg_events", ChangedOutcome::Unchanged), + ("fct_events", ChangedOutcome::Unchanged), + ], + }, + DecideTestCase { + description: "counting only source data, new data runs the source's reader and downstream", + outcomes: new_data.decide(&[SourceData], None), + expected_outcomes: vec![ + ( + "stg_events", + runs(RunReason::InputChanged("src_events".to_owned())), + ), + ("fct_events", after("stg_events")), + ], + }, + DecideTestCase { + description: "an upstream rebuild whose earlier execution was pruned counts as a change", + outcomes: pruned.decide(&[Definition], None), + expected_outcomes: vec![ + ("stg_events", ChangedOutcome::Unchanged), + ( + "fct_events", + runs(RunReason::InputChanged("model:stg_events".to_owned())), + ), + ], + }, + DecideTestCase { + description: "a change stops at another triggered job; a node in no job runs", + outcomes: two_jobs.decide(&[Definition], Some(&[JOB_A, JOB_B])), + expected_outcomes: vec![ + ("stg_orders", runs(RunReason::DefinitionChanged)), + ("int_orders", after("stg_orders")), + ("fct_revenue", refreshed_by("job_b")), + ("rpt_pnl", refreshed_by("job_b")), + ("orders_export", after("stg_orders")), + ], + }, + DecideTestCase { + description: "a job without a trigger counts as no job, so its nodes run", + outcomes: two_jobs.decide(&[Definition], Some(&[JOB_A])), + expected_outcomes: vec![ + ("stg_orders", runs(RunReason::DefinitionChanged)), + ("int_orders", after("stg_orders")), + ("fct_revenue", after("int_orders")), + ("rpt_pnl", after("fct_revenue")), + ("orders_export", after("stg_orders")), + ], + }, + DecideTestCase { + description: "with no jobs at all, everything downstream runs", + outcomes: two_jobs.decide(&[Definition], Some(&[])), + expected_outcomes: vec![ + ("stg_orders", runs(RunReason::DefinitionChanged)), + ("int_orders", after("stg_orders")), + ("fct_revenue", after("int_orders")), + ("rpt_pnl", after("fct_revenue")), + ("orders_export", after("stg_orders")), + ], + }, + DecideTestCase { + description: "--reach all ignores job edges", + outcomes: two_jobs.decide(&[Definition], None), + expected_outcomes: vec![ + ("stg_orders", runs(RunReason::DefinitionChanged)), + ("int_orders", after("stg_orders")), + ("fct_revenue", after("int_orders")), + ("rpt_pnl", after("fct_revenue")), + ("orders_export", after("stg_orders")), + ], + }, + DecideTestCase { + description: "a model in two jobs runs when one of them holds the change", + outcomes: two_jobs.decide( + &[Definition], + Some(&[ + ("job_a", &["stg_orders", "int_orders", "fct_revenue"]), + JOB_B, + ]), + ), + expected_outcomes: vec![ + ("stg_orders", runs(RunReason::DefinitionChanged)), + ("int_orders", after("stg_orders")), + ("fct_revenue", after("int_orders")), + ("rpt_pnl", refreshed_by("job_b")), + ("orders_export", after("stg_orders")), + ], + }, + DecideTestCase { + description: "once the change reached a model in both jobs, its downstream still waits for the job without it", + outcomes: propagated.decide( + &[Definition], + Some(&[JOB_A, ("job_b", &["int_orders", "fct_revenue", "rpt_pnl"])]), + ), + expected_outcomes: vec![ + ("stg_orders", ChangedOutcome::Unchanged), + ("int_orders", ChangedOutcome::Unchanged), + ("fct_revenue", refreshed_by("job_b")), + ("rpt_pnl", refreshed_by("job_b")), + ( + "orders_export", + runs(RunReason::UpstreamDefinitionChanged( + "model:stg_orders".to_owned(), + )), + ), + ], + }, + DecideTestCase { + description: "branches meeting behind a held-back node are held back; untouched parents never block", + outcomes: converging.decide(&[Definition], Some(&[("job_b", &["fct_revenue"])])), + expected_outcomes: vec![ + ("helper", runs(RunReason::DefinitionChanged)), + ("clean_orders", after("helper")), + ("fct_revenue", refreshed_by("job_b")), + ("dim_dates", ChangedOutcome::Unchanged), + ("summary", reads_held("fct_revenue")), + ], + }, + DecideTestCase { + description: "a node runs when any job holding one of its changes contains it, whatever the input order", + outcomes: both_redefined.decide( + &[Definition], + Some(&[("job_a", &["stg_a", "joined"]), ("job_b", &["stg_b"])]), + ), + expected_outcomes: vec![ + ("stg_a", ChangedOutcome::Unchanged), + ("stg_b", ChangedOutcome::Unchanged), + ( + "joined", + runs(RunReason::UpstreamDefinitionChanged( + "model:stg_b".to_owned(), + )), + ), + ], + }, + DecideTestCase { + description: "a node runs when its job holds any definition change two levels up, whatever the input order", + outcomes: redefined_two_up.decide( + &[Definition], + Some(&[ + ("job_a", &["stg_b", "mart"]), + ("job_b", &["stg_a", "int_joined"]), + ]), + ), + expected_outcomes: vec![ + ("stg_a", ChangedOutcome::Unchanged), + ("stg_b", ChangedOutcome::Unchanged), + ("int_joined", ChangedOutcome::Unchanged), + ( + "mart", + runs(RunReason::UpstreamDefinitionChanged( + "model:int_joined".to_owned(), + )), + ), + ], + }, + DecideTestCase { + description: "a planner change that is not a definition edit runs even when only source data counts", + outcomes: renamed.decide(&[SourceData], None), + expected_outcomes: vec![ + ("stg_events", ChangedOutcome::Unchanged), + ( + "fct_events", + runs(RunReason::Forced(ForcedRun::PlannerChange( + "renamed".to_owned(), + ))), + ), + ], + }, + DecideTestCase { + description: "new data through a chain rebuilt more often than the recent window is still not a change", + outcomes: deep.decide(&[Definition], None), + expected_outcomes: vec![ + ("raw_copy", ChangedOutcome::Unchanged), + ("stg_events", ChangedOutcome::Unchanged), + ("daily", ChangedOutcome::Unchanged), + ], + }, + ]; + for test_case in test_cases { + let outcomes: Vec<(&str, ChangedOutcome)> = test_case + .outcomes + .iter() + .map(|(name, outcome)| (name.as_str(), outcome.clone())) + .collect(); + assert_eq!( + outcomes, test_case.expected_outcomes, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/consumption/tests/test_types.rs b/crates/sqlbuild-state/src/consumption/tests/test_types.rs index badcfb5b78..7ed8e9b34d 100644 --- a/crates/sqlbuild-state/src/consumption/tests/test_types.rs +++ b/crates/sqlbuild-state/src/consumption/tests/test_types.rs @@ -78,3 +78,17 @@ pub(crate) struct SimulationTestCase { pub(crate) miss: Option<(u64, String, Vec)>, pub(crate) expected_miss: bool, } + +/// A small project recorded with real `started` and `succeeded` rows, for decision cases. +pub(crate) struct Scenario { + pub(crate) nodes: Vec, + pub(crate) sources: std::collections::BTreeMap, + pub(crate) rows: Vec, + pub(crate) clock: i64, +} + +pub(crate) struct DecideTestCase { + pub(crate) description: &'static str, + pub(crate) outcomes: Vec<(String, crate::consumption::models::ChangedOutcome)>, + pub(crate) expected_outcomes: Vec<(&'static str, crate::consumption::models::ChangedOutcome)>, +} diff --git a/fensu.toml b/fensu.toml index 66dcc9a651..4321b73727 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2236,7 +2236,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=12", + "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs=10", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 843fa32d69..f2ab366dba 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -57,7 +57,8 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | -| `--changed` | Build only the selected models and seeds whose definition or inputs changed since they last succeeded; see [Building only what changed](#building-only-what-changed) | +| `--changed` | Build only the selected models and seeds that changed, and what they reach downstream; `--changed=definition,source_data` picks what counts. See [Building only what changed](#building-only-what-changed) | +| `--reach` | With `--changed`: `job` stops at models another scheduled job refreshes, `all` builds everything downstream | | `--defer-to` | Resolve unselected model references against another target | | `--defer-sources-to` | Read managed source data from another target | | `--fail-fast` | Stop on first failure and skip remaining nodes | @@ -92,40 +93,101 @@ This replaces the former `sqb run` command. The full lifecycle (tests + audits) ## Building only what changed -`--changed` skips each selected model or seed that would read exactly what it read the last time -it built successfully: +`--changed` builds the selected models and seeds that changed since they last built, and what +they reach downstream. From the command line only definition changes count by default, and a change +stops at models that another scheduled job refreshes: ```bash sqb build --changed ``` ``` -Changed: 2 of 3 selected models and seeds run; 1 unchanged are skipped. +Counting: definition (source data not counted) · reach: job +Changed: 4 of 8 selected models and seeds run; 1 unchanged and 3 held back are skipped. ``` -Add `--verbose` to print why each one runs, such as `input raw_orders changed`. +Preview it with `sqb plan --changed`, which lists each node under Changed, Downstream of changes or +Held back with the reason. Add `--verbose` to a build to print the same reasons. -A selected model or seed runs when any of these is true: +### What counts as a change + +| Kind | Counts when | Example | +|------|-------------|---------| +| `definition` | Its SQL, config, hooks, contract or a function it calls changed, or an upstream was rebuilt with such a change | Edited `stg_orders.sql` | +| `source_data` | A source it reads has a different [freshness](../concepts/sources.md#source-freshness) value, or an upstream was rebuilt with new data only | New rows raise `MAX(loaded_at)` | + +Pick the kinds on the command line or in `sqlbuild_project.toml`: + +```bash +sqb build --changed=definition # the default +sqb build --changed=source_data # new source data only +sqb build --changed=definition,source_data # both +``` + +```toml +[changes] +counts = ["definition"] # add "source_data" to count new source data +reach = "job" # or "all" +``` + +Without `[changes] counts`, the default depends on how the build starts: + +| Build | Counts by default | +|-------|-------------------| +| `sqb build --changed` and `sqb plan --changed` | `definition` | +| A job's run with `settings={'changed': True}`, started by a schedule, a sensor or by hand | `definition` and `source_data` | + +`[changes] counts` sets both, and a list after `--changed=` or in the job's settings overrides it. + +Write the kinds with `=`: `sqb build --changed definition` reads `definition` as a selector. + +A model or seed always runs, whatever counts, when: | Reason | Example | |--------|---------| | It has no successful build on record | A new model | | Its last build did not finish | The build was killed or failed after the model started writing | | Its relation is missing, or `--full-refresh` or a cursor override applies | `sqb build --changed --full-refresh` | -| Its definition changed | Edited SQL, config, hooks, contract or a function it calls | -| A source it reads has a different [freshness](../concepts/sources.md#source-freshness) value | New rows raise `MAX(loaded_at)` | -| An upstream model or seed was rebuilt since, or runs in this build | `stg_orders` ran, so `customer_orders` runs | -| An upstream model or seed needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` on every build until a changed `stg_orders` is built | +| The planner must rebuild it for a reason other than an edit to its definition | A renamed model | | What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python task or asset upstream | +| An upstream needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` until a changed `stg_orders` is built | + +### How far a change reaches + +With `--reach job`, the default, a change runs everything downstream until it reaches a model that +belongs to a scheduled job without the changed model. That job's next run refreshes it, so the +model is held back: + +``` + staging_daily (schedule) finance_daily (schedule) + stg_orders* ─▶ int_orders ✓ ──────────▶ fct_revenue ⏸ ─▶ rpt_pnl ⏸ + │ + └──▶ orders_export ✓ (in no scheduled job) +``` + +- A job counts when a schedule, calendar schedule or sensor starts it, directly or through its + group, for the target you build. A job nothing starts counts as no job. +- A model held back by one job runs if another job that holds the change also contains it. +- A model that reads a held-back model is held back too. Unchanged parents never hold a model back. +- A job never holds back its own models while it runs: a scheduled run, or a build that selects the + job, such as `sqb build finance_daily --changed`. +- With no scheduled jobs, everything downstream runs. + +`--reach all` builds everything downstream of a change. `--reach` without `--changed` is refused. + +### In a scheduled job + +Set `settings={'changed': True}` to count definition and source data changes, or what +`[changes] counts` sets, or name the kinds, such as `settings={'changed': ['definition']}`. A +sensor on new source data therefore still rebuilds its job's models. A job's run builds every +changed model it selects, whatever its `reach` setting. Skipped models count as successful nodes +of the run. A model that runs under `--changed` plans exactly as in a plain build: its [`replay_on_change`](../concepts/planning/replay-decisions.md) policy still decides how much history an incremental model reprocesses. A model that runs only because an upstream was rebuilt takes its normal incremental step. -In a scheduled job, set `settings={'changed': True}`. Skipped models count as successful nodes of -the run. - Every build, with or without `--changed`, records what each model and seed read in `_sqlbuild_consumption` in its target schema, so the build needs permission to create and insert into that table. [`sqb janitor`](janitor.md) keeps the newest @@ -134,12 +196,14 @@ into that table. [`sqb janitor`](janitor.md) keeps the newest Limitations: - A source is compared only through its freshness value. A change that doesn't move it, such as an - update that keeps the old `loaded_at`, isn't seen. A source without freshness configuration counts - as changed on every build. + update that keeps the old `loaded_at`, isn't seen. With `source_data` counted, a source without + freshness configuration counts as changed on every build. - SQL that depends on the time, such as `current_date`, or on random values never counts as changed. Build those models without `--changed`. - Two builds of one model less than 5 seconds apart can't be ordered, so its downstream models run on the next `--changed` build. +- When the record of an upstream's earlier build has been pruned, SQLBuild can't tell a definition + change from new data, so the upstream counts as changed. ## Execution order diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 5524f18908..e05e65aa7c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -37,6 +37,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--json` | Output the plan as JSON | | `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](../concepts/planning/selection-and-staleness.md) | | `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | +| `--changed` | Show what `sqb build --changed` would run, which nodes it holds back and why; takes `--changed=definition,source_data` and `--reach` like [`sqb build`](build.md#building-only-what-changed). In `--json`, the `changed` object lists `runs`, `held_back` and `unchanged` | | `--start-cursor` | Override the start cursor, typed from the selected incremental models' cursor type; fails with `S303` if the selection mixes timestamp and integer cursors. With no cursor model selected, an integer value is read as an integer cursor and anything else as an ISO timestamp | | `--end-cursor` | Override the end cursor, typed the same way as `--start-cursor` | | `--start-cursor-ts` | Override start cursor for timestamp incremental models (ISO format) | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7fa3ba1a37..c0368ba850 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6524,6 +6524,8 @@ class BuildCommandRequest(NativeRecord): fail_fast: bool full_refresh: bool changed: bool + changed_counts: tuple[str, ...] | None + reach: str | None load_sources: bool | None reload_sources: bool include_python: bool @@ -6563,6 +6565,8 @@ class BuildCommandRequest(NativeRecord): fail_fast: bool = False, full_refresh: bool = False, changed: bool = False, + changed_counts: tuple[str, ...] | None = None, + reach: str | None = None, load_sources: bool | None = None, reload_sources: bool = False, include_python: bool = True, @@ -6926,6 +6930,9 @@ class PlanCommandRequest(NativeRecord): cursor_overrides: CursorOverrides | None json_output: bool full_refresh: bool + changed: bool + changed_counts: tuple[str, ...] | None + reach: str | None load_sources: bool | None include_python: bool no_color: bool @@ -6947,6 +6954,9 @@ class PlanCommandRequest(NativeRecord): cursor_overrides: CursorOverrides | None = None, json_output: bool = False, full_refresh: bool = False, + changed: bool = False, + changed_counts: tuple[str, ...] | None = None, + reach: str | None = None, load_sources: bool | None = None, include_python: bool = True, no_color: bool = False, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index e9449ca7af..ae15ae5ad1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1595,3 +1595,22 @@ class ChangedCrashE2ETestCase: warehouse: ChangedWarehouse expected_ran: frozenset[str] expected_killed_rows: list[tuple[object, ...]] + + +@dataclass(frozen=True) +class ChangedReachE2ETestCase: + description: str + warehouse: ChangedWarehouse + expected_redefined: frozenset[str] + expected_summary: str + expected_plan_lines: tuple[str, ...] + expected_job_run: frozenset[str] + expected_reach_all: frozenset[str] + + +@dataclass(frozen=True) +class ChangedTwoUpE2ETestCase: + description: str + warehouse: ChangedWarehouse + expected_crm_run: frozenset[str] + expected_downstream: frozenset[str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 29dad07049..fc8da19a0e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1252,6 +1252,14 @@ def target_concurrency_microbatch_files() -> dict[str, str]: CHANGED_BUILDS_UNQUALIFIED_FIXTURE: Path = ( REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_unqualified" ) +CHANGED_BUILDS_REACH_FIXTURE: Path = ( + REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_reach" +) +CHANGED_BUILDS_TWO_UP_FIXTURE: Path = ( + REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_two_up" +) +# Counts new source data too, so a build follows consumed inputs alone, whatever the defaults. +CHANGED_ANY: str = "--changed=definition,source_data" CHANGED_ORDER_SKEW_SECONDS: float = 5.2 _CHANGED_PROGRESS_PATTERN: re.Pattern[str] = re.compile( r"^\s+\d+/\d+\s+\S+\s+(\S+)\s+OK\b", re.MULTILINE @@ -1336,7 +1344,7 @@ def prepare_changed_project(*, tmp_path: Path, warehouse: ChangedWarehouse) -> P def add_changed_overlay(project_dir: Path, fixture: Path) -> None: """Overlay one of the DuckShop changed-builds cases on the base case.""" - for path in fixture.rglob("*.sql"): + for path in (*fixture.rglob("*.sql"), *fixture.rglob("*.py")): destination: Path = project_dir / path.relative_to(fixture) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8") @@ -1345,12 +1353,20 @@ def add_changed_overlay(project_dir: Path, fixture: Path) -> None: def run_changed_build(project_dir: Path, arguments: tuple[str, ...]) -> frozenset[str]: """Wait out the order skew, build successfully, and return the models and seeds written.""" + return changed_build_output(project_dir, arguments)[0] + + +def changed_build_output( + project_dir: Path, arguments: tuple[str, ...] +) -> tuple[frozenset[str], str]: + """Wait out the order skew, build successfully, and return what it wrote and its output.""" + time.sleep(CHANGED_ORDER_SKEW_SECONDS) result: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build", *arguments), project_dir=project_dir ) assert result.returncode == 0, result.stdout + result.stderr - return frozenset(_CHANGED_PROGRESS_PATTERN.findall(result.stdout)) + return frozenset(_CHANGED_PROGRESS_PATTERN.findall(result.stdout)), result.stdout def apply_changed_step( @@ -1408,13 +1424,13 @@ def changed_build_steps(warehouse: ChangedWarehouse) -> tuple[ChangedBuildStep, """Mutations and builds whose `--changed` run sets follow from consumed inputs alone.""" return ( - ChangedBuildStep("first --changed build", (), {}, ("--changed",)), - ChangedBuildStep("nothing changed", (), {}, ("--changed",)), + ChangedBuildStep("first --changed build", (), {}, (CHANGED_ANY,)), + ChangedBuildStep("nothing changed", (), {}, (CHANGED_ANY,)), ChangedBuildStep( "a new order", (f"INSERT INTO {warehouse.orders_table} VALUES (2, 1, 2)",), {}, - ("--changed",), + (CHANGED_ANY,), ), ChangedBuildStep( "a definition change", @@ -1425,19 +1441,19 @@ def changed_build_steps(warehouse: ChangedWarehouse) -> tuple[ChangedBuildStep, 'SELECT customer_id FROM __source("raw_customers") WHERE customer_id > 0\n' ) }, - ("--changed",), + (CHANGED_ANY,), ), ChangedBuildStep( "a stale unselected upstream", (f"INSERT INTO {warehouse.customers_table} VALUES (2, 2)",), {}, - ("--changed", "--select", "customer_orders"), + (CHANGED_ANY, "--select", "customer_orders"), ), ChangedBuildStep( - "the upstream is still stale", (), {}, ("--changed", "--select", "customer_orders") + "the upstream is still stale", (), {}, (CHANGED_ANY, "--select", "customer_orders") ), - ChangedBuildStep("building the whole selection", (), {}, ("--changed",)), + ChangedBuildStep("building the whole selection", (), {}, (CHANGED_ANY,)), ChangedBuildStep("a plain build", (), {}, ()), - ChangedBuildStep("after the plain build", (), {}, ("--changed",)), - ChangedBuildStep("full refresh", (), {}, ("--changed", "--full-refresh")), + ChangedBuildStep("after the plain build", (), {}, (CHANGED_ANY,)), + ChangedBuildStep("full refresh", (), {}, (CHANGED_ANY, "--full-refresh")), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py index 0578a7d699..54c79457f2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py @@ -2,6 +2,7 @@ from __future__ import annotations +import subprocess from pathlib import Path import pytest @@ -9,19 +10,27 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( ChangedBuildE2ETestCase, ChangedCrashE2ETestCase, + ChangedReachE2ETestCase, + ChangedTwoUpE2ETestCase, + ChangedWarehouse, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + CHANGED_ANY, + CHANGED_BUILDS_REACH_FIXTURE, + CHANGED_BUILDS_TWO_UP_FIXTURE, CHANGED_BUILDS_UDF_FIXTURE, CHANGED_BUILDS_UNQUALIFIED_FIXTURE, DUCKDB_CHANGED, SQLITE_CHANGED, add_changed_overlay, apply_changed_step, + changed_build_output, changed_build_steps, kill_changed_build_mid_write, prepare_changed_project, run_changed_build, ) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb EVERY_MODEL: frozenset[str] = frozenset({"stg_orders", "stg_customers", "customer_orders"}) # Functions deploy on every build; `--changed` decides models and seeds only. @@ -191,3 +200,122 @@ def test_given_a_build_killed_mid_write_when_building_with_changed_then_every_in assert killed_rows == test_case.expected_killed_rows assert run_changed_build(project_dir, ("--changed",)) == test_case.expected_ran assert run_changed_build(project_dir, ("--changed",)) == frozenset() + + +REACH_MODELS: frozenset[str] = EVERY_MODEL | { + "int_orders", + "fct_revenue", + "rpt_pnl", + "orders_export", + "summary", +} +STG_ORDERS_REDEFINED: str = ( + 'MODEL (description "Staged orders.", materialized table);\n' + 'SELECT order_id, customer_id FROM __source("raw_orders") WHERE order_id > 0\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedReachE2ETestCase( + description=DUCKDB_CHANGED.name, + warehouse=DUCKDB_CHANGED, + expected_redefined=frozenset( + {"stg_orders", "int_orders", "orders_export", "customer_orders"} + ), + expected_summary="Changed: 4 of 8 selected models and seeds run; " + "1 unchanged and 3 held back", + expected_plan_lines=( + "Counting: definition (source data not counted) · reach: job", + "Changed (1)", + "Downstream of changes (3)", + "Held back (3)", + "job finance_daily", + "reads fct_revenue, which is held back", + ), + expected_job_run=frozenset({"fct_revenue", "rpt_pnl"}), + expected_reach_all=frozenset({"summary"}), + ) + ], + ids=lambda case: case.description, +) +def test_given_scheduled_jobs_when_building_with_changed_then_a_change_stops_at_another_jobs_edge( + test_case: ChangedReachE2ETestCase, tmp_path: Path +) -> None: + warehouse: ChangedWarehouse = test_case.warehouse + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=warehouse) + add_changed_overlay(project_dir, CHANGED_BUILDS_REACH_FIXTURE) + first: frozenset[str] = run_changed_build(project_dir, (CHANGED_ANY,)) + warehouse.execute( + project_dir / warehouse.warehouse_file, "INSERT INTO raw.orders VALUES (2, 1, 2)" + ) + new_data: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + (project_dir / "models/stg_orders.sql").write_text(STG_ORDERS_REDEFINED, encoding="utf-8") + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--changed"), project_dir=project_dir + ) + redefined, build_output = changed_build_output(project_dir, ("--changed",)) + held: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + job_run: frozenset[str] = run_changed_build( + project_dir, ("--changed", "--select", "finance_daily") + ) + reach_all: frozenset[str] = run_changed_build(project_dir, ("--changed", "--reach", "all")) + refused: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--reach", "all"), project_dir=project_dir + ) + + assert first == REACH_MODELS + assert new_data == frozenset(), "new source data is not a change by default" + assert redefined == test_case.expected_redefined, ( + "the change stops at finance_daily; a job without a trigger and a model in no job run" + ) + assert test_case.expected_summary in build_output + assert plan.returncode == 0, plan.stdout + plan.stderr + for line in test_case.expected_plan_lines: + assert line in plan.stdout, plan.stdout + assert held == frozenset(), "held-back models wait for their job, not the next --changed" + assert job_run == test_case.expected_job_run, "running a job never holds back its own models" + assert reach_all == test_case.expected_reach_all + assert refused.returncode != 0 + assert "--reach needs --changed" in refused.stdout + refused.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedTwoUpE2ETestCase( + description=DUCKDB_CHANGED.name, + warehouse=DUCKDB_CHANGED, + expected_crm_run=frozenset({"stg_customers", "customer_orders"}), + expected_downstream=frozenset({"customer_sales"}), + ) + ], + ids=lambda case: case.description, +) +def test_given_two_jobs_each_holding_a_redefined_upstream_when_building_changed_then_a_model_two_levels_down_runs( + test_case: ChangedTwoUpE2ETestCase, tmp_path: Path +) -> None: + warehouse: ChangedWarehouse = test_case.warehouse + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=warehouse) + add_changed_overlay(project_dir, CHANGED_BUILDS_TWO_UP_FIXTURE) + first: frozenset[str] = run_changed_build(project_dir, (CHANGED_ANY,)) + (project_dir / "models/stg_orders.sql").write_text(STG_ORDERS_REDEFINED, encoding="utf-8") + (project_dir / "models/stg_customers.sql").write_text( + 'MODEL (description "Staged customers.", materialized table);\n' + 'SELECT customer_id FROM __source("raw_customers") WHERE customer_id > 0\n', + encoding="utf-8", + ) + sales_staged: frozenset[str] = run_changed_build( + project_dir, ("--changed", "--select", "stg_orders") + ) + crm_run: frozenset[str] = run_changed_build(project_dir, ("--changed", "--select", "crm_daily")) + downstream: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + + assert first == {"stg_customers", "stg_orders", "customer_orders", "customer_sales"} + assert sales_staged == {"stg_orders"} + assert crm_run == test_case.expected_crm_run + assert downstream == test_case.expected_downstream, ( + "sales_daily holds the stg_orders change that reached customer_orders, so customer_sales " + "runs even though crm_daily holds the stg_customers change too" + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 2e4d50dc5f..cc854af7c3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1579,3 +1579,11 @@ def run_write_set(control_store: Path, run_id: str) -> frozenset[str]: "SELECT request_json FROM sqb_runs WHERE run_id=?", (run_id,) ).fetchone() return frozenset(json.loads(request)["write_set"]) + + +def create_raw_orders(warehouse: Path) -> None: + """The source table a changed job's model reads.""" + with duckdb.connect(str(warehouse)) as connection: + _ = connection.execute( + "CREATE TABLE main.raw_orders AS SELECT CAST(1 AS BIGINT) AS order_id" + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py index 5490cd0a8b..18be7d027e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py @@ -22,6 +22,7 @@ WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + create_raw_orders, location_process, scheduled_worker, wait_for_generation, @@ -70,6 +71,23 @@ ).read_text(), }, ), + WorkerExecutionTestCase( + "a changed: True job still rebuilds on new source data", + 'SELECT 73 AS amount FROM __source("raw_orders") LIMIT 1', + "SELECT amount FROM orders", + "success", + [(73,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py" + ).read_text(), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + warehouse_setup=create_raw_orders, + ), ], ids=lambda case: case.description, ) @@ -183,3 +201,8 @@ def test_given_external_source_versions_when_a_location_resyncs_then_only_change ) with sqlite3.connect(worker.control_store) as control: assert control.execute("SELECT COUNT(*) FROM sqb_run_requests").fetchone() == (2,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.execute( + "SELECT COUNT(DISTINCT execution_id) FROM main._sqlbuild_consumption " + "WHERE node_name = 'orders' AND phase = 'succeeded'" + ).fetchall() == [(2,)], "each sensor run builds orders again" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 5bfb5e9d9a..c1237066e2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -287,14 +287,15 @@ def test_given_semantic_edits_when_publishing_then_local_identities_keep_planner ).to_json() ) hashes: dict[str, str] = {} - for key in ( - "model:orders", - "model:stg_orders", - "udf:edge_next_quantity", - "seed:edge_quantity_steps", + for key, expected_identity_keys in ( + ("model:orders", {"local_hash", "dialect"}), + ("model:stg_orders", {"local_hash", "dialect"}), + ("udf:edge_next_quantity", {"local_hash"}), + ("seed:edge_quantity_steps", {"local_hash"}), ): identity: dict[str, str] = snapshot["nodes"][key]["definition"]["planner_identity"] - assert set(identity) == {"local_hash"} + assert set(identity) == expected_identity_keys + assert identity.get("dialect", "duckdb") == "duckdb" assert len(identity["local_hash"]) == 64 hashes[key] = identity["local_hash"] identities.append(hashes) diff --git a/ui/scripts/extract-command-table.mjs b/ui/scripts/extract-command-table.mjs index 153bfba975..14de1341a4 100644 --- a/ui/scripts/extract-command-table.mjs +++ b/ui/scripts/extract-command-table.mjs @@ -74,6 +74,8 @@ const REQUIRES = new Map([ function valueType(action) { const flag = action.options[0]; if (action.kind === 'store_true' || action.kind === 'store_false') return 'bool'; + // A value only `--flag=value` sets; a bare flag never takes the next token. + if (action.nargs === '=?') return 'attached'; if (flag.endsWith('-ts')) return 'timestamp'; if (flag.endsWith('-int') || flag === '--max-microbatches' || flag === '--concurrency') return 'int'; if (action.kind === 'extend') return 'selector'; diff --git a/ui/src/lib/run/command-table.generated.json b/ui/src/lib/run/command-table.generated.json index 7a49b42549..8498e446d6 100644 --- a/ui/src/lib/run/command-table.generated.json +++ b/ui/src/lib/run/command-table.generated.json @@ -131,6 +131,27 @@ "requires": "run.rewrite_history", "description": "Rebuild incremental models and snapshots from scratch" }, + { + "flag": "--changed", + "aliases": [], + "type": "attached", + "choices": null, + "repeatable": false, + "requires": null, + "description": "run only what changed, and what it reaches downstream; COUNTS lists what counts as a change, such as --changed=definition,source_data (default: [changes] counts, else definition)" + }, + { + "flag": "--reach", + "aliases": [], + "type": "string", + "choices": [ + "job", + "all" + ], + "repeatable": false, + "requires": null, + "description": "with --changed: job stops at nodes another triggered job refreshes; all runs everything downstream (default: [changes] reach, else job)" + }, { "flag": "--allow-snapshot-full-refresh", "aliases": [], diff --git a/ui/src/lib/run/command-table.ts b/ui/src/lib/run/command-table.ts index 27bbb8e1a4..c1c29e9a34 100644 --- a/ui/src/lib/run/command-table.ts +++ b/ui/src/lib/run/command-table.ts @@ -4,7 +4,8 @@ import type { Capability, LaunchVerb } from '$lib/api'; import table from './command-table.generated.json'; -export type FlagType = 'bool' | 'string' | 'int' | 'timestamp' | 'selector'; +/** `attached` takes a value only as `--flag=value`; a bare flag is a switch. */ +export type FlagType = 'bool' | 'attached' | 'string' | 'int' | 'timestamp' | 'selector'; export type FlagSpec = { flag: string; diff --git a/ui/src/lib/run/command.test.ts b/ui/src/lib/run/command.test.ts index 06e6229126..d63198e55d 100644 --- a/ui/src/lib/run/command.test.ts +++ b/ui/src/lib/run/command.test.ts @@ -24,6 +24,13 @@ describe('the plan page command line', () => { expect(values('sqb build --select marts')).toEqual(['path:models/marts']); }); + it('reads --changed as the CLI does: its counts only after =, never the next token', () => { + expect(parseCommand('sqb build --changed customers').select).toEqual(['customers']); + expect(parseCommand('sqb build --changed customers').flags).toEqual({ '--changed': true }); + expect(parseCommand('sqb build --changed=definition,source_data --reach all').flags).toEqual({ '--changed': 'definition,source_data', '--reach': 'all' }); + expect(withVerb('sqb build --select customers --changed=source_data --reach all', 'build')).toBe('sqb build --select customers --changed=source_data --reach all '); + }); + it('keeps the selection and only the flags the new verb accepts when switching verbs', () => { expect(withVerb('sqb build --select customers --full-refresh --warehouse WH_S', 'test')).toBe('sqb test --select customers --warehouse WH_S '); }); diff --git a/ui/src/lib/run/command.ts b/ui/src/lib/run/command.ts index cd542c25e0..b347628ff8 100644 --- a/ui/src/lib/run/command.ts +++ b/ui/src/lib/run/command.ts @@ -43,9 +43,15 @@ export function parseCommand(text: string): CommandParts { continue; } mode = 'select'; - const spec = flagSpec(verb, token); + const equals = token.indexOf('='); + const name = equals > 0 ? token.slice(0, equals) : token; + const spec = flagSpec(verb, name); + if (equals > 0) { + flags[spec?.flag ?? name] = token.slice(equals + 1); + continue; + } const value = tokens[i + 1]; - if (!spec || spec.type === 'bool' || value === undefined || value.startsWith('-')) flags[spec?.flag ?? token] = true; + if (!spec || spec.type === 'bool' || spec.type === 'attached' || value === undefined || value.startsWith('-')) flags[spec?.flag ?? token] = true; else { flags[spec.flag] = value; i++; @@ -62,7 +68,10 @@ export function buildCommand(parts: CommandParts): string { const out = ['sqb', parts.verb]; if (parts.select.length) out.push('--select', ...parts.select); if (parts.exclude.length) out.push('--exclude', ...parts.exclude); - for (const [flag, value] of Object.entries(parts.flags)) out.push(value === true ? flag : `${flag} ${value}`); + for (const [flag, value] of Object.entries(parts.flags)) { + if (value === true) out.push(flag); + else out.push(flagSpec(parts.verb, flag)?.type === 'attached' ? `${flag}=${value}` : `${flag} ${value}`); + } return out.join(' '); } @@ -206,7 +215,7 @@ export function suggest(text: string, caret: number, nodes: SnapshotNode[]): Sug } const spec: FlagSpec | undefined = ctx.previousFlag ? flagSpec(parts.verb, ctx.previousFlag) : undefined; const prevToken = tokenize(text.slice(0, ctx.start)).at(-1) ?? ''; - if (spec && spec.type !== 'selector' && prevToken === ctx.previousFlag) { + if (spec && spec.type !== 'selector' && spec.type !== 'attached' && prevToken === ctx.previousFlag) { if (spec.choices) return spec.choices.filter((c) => c.startsWith(ctx.text)).map((c) => ({ value: c, label: c, hint: spec.flag })); if (spec.type === 'timestamp' && !ctx.text) return [{ value: new Date().toISOString().slice(0, 10) + 'T00:00:00Z', label: 'today 00:00 UTC', hint: 'timestamp' }]; return []; diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 46a5523148..c8956b6929 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -41,7 +41,8 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | -| `--changed` | Build only the selected models and seeds whose definition or inputs changed since they last succeeded; see [Building only what changed](#building-only-what-changed) | +| `--changed` | Build only the selected models and seeds that changed, and what they reach downstream; `--changed=definition,source_data` picks what counts. See [Building only what changed](#building-only-what-changed) | +| `--reach` | With `--changed`: `job` stops at models another scheduled job refreshes, `all` builds everything downstream | | `--defer-to` | Resolve unselected model references against another target | | `--defer-sources-to` | Read managed source data from another target | | `--fail-fast` | Stop on first failure and skip remaining nodes | @@ -76,40 +77,101 @@ This replaces the former `sqb run` command. The full lifecycle (tests + audits) ## Building only what changed -`--changed` skips each selected model or seed that would read exactly what it read the last time -it built successfully: +`--changed` builds the selected models and seeds that changed since they last built, and what +they reach downstream. From the command line only definition changes count by default, and a change +stops at models that another scheduled job refreshes: ```bash sqb build --changed ``` ``` -Changed: 2 of 3 selected models and seeds run; 1 unchanged are skipped. +Counting: definition (source data not counted) · reach: job +Changed: 4 of 8 selected models and seeds run; 1 unchanged and 3 held back are skipped. ``` -Add `--verbose` to print why each one runs, such as `input raw_orders changed`. +Preview it with `sqb plan --changed`, which lists each node under Changed, Downstream of changes or +Held back with the reason. Add `--verbose` to a build to print the same reasons. -A selected model or seed runs when any of these is true: +### What counts as a change + +| Kind | Counts when | Example | +|------|-------------|---------| +| `definition` | Its SQL, config, hooks, contract or a function it calls changed, or an upstream was rebuilt with such a change | Edited `stg_orders.sql` | +| `source_data` | A source it reads has a different [freshness](/docs/concepts/sources#source-freshness) value, or an upstream was rebuilt with new data only | New rows raise `MAX(loaded_at)` | + +Pick the kinds on the command line or in `sqlbuild_project.toml`: + +```bash +sqb build --changed=definition # the default +sqb build --changed=source_data # new source data only +sqb build --changed=definition,source_data # both +``` + +```toml +[changes] +counts = ["definition"] # add "source_data" to count new source data +reach = "job" # or "all" +``` + +Without `[changes] counts`, the default depends on how the build starts: + +| Build | Counts by default | +|-------|-------------------| +| `sqb build --changed` and `sqb plan --changed` | `definition` | +| A job's run with `settings={'changed': True}`, started by a schedule, a sensor or by hand | `definition` and `source_data` | + +`[changes] counts` sets both, and a list after `--changed=` or in the job's settings overrides it. + +Write the kinds with `=`: `sqb build --changed definition` reads `definition` as a selector. + +A model or seed always runs, whatever counts, when: | Reason | Example | |--------|---------| | It has no successful build on record | A new model | | Its last build did not finish | The build was killed or failed after the model started writing | | Its relation is missing, or `--full-refresh` or a cursor override applies | `sqb build --changed --full-refresh` | -| Its definition changed | Edited SQL, config, hooks, contract or a function it calls | -| A source it reads has a different [freshness](/docs/concepts/sources#source-freshness) value | New rows raise `MAX(loaded_at)` | -| An upstream model or seed was rebuilt since, or runs in this build | `stg_orders` ran, so `customer_orders` runs | -| An upstream model or seed needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` on every build until a changed `stg_orders` is built | +| The planner must rebuild it for a reason other than an edit to its definition | A renamed model | | What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python task or asset upstream | +| An upstream needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` until a changed `stg_orders` is built | + +### How far a change reaches + +With `--reach job`, the default, a change runs everything downstream until it reaches a model that +belongs to a scheduled job without the changed model. That job's next run refreshes it, so the +model is held back: + +``` + staging_daily (schedule) finance_daily (schedule) + stg_orders* ─▶ int_orders ✓ ──────────▶ fct_revenue ⏸ ─▶ rpt_pnl ⏸ + │ + └──▶ orders_export ✓ (in no scheduled job) +``` + +- A job counts when a schedule, calendar schedule or sensor starts it, directly or through its + group, for the target you build. A job nothing starts counts as no job. +- A model held back by one job runs if another job that holds the change also contains it. +- A model that reads a held-back model is held back too. Unchanged parents never hold a model back. +- A job never holds back its own models while it runs: a scheduled run, or a build that selects the + job, such as `sqb build finance_daily --changed`. +- With no scheduled jobs, everything downstream runs. + +`--reach all` builds everything downstream of a change. `--reach` without `--changed` is refused. + +### In a scheduled job + +Set `settings={'changed': True}` to count definition and source data changes, or what +`[changes] counts` sets, or name the kinds, such as `settings={'changed': ['definition']}`. A +sensor on new source data therefore still rebuilds its job's models. A job's run builds every +changed model it selects, whatever its `reach` setting. Skipped models count as successful nodes +of the run. A model that runs under `--changed` plans exactly as in a plain build: its [`replay_on_change`](/docs/concepts/planning/replay-decisions) policy still decides how much history an incremental model reprocesses. A model that runs only because an upstream was rebuilt takes its normal incremental step. -In a scheduled job, set `settings={'changed': True}`. Skipped models count as successful nodes of -the run. - Every build, with or without `--changed`, records what each model and seed read in `_sqlbuild_consumption` in its target schema, so the build needs permission to create and insert into that table. [`sqb janitor`](/docs/cli/janitor) keeps the newest @@ -118,12 +180,14 @@ into that table. [`sqb janitor`](/docs/cli/janitor) keeps the newest Limitations: - A source is compared only through its freshness value. A change that doesn't move it, such as an - update that keeps the old `loaded_at`, isn't seen. A source without freshness configuration counts - as changed on every build. + update that keeps the old `loaded_at`, isn't seen. With `source_data` counted, a source without + freshness configuration counts as changed on every build. - SQL that depends on the time, such as `current_date`, or on random values never counts as changed. Build those models without `--changed`. - Two builds of one model less than 5 seconds apart can't be ordered, so its downstream models run on the next `--changed` build. +- When the record of an upstream's earlier build has been pruned, SQLBuild can't tell a definition + change from new data, so the upstream counts as changed. ## Execution order diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 31670de0e0..a1a4d562c0 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -27,6 +27,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--json` | Output the plan as JSON | | `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](/docs/concepts/planning/selection-and-staleness) | | `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | +| `--changed` | Show what `sqb build --changed` would run, which nodes it holds back and why; takes `--changed=definition,source_data` and `--reach` like [`sqb build`](/docs/cli/build#building-only-what-changed). In `--json`, the `changed` object lists `runs`, `held_back` and `unchanged` | | `--start-cursor` | Override the start cursor, typed from the selected incremental models' cursor type; fails with `S303` if the selection mixes timestamp and integer cursors. With no cursor model selected, an integer value is read as an integer cursor and anything else as an ISO timestamp | | `--end-cursor` | Override the end cursor, typed the same way as `--start-cursor` | | `--start-cursor-ts` | Override start cursor for timestamp incremental models (ISO format) |