From 4a5b46cf715cda8975fecfc0d7826359d688e209 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 05:19:52 +0000 Subject: [PATCH 01/17] wip: catalogue entries for the performance rules, off by default --- .../src/engine/tests/test_incremental_rules.rs | 2 ++ .../src/rule_catalogue/constants.rs | 10 +++++++++- .../src/rules/_helpers/catalogue.rs | 18 +++++++++++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index f8521c390b..f48de345e7 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -36,6 +36,8 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRMODEL103", RuleScope::Model), ("SQBRMODEL104", RuleScope::Model), ("SQBRORCHESTRATION101", RuleScope::Project), + ("SQBRPERFORMANCE101", RuleScope::Model), + ("SQBRPERFORMANCE102", RuleScope::Model), ("SQBRPROJECT101", RuleScope::Model), ("SQBRPROJECT102", RuleScope::Model), ("SQBRPROJECT103", RuleScope::Model), diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs index 222049ed26..f098f35a22 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -33,7 +33,15 @@ pub(crate) const LAYOUT_RULE_PREFIXES: [&str; 2] = ["SQBRPROJECT2", "SQBRDECLARA pub(crate) const TEST_RULE_PREFIX: &str = "SQBRTEST"; pub(crate) const GRAPH_EDGE_RULE_CODE: &str = "SQBRGRAPH101"; pub(crate) const REPLACEABLE_SKILL_STATES: [&str; 3] = ["fresh", "missing", "stale"]; -pub(crate) const RULE_EXAMPLES: [(&str, &str); 28] = [ +pub(crate) const RULE_EXAMPLES: [(&str, &str); 30] = [ + ( + "SQBRPERFORMANCE101", + "SELECT o.value:customer_id, l.value:sku FROM raw_orders r, LATERAL FLATTEN(r.payload:orders) o, LATERAL FLATTEN(o.value:lines) l.", + ), + ( + "SQBRPERFORMANCE102", + "order_lines (order_id, line_no) selects o.payload:status, not the whole orders payload keyed by order_id.", + ), ( "SQLCLEAN040", "Compute category_key in an input CTE, then JOIN categories AS c ON o.category_key = c.category_key.", diff --git a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs index 812a79497f..b414c887d7 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs @@ -20,7 +20,7 @@ macro_rules! rule { remediation: $remediation.into(), guidance: None, implementation_fingerprint: NATIVE_BUILD_IDENTITY.into(), - enabled_by_default: true, + enabled_by_default: !matches!($family, "performance"), project_wide: matches!( $code, "SQBRDECLARATION201" @@ -374,6 +374,22 @@ pub(crate) fn catalogue() -> Vec { "SQL unit tests must not only prove that empty inputs produce no rows", "Mock representative input rows and assert concrete transformed output in __expected__ or a targeted __assert__ CTE, or delete this test. An empty-input test is legitimate only when it asserts concrete output, for example a global aggregate that must return one zero-valued summary row compared against expected rows. List a reviewed exception by test name in [rules.rule_options.SQBRTEST203] allowed_tests.", ), + rule!( + "SQBRPERFORMANCE101", + RuleScope::Model, + "performance", + "flatten-parent-copy", + "flattened rows must not copy the parent element whose child array they flatten", + "Read the fields you need from the parent (o.value:customer_id), or remove the flattened child array first with OBJECT_DELETE(o.value, 'lines').", + ), + rule!( + "SQBRPERFORMANCE102", + RuleScope::Model, + "performance", + "json-finer-grain", + "semi-structured columns must not repeat on a finer grain than their input", + "Keep the payload on the coarser model and join to it when needed, or select only the fields this grain needs.", + ), rule!( "SQBRTEST301", RuleScope::Project, From b8f6c86d31de39a0e1014dfc5a6f433832c8572f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 05:29:23 +0000 Subject: [PATCH 02/17] wip: SQBRPERFORMANCE101 flags a FLATTEN parent element copied whole onto the rows flattened from it --- .../src/engine/tests/test_query_facts.rs | 111 ++++++++++++++++ .../src/query_facts/_helpers/facts.rs | 17 +++ .../src/query_facts/_helpers/flatten.rs | 115 ++++++++++++++++ .../src/query_facts/_helpers/mod.rs | 1 + .../sqlbuild-rules/src/query_facts/models.rs | 22 ++++ .../src/rules/_helpers/evaluation.rs | 11 +- .../src/rules/_helpers/flatten_parent_copy.rs | 124 ++++++++++++++++++ .../sqlbuild-rules/src/rules/_helpers/mod.rs | 1 + 8 files changed, 400 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs create mode 100644 crates/sqlbuild-rules/src/rules/_helpers/flatten_parent_copy.rs diff --git a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs index fca1058f78..ad95d7ae1b 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs @@ -8,6 +8,12 @@ const ENUM_RULE: &[&str] = &["SQBRDECLARATION101"]; const NUMERIC_RULE: &[&str] = &["SQBRDECLARATION102"]; const OUTPUT_TYPE_RULE: &[&str] = &["SQBRCONTRACT105"]; const STATUS: &[&str] = &["status"]; +const FLATTEN_RULE: &[&str] = &["SQBRPERFORMANCE101"]; +macro_rules! orders_flattened { + () => { + "FROM __ref(\"raw_orders\") r, LATERAL FLATTEN(input => r.payload:orders) o, LATERAL FLATTEN(o.value:lines) l" + }; +} #[test] fn given_dialect_sql_when_reading_query_facts_then_findings_match_the_authored_sql() @@ -303,6 +309,111 @@ fn given_dialect_sql_when_reading_query_facts_then_findings_match_the_authored_s columns: json!([]), expected_faults: &[("SQBRDECLARATION102", 1, 64, "amount < 9")], }, + QueryFactsTestCase { + description: "the parent element copied whole onto each line is flagged at the column", + dialect: "snowflake", + query_sql: concat!( + "SELECT l.value:sku::string AS sku, o.value AS order_json\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[( + "SQBRPERFORMANCE101", + 1, + 36, + "order_json copies o.value whole onto every row flattened from o.value:lines", + )], + }, + QueryFactsTestCase { + description: "a field read from the parent is fine", + dialect: "snowflake", + query_sql: concat!( + "SELECT o.value:customer_id::string AS customer_id, l.value:sku::string AS sku\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, + QueryFactsTestCase { + description: "the parent without its flattened child array is fine", + dialect: "snowflake", + query_sql: concat!( + "SELECT OBJECT_DELETE(o.value, 'lines') AS order_json, l.value:sku AS sku\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, + QueryFactsTestCase { + description: "removing a different key still copies the child array", + dialect: "snowflake", + query_sql: concat!( + "SELECT OBJECT_DELETE(o.value, 'notes') AS order_json\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "copies o.value whole")], + }, + QueryFactsTestCase { + description: "the parent embedded in a built object is flagged", + dialect: "snowflake", + query_sql: concat!( + "SELECT OBJECT_CONSTRUCT('order', o.value, 'sku', l.value:sku) AS line_json\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "line_json copies o.value whole")], + }, + QueryFactsTestCase { + description: "an element no deeper FLATTEN reads is fine", + dialect: "snowflake", + query_sql: "SELECT o.value AS order_json FROM __ref(\"raw_orders\") r, LATERAL FLATTEN(input => r.payload:orders) o", + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, + QueryFactsTestCase { + description: "TABLE(FLATTEN(...)) and GET paths are read the same way", + dialect: "snowflake", + query_sql: "SELECT o.value AS order_json\nFROM __ref(\"raw_orders\") r, TABLE(FLATTEN(r.payload:orders)) o, TABLE(FLATTEN(GET(o.value, 'lines'))) l", + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "flattened from o.value:lines")], + }, + QueryFactsTestCase { + description: "a copy inside a CTE is flagged where it is made", + dialect: "snowflake", + query_sql: concat!( + "WITH lines AS (\n SELECT o.value AS order_json, l.value AS line_json\n ", + orders_flattened!(), + "\n)\nSELECT line_json FROM lines" + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 2, 10, "order_json copies o.value")], + }, + QueryFactsTestCase { + description: "the child element itself is not a parent copy", + dialect: "snowflake", + query_sql: concat!("SELECT l.value AS line_json\n", orders_flattened!()), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, ]; for test_case in test_cases { let faults = helpers::query_facts_faults(&test_case)?; diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs index a70b2a101a..46ca4fa8af 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs @@ -1,3 +1,4 @@ +use crate::query_facts::_helpers::flatten::{flatten_call, flatten_input, whole_values}; use crate::query_facts::_helpers::shape::stub_call; use crate::query_facts::_helpers::tokens::{ComparisonOperator, QueryTokens, SelectTokens}; use crate::query_facts::constants::{ @@ -340,6 +341,7 @@ impl Builder<'_> { ) }), start: range.and_then(|range| self.tokens.leaf_start(&range)), + whole_values: whole_values(value), }) } @@ -599,10 +601,25 @@ fn relation(expression: &Expression, tokens: &QueryTokens<'_>) -> RelationFacts last, alias: name, }, + RelationFacts::Flatten { input, .. } => RelationFacts::Flatten { + alias: name.map(|name| name.to_ascii_lowercase()), + input, + }, other => other, } } Expression::Table(table) => table_relation(table, tokens), + Expression::Lateral(lateral) => match flatten_call(expression) { + Some(function) => RelationFacts::Flatten { + alias: lateral.alias.as_ref().map(|alias| alias.to_ascii_lowercase()), + input: flatten_input(function), + }, + None => RelationFacts::Other, + }, + Expression::Function(_) if flatten_call(expression).is_some() => RelationFacts::Flatten { + alias: None, + input: flatten_call(expression).and_then(flatten_input), + }, Expression::Pivot(pivot) if !pivot.unpivot => { if pivot.fields.iter().any(|field| { matches!(field, Expression::In(list) diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs new file mode 100644 index 0000000000..bbaea5adf5 --- /dev/null +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs @@ -0,0 +1,115 @@ +//! FLATTEN facts: the element each FLATTEN reads, and the whole elements a select item copies. + +use sqlquack_syntax::expressions::functions::models::Function; +use sqlquack_syntax::expressions::names::models::Literal; +use sqlquack_syntax::{Expression, ExpressionWalk}; + +use crate::query_facts::models::{FlattenInput, WholeValue}; + +const VALUE_COLUMN: &str = "value"; + +/// The `FLATTEN` call a FROM item runs, as `LATERAL FLATTEN(...)` or `TABLE(FLATTEN(...))`. +pub(crate) fn flatten_call(expression: &Expression) -> Option<&Function> { + match expression { + Expression::Lateral(lateral) => flatten_call(&lateral.this), + Expression::Function(function) if function.name.eq_ignore_ascii_case("table") => { + function.args.first().and_then(flatten_call) + } + Expression::Function(function) if function.name.eq_ignore_ascii_case("flatten") => { + Some(function) + } + _ => None, + } +} + +/// The column path a `FLATTEN` reads, such as `o.value:lines`, when its input is one. +pub(crate) fn flatten_input(function: &Function) -> Option { + let input: &Expression = function.args.iter().find_map(|arg| match arg { + Expression::NamedArgument(named) => named + .name + .name + .eq_ignore_ascii_case("input") + .then_some(&named.value), + positional => Some(positional), + })?; + path_root(input) +} + +/// The column `expression` reads a path of, with the path's first key. +fn path_root(expression: &Expression) -> Option { + let (base, key): (&Expression, Option) = match expression { + Expression::Column(column) => { + return Some(FlattenInput { + qualifier: column.table.as_ref()?.name.to_ascii_lowercase(), + column: column.name.name.to_ascii_lowercase(), + key: None, + }); + } + Expression::Cast(cast) => return path_root(&cast.this), + Expression::JSONExtract(extract) => (&extract.this, literal_key(&extract.expression)), + Expression::Subscript(subscript) => (&subscript.this, literal_key(&subscript.index)), + Expression::Function(function) if is_path_function(function) => ( + function.args.first()?, + function.args.get(1).and_then(literal_key), + ), + _ => return None, + }; + let mut root: FlattenInput = path_root(base)?; + if root.key.is_none() { + root.key = key; + } + Some(root) +} + +fn is_path_function(function: &Function) -> bool { + ["get", "get_path"] + .iter() + .any(|name| function.name.eq_ignore_ascii_case(name)) +} + +/// The first key of a path literal such as `'lines.sku'` or `'lines'`. +fn literal_key(expression: &Expression) -> Option { + let Expression::Literal(literal) = expression else { + return None; + }; + let Literal::String(path) = literal.as_ref() else { + return None; + }; + let key: &str = path + .trim_start_matches('$') + .trim_start_matches('.') + .split(['.', '[']) + .next()?; + (!key.is_empty()).then(|| key.trim_matches('"').to_ascii_lowercase()) +} + +/// Every `.value` `expression` copies whole, with the keys `OBJECT_DELETE` removes from it. +pub(crate) fn whole_values(expression: &Expression) -> Vec { + let mut found: Vec = Vec::new(); + let mut pending: Vec<(&Expression, Vec)> = vec![(expression, Vec::new())]; + while let Some((node, removed)) = pending.pop() { + match node { + Expression::Column(column) if column.name.name.eq_ignore_ascii_case(VALUE_COLUMN) => { + if let Some(table) = &column.table { + found.push(WholeValue { + qualifier: table.name.to_ascii_lowercase(), + removed, + }); + } + } + Expression::JSONExtract(_) | Expression::Subscript(_) if path_root(node).is_some() => {} + Expression::Function(function) + if is_path_function(function) && path_root(node).is_some() => {} + Expression::Function(function) + if function.name.eq_ignore_ascii_case("object_delete") => + { + let mut args = function.args.iter(); + if let Some(object) = args.next() { + pending.push((object, args.filter_map(literal_key).collect())); + } + } + _ => pending.extend(node.children().into_iter().map(|child| (child, Vec::new()))), + } + } + found +} diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/mod.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/mod.rs index 90c1051a7c..a6eda9b1fa 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/mod.rs @@ -1,3 +1,4 @@ pub mod facts; +pub mod flatten; pub mod shape; pub mod tokens; diff --git a/crates/sqlbuild-rules/src/query_facts/models.rs b/crates/sqlbuild-rules/src/query_facts/models.rs index a5688f1e06..bd64b65dd8 100644 --- a/crates/sqlbuild-rules/src/query_facts/models.rs +++ b/crates/sqlbuild-rules/src/query_facts/models.rs @@ -95,6 +95,8 @@ pub(crate) struct ExpressionItem { pub(crate) start: Option, /// The normalized characters from the expression's first token to its last, alias excluded. pub(crate) span: Option<(usize, usize)>, + /// The FLATTEN elements the expression copies whole. + pub(crate) whole_values: Vec, } #[derive(Debug)] @@ -112,9 +114,29 @@ pub(crate) enum RelationFacts { alias: Option, }, DynamicPivot, + /// A `LATERAL FLATTEN(...)` or `TABLE(FLATTEN(...))` item and the column path it reads. + Flatten { + alias: Option, + input: Option, + }, Other, } +/// The column path a FLATTEN reads: `o.value:lines` is qualifier `o`, column `value`, key `lines`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct FlattenInput { + pub(crate) qualifier: String, + pub(crate) column: String, + pub(crate) key: Option, +} + +/// An `.value` a select item copies whole, less the keys `OBJECT_DELETE` removes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct WholeValue { + pub(crate) qualifier: String, + pub(crate) removed: Vec, +} + #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub(crate) struct ColumnFact { pub(crate) name: String, diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index 4e3256ff5e..51530509d6 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -10,6 +10,7 @@ use crate::constants::{ }; use crate::lint_text::constants::LINT_UNAVAILABLE_CODE; use crate::models::{Declaration, EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; +use crate::query_facts::constants::SNOWFLAKE_DIALECT; use crate::query_facts::main::dependency_texts::dependency_texts; use crate::query_facts::main::facts_available::facts_available; use crate::query_facts::main::model_shape::model_shape; @@ -20,8 +21,8 @@ use crate::query_facts::models::{ }; use crate::rules::_helpers::domain_layout::folder_layer_details; use crate::rules::_helpers::{ - contract_name_types, dynamic_contracts, explicit_output_types, model_layers, numeric_decisions, - sql_test_rules, typed_contract_columns, + contract_name_types, dynamic_contracts, explicit_output_types, flatten_parent_copy, + model_layers, numeric_decisions, sql_test_rules, typed_contract_columns, }; use crate::rules::models::{ FaultCollector, ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride, @@ -215,6 +216,11 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result bool { !rule.custom && (rule.code.starts_with("SQBRMODEL") || rule.code.starts_with("SQBRCONTRACT") + || rule.code == "SQBRPERFORMANCE101" || matches!( rule.code.as_str(), "SQBRGRAPH102" diff --git a/crates/sqlbuild-rules/src/rules/_helpers/flatten_parent_copy.rs b/crates/sqlbuild-rules/src/rules/_helpers/flatten_parent_copy.rs new file mode 100644 index 0000000000..7f16054252 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/flatten_parent_copy.rs @@ -0,0 +1,124 @@ +//! SQBRPERFORMANCE101: a flattened row must not copy the parent element whose child array it flattens. + +use std::collections::BTreeMap; + +use crate::models::{Fault, RuleMetadata}; +use crate::query_facts::models::{ + ExpressionItem, FlattenInput, ItemFacts, QueryFacts, RelationFacts, SelectFacts, SetFacts, +}; +use crate::rules::_helpers::evaluation::ParsedModel; +use crate::rules::models::FaultCollector; + +const VALUE_COLUMN: &str = "value"; + +pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &FaultCollector) { + let mut selects: Vec<&SelectFacts> = Vec::new(); + collect_query(parsed.facts, &mut selects); + for select in selects { + let children: BTreeMap<&str, Vec>> = flattened_parents(select); + for item in &select.items { + let ItemFacts::Expression(expression) = item else { + continue; + }; + if let Some((parent, key)) = copied_parent(expression, &children) { + push(parsed, rule, faults, expression, parent, key); + } + } + } +} + +fn collect_query<'a>(query: &'a QueryFacts, selects: &mut Vec<&'a SelectFacts>) { + for cte in &query.ctes { + collect_query(&cte.query, selects); + } + collect_set(&query.body, selects); +} + +fn collect_set<'a>(set: &'a SetFacts, selects: &mut Vec<&'a SelectFacts>) { + match set { + SetFacts::Select(select) => selects.push(select), + SetFacts::Query(query) => collect_query(query, selects), + SetFacts::SetOperation { left, right, .. } => { + collect_set(left, selects); + collect_set(right, selects); + } + SetFacts::Other => {} + } +} + +/// Each FLATTEN alias in `select` that a deeper FLATTEN reads, with the child keys read from it. +fn flattened_parents(select: &SelectFacts) -> BTreeMap<&str, Vec>> { + let relations = || select.from.iter().chain(&select.joins); + let aliases: Vec<&str> = relations() + .filter_map(|relation| match relation { + RelationFacts::Flatten { + alias: Some(alias), .. + } => Some(alias.as_str()), + _ => None, + }) + .collect(); + let mut parents: BTreeMap<&str, Vec>> = BTreeMap::new(); + for relation in relations() { + if let RelationFacts::Flatten { + input: + Some(FlattenInput { + qualifier, + column, + key, + }), + .. + } = relation + && column == VALUE_COLUMN + && aliases.contains(&qualifier.as_str()) + { + parents + .entry(qualifier.as_str()) + .or_default() + .push(key.as_deref()); + } + } + parents +} + +/// The parent `expression` copies whole and a child key it keeps, unless `OBJECT_DELETE` removes them all. +fn copied_parent<'a>( + expression: &'a ExpressionItem, + parents: &BTreeMap<&str, Vec>>, +) -> Option<(&'a str, Option<&'a str>)> { + expression.whole_values.iter().find_map(|whole| { + let keys = parents.get(whole.qualifier.as_str())?; + let kept = keys.iter().find(|key| { + key.is_none_or(|key| !whole.removed.iter().any(|removed| removed == key)) + })?; + Some((whole.qualifier.as_str(), *kept)) + }) +} + +fn push( + parsed: &ParsedModel<'_>, + rule: &RuleMetadata, + faults: &FaultCollector, + expression: &ExpressionItem, + parent: &str, + key: Option<&str>, +) { + let (line, column) = expression + .start + .map_or((1, 1), |offset| parsed.source.line_column(offset)); + let output: &str = expression.output_name.as_deref().unwrap_or("this column"); + let child: String = key.map_or_else( + || "its child array".to_owned(), + |key| format!("{parent}.value:{key}"), + ); + faults.push(Fault { + unevaluated: false, + code: rule.code.clone(), + path: parsed.model.relative_path.clone(), + line, + column, + message: format!( + "{output} copies {parent}.value whole onto every row flattened from {child}" + ), + remediation: rule.remediation.clone(), + }); +} diff --git a/crates/sqlbuild-rules/src/rules/_helpers/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs index 18833e9d3d..8c51a16dee 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs @@ -4,6 +4,7 @@ pub mod domain_layout; pub mod dynamic_contracts; pub mod evaluation; pub mod explicit_output_types; +pub mod flatten_parent_copy; pub mod model_layers; pub mod numeric_decisions; pub mod sql_test_rules; From e064a5de4cbb6da82fb362b3239e1d5c839a27df Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 05:41:46 +0000 Subject: [PATCH 03/17] wip: SQBRPERFORMANCE102 rule and grain facts on rules models --- .../src/engine/_helpers/project_rows.rs | 1 + .../src/engine/_helpers/retained.rs | 10 ++ .../src/engine/_helpers/rows.rs | 1 + .../src/engine/tests/helpers.rs | 20 +++- crates/sqlbuild-rules/src/models.rs | 28 +++++ .../src/rules/_helpers/evaluation.rs | 9 +- .../src/rules/_helpers/json_finer_grain.rs | 100 ++++++++++++++++++ .../sqlbuild-rules/src/rules/_helpers/mod.rs | 1 + 8 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index 46e814d073..cffac3626c 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -65,6 +65,7 @@ pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result .collect(), typed_calls: facts.typed_calls.clone(), sql_analysis_skipped: facts.sql_analysis_skipped, + json_grain: None, }) } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs index 006c1474f4..db58807940 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs @@ -152,6 +152,7 @@ impl RowWriter { empty_input_only_test_count: _, payload_digest: _, stripped: _, + json_grain, } = model; self.flag(*sql_analysis_disabled); self.text(name); @@ -217,6 +218,10 @@ impl RowWriter { self.declarations(constant_declarations); self.record.put_u64(u64::from(*declared_audit_count)); self.record.put_u64(u64::from(*targeting_test_count)); + let grain: Option = json_grain + .as_ref() + .map(|grain| serde_json::to_value(grain).unwrap_or(Value::Null)); + self.optional_value(grain.as_ref()); } } @@ -386,6 +391,11 @@ impl RowReader<'_> { empty_input_only_test_count: 0, payload_digest: None, stripped: false, + json_grain: self + .optional_value()? + .map(serde_json::from_value) + .transpose() + .map_err(|_| StoreDecodeError)?, }) } } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index c7a9a6bdee..b31ddf2e3d 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -104,6 +104,7 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result Model { empty_input_only_test_count: 0, payload_digest: Some("digest".to_owned()), stripped: false, + json_grain: Some(JsonGrain { + model_keys: vec![vec!["order_id".to_owned(), "line_id".to_owned()]], + columns: vec![GrainColumn { + name: "order_json".to_owned(), + data_type: Some("VARIANT".to_owned()), + inputs: vec!["ref:orders".to_owned()], + }], + input_keys: BTreeMap::from([( + "ref:orders".to_owned(), + vec![vec!["order_id".to_owned()]], + )]), + backs_published_view: true, + skipped: Some("no lineage".to_owned()), + }), } } diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index dfcdb43bf5..cd1b784cd5 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -608,6 +608,32 @@ pub struct Model { /// Whether this row's SQL text was dropped, so only a cached result may stand for it. #[serde(skip)] pub stripped: bool, + /// SQBRPERFORMANCE102's grain facts, present only when that rule is selected. + pub json_grain: Option, +} + +/// What SQBRPERFORMANCE102 compares: the model's keys, each output's type and lineage inputs, and the inputs' keys. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct JsonGrain { + /// Column sets each proving one row per value in this model. + pub model_keys: Vec>, + pub columns: Vec, + /// Each lineage input's keys by `kind:name`, such as `ref:orders`. + pub input_keys: BTreeMap>>, + /// Whether every reader is a published view of this model's own entity, as SQBRGRAPH103 defines one. + pub backs_published_view: bool, + /// Why the comparison cannot run, such as missing lineage; the model is then skipped, not flagged. + pub skipped: Option, +} + +/// One output column's inferred type and the relations its lineage reads, by `kind:name`. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct GrainColumn { + pub name: String, + pub data_type: Option, + pub inputs: Vec, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -957,6 +983,8 @@ pub struct ModelRow { pub typed_calls: Vec, /// Whether analysis was skipped because the dialect's native parser cannot read the model. pub sql_analysis_skipped: bool, + /// SQBRPERFORMANCE102's grain facts, present only when that rule is selected. + pub json_grain: Option, } /// A model's declared schema entry. diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index 51530509d6..22ab524809 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -22,7 +22,7 @@ use crate::query_facts::models::{ use crate::rules::_helpers::domain_layout::folder_layer_details; use crate::rules::_helpers::{ contract_name_types, dynamic_contracts, explicit_output_types, flatten_parent_copy, - model_layers, numeric_decisions, sql_test_rules, typed_contract_columns, + json_finer_grain, model_layers, numeric_decisions, sql_test_rules, typed_contract_columns, }; use crate::rules::models::{ FaultCollector, ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride, @@ -221,6 +221,11 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result bool { !rule.custom && (rule.code.starts_with("SQBRMODEL") || rule.code.starts_with("SQBRCONTRACT") - || rule.code == "SQBRPERFORMANCE101" + || rule.code.starts_with("SQBRPERFORMANCE") || matches!( rule.code.as_str(), "SQBRGRAPH102" diff --git a/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs b/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs new file mode 100644 index 0000000000..86bfdb21e0 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs @@ -0,0 +1,100 @@ +//! SQBRPERFORMANCE102: a semi-structured column must not repeat on a finer grain than its input. + +use crate::models::{Fault, GrainColumn, JsonGrain, RuleMetadata}; +use crate::query_facts::models::{ItemFacts, SetFacts}; +use crate::rules::_helpers::evaluation::ParsedModel; +use crate::rules::models::FaultCollector; + +const SEMI_STRUCTURED_TYPES: [&str; 3] = ["VARIANT", "OBJECT", "ARRAY"]; + +pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &FaultCollector) { + let Some(grain) = &parsed.model.json_grain else { + return; + }; + if grain.skipped.is_some() || grain.backs_published_view { + return; + } + for column in &grain.columns { + if !column.data_type.as_deref().is_some_and(|data_type| { + SEMI_STRUCTURED_TYPES + .iter() + .any(|kind| starts_with_word(data_type, kind)) + }) { + continue; + } + if let Some((input, input_key, model_key)) = coarser_input(column, grain) { + let (line, column_number) = output_start(&parsed.facts.body, &column.name) + .map_or((1, 1), |offset| parsed.source.line_column(offset)); + faults.push(Fault { + unevaluated: false, + code: rule.code.clone(), + path: parsed.model.relative_path.clone(), + line, + column: column_number, + message: format!( + "{} copies {} from {input}, one row per ({}), onto rows keyed by ({}), so the \ + same value repeats on every row", + column.name, + column.data_type.as_deref().unwrap_or_default(), + input_key.join(", "), + model_key.join(", "), + ), + remediation: rule.remediation.clone(), + }); + } + } +} + +fn starts_with_word(data_type: &str, kind: &str) -> bool { + let upper: String = data_type.trim().to_ascii_uppercase(); + upper == kind + || upper + .strip_prefix(kind) + .is_some_and(|rest| rest.starts_with(['(', '<', ' '])) +} + +/// An input of `column` whose key is a strict subset of one of the model's keys. +fn coarser_input<'a>( + column: &'a GrainColumn, + grain: &'a JsonGrain, +) -> Option<(&'a str, &'a [String], &'a [String])> { + column.inputs.iter().find_map(|input| { + let keys = grain.input_keys.get(input)?; + keys.iter().find_map(|input_key| { + grain + .model_keys + .iter() + .find(|model_key| strict_subset(input_key, model_key)) + .map(|model_key| (input.as_str(), input_key.as_slice(), model_key.as_slice())) + }) + }) +} + +fn strict_subset(smaller: &[String], larger: &[String]) -> bool { + smaller.len() < larger.len() + && smaller.iter().all(|column| { + larger + .iter() + .any(|other| other.eq_ignore_ascii_case(column)) + }) +} + +/// Where the root select projects `name`, following the left branch of set operations. +fn output_start(body: &SetFacts, name: &str) -> Option { + match body { + SetFacts::Select(select) => select.items.iter().find_map(|item| match item { + ItemFacts::Expression(expression) + if expression + .output_name + .as_deref() + .is_some_and(|output| output.eq_ignore_ascii_case(name)) => + { + expression.start + } + _ => None, + }), + SetFacts::Query(query) => output_start(&query.body, name), + SetFacts::SetOperation { left, .. } => output_start(left, name), + SetFacts::Other => None, + } +} diff --git a/crates/sqlbuild-rules/src/rules/_helpers/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs index 8c51a16dee..9a76967985 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs @@ -5,6 +5,7 @@ pub mod dynamic_contracts; pub mod evaluation; pub mod explicit_output_types; pub mod flatten_parent_copy; +pub mod json_finer_grain; pub mod model_layers; pub mod numeric_decisions; pub mod sql_test_rules; From 531721731d2872b1e596cc43a6df97d25b953553 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 05:54:12 +0000 Subject: [PATCH 04/17] wip: build SQBRPERFORMANCE102 grains from lineage, keys and readers --- .../src/assembly/analysis_session/models.rs | 7 + .../_helpers/compile_delta/rules_rows.rs | 6 + .../src/bindings/_helpers/rules/rows.rs | 46 ++- .../src/engine/main/json_grains.rs | 123 ++++++++ crates/sqlbuild-rules/src/engine/main/mod.rs | 1 + crates/sqlbuild-rules/src/engine/tests.rs | 2 + .../src/engine/tests/helpers.rs | 22 +- .../src/engine/tests/test_json_grains.rs | 263 ++++++++++++++++++ .../src/engine/tests/test_types.rs | 10 + crates/sqlbuild-rules/src/models.rs | 12 +- .../src/rules/_helpers/evaluation.rs | 2 +- .../src/rules/_helpers/json_finer_grain.rs | 30 +- .../src/rules/_helpers/model_layers.rs | 18 ++ 13 files changed, 510 insertions(+), 32 deletions(-) create mode 100644 crates/sqlbuild-rules/src/engine/main/json_grains.rs create mode 100644 crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index b8a426f348..928a8cb371 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -241,6 +241,13 @@ pub struct LineageRow { pub expression_range: Option<(usize, usize)>, } +impl LineageRow { + /// Whether the column aggregates its sources rather than carrying one source row's value. + pub fn aggregated(&self) -> bool { + self.transform_code == crate::assembly::analysis_session::constants::TRANSFORM_AGGREGATION + } +} + /// Where a model's lineage facts live. #[derive(Debug, Clone, PartialEq, Eq)] pub enum LineageFacts { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs index a22e5f2103..1d462f372e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use pyo3::PyResult; +use sqlbuild_rules::engine::main::json_grains::needs_json_grains; use sqlbuild_rules::models::{EvaluateRequest, Model, SqlTestFact}; use crate::bindings::_helpers::compile_context::delta::BaseRules; @@ -14,6 +15,11 @@ pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult = injection .rules .lock() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 2f1268aef5..b857d982af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -1,6 +1,6 @@ //! Build and evaluate the built-in rules request natively from compiled-project rows. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::path::PathBuf; use std::sync::Mutex; use std::sync::atomic::AtomicBool; @@ -18,19 +18,22 @@ use sqlbuild_rules::constants::API_VERSION; use sqlbuild_rules::engine::main::build_models::build_models; use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; use sqlbuild_rules::engine::main::finalize_rows::finalize_rows; +use sqlbuild_rules::engine::main::json_grains::{json_grains, needs_json_grains}; use sqlbuild_rules::engine::main::project_model_rows::project_model_rows; use sqlbuild_rules::errors::RowsError; use sqlbuild_rules::models::{ - Declaration, DeclarationMember, EvaluateRequest, Fault, Model, ModelRow, ModelRowsContext, - RowsEvaluation, RowsRequest, RulesConfig, + Declaration, DeclarationMember, EvaluateRequest, Fault, LineageColumn, Model, ModelRow, + ModelRowsContext, RowsEvaluation, RowsRequest, RulesConfig, }; use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; +use crate::bindings::_helpers::compiled_views::model_reads::native_model; use crate::bindings::_helpers::compiled_views::records::compiled_project_facts; use crate::bindings::_helpers::compiled_views::records::declaration_rows; +use crate::bindings::_helpers::compiled_views::relation_keys::project_relation_keys; use crate::bindings::_helpers::rules::config_json::asdict_json; use crate::bindings::_helpers::rules::early_builtin_rules::EarlyBuiltinRules; use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; @@ -293,10 +296,8 @@ fn built_request( Some(facts) => facts.clone(), None => compiled_project_facts(&project)?, }; - let names: Vec = project_models(&project)? - .into_iter() - .map(|model| model.name) - .collect(); + let models = project_models(&project)?; + let names: Vec = models.iter().map(|model| model.name.clone()).collect(); let sql_tests: Vec> = project_attribute(&project, "sql_tests")? .try_iter()? .collect::>()?; @@ -320,8 +321,17 @@ fn built_request( .iter() .map(finding_fault) .collect::>()?; - let model_rows: Vec = + let mut model_rows: Vec = project_model_rows(facts.borrow().facts(), &names).map_err(value_error)?; + if needs_json_grains(&request).map_err(value_error)? { + let lineage: Vec>> = models + .iter() + .map(|entry| model_lineage(&entry.model)) + .collect::>()?; + let keys: BTreeMap>> = + project_relation_keys(&project)?.into_iter().collect(); + json_grains(&mut model_rows, lineage, &keys); + } request.models = built_models(&rows, &request, model_rows)?; let request: EvaluateRequest = crate::bindings::_helpers::compile_delta::rules_rows::whole_project_request(request)?; @@ -521,6 +531,26 @@ fn built_models( built.map_err(|message| python_error.take().unwrap_or_else(|| value_error(message))) } +/// The model's lineage columns, or `None` when it has no kept lineage. +fn model_lineage(model: &Bound<'_, PyAny>) -> PyResult>> { + let Some(view) = native_model(model)? else { + return Ok(None); + }; + Ok(view.lineage_rows(model.py())?.map(|rows| { + rows.into_iter() + .map(|row| LineageColumn { + aggregated: row.aggregated(), + output: row.output_column, + sources: row + .sources + .into_iter() + .map(|(resource, name, _)| (resource, name)) + .collect(), + }) + .collect() + })) +} + fn declarations(rows: Vec>) -> PyResult> { rows.into_iter().map(declaration).collect() } diff --git a/crates/sqlbuild-rules/src/engine/main/json_grains.rs b/crates/sqlbuild-rules/src/engine/main/json_grains.rs new file mode 100644 index 0000000000..6e0b9c595f --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/json_grains.rs @@ -0,0 +1,123 @@ +//! SQBRPERFORMANCE102's grain facts on each model row, built only when the request selects it. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::models::{EvaluateRequest, GrainColumn, JsonGrain, LineageColumn, ModelRow}; +use crate::query_facts::constants::SNOWFLAKE_DIALECT; +use crate::rules::main::request_selects::request_selects; + +const RULE_CODE: &str = "SQBRPERFORMANCE102"; +const REFERENCE_KIND: &str = "ref"; +const SEMI_STRUCTURED_TYPES: [&str; 3] = ["VARIANT", "OBJECT", "ARRAY"]; +/// Lineage resource types and the relation-key kinds they are declared under. +const RESOURCE_KINDS: [(&str, &str); 3] = + [("model", "ref"), ("source", "source"), ("seed", "seed")]; +const NO_LINEAGE: &str = "no column lineage"; +const NO_TYPES: &str = "no inferred column types"; + +/// Whether `request` selects SQBRPERFORMANCE102 for a dialect it runs on. +pub fn needs_json_grains(request: &EvaluateRequest) -> Result { + Ok(request.dialect.eq_ignore_ascii_case(SNOWFLAKE_DIALECT) + && request_selects(request, |rule| rule.code == RULE_CODE)?) +} + +/// Set each keyed row's grain from its lineage (`None` when unknown), in row order, and `keys` by `kind:name`. +pub fn json_grains( + rows: &mut [ModelRow], + lineage: Vec>>, + keys: &BTreeMap>>, +) { + let mut readers: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + for row in rows.iter() { + for reference in &row.references { + if reference.ref_kind == REFERENCE_KIND { + readers + .entry(reference.ref_name.as_str()) + .or_default() + .insert(row.name.clone()); + } + } + } + let grains: Vec> = rows + .iter() + .zip(lineage) + .map(|(row, lineage)| { + let model_keys: &Vec> = + keys.get(&format!("{REFERENCE_KIND}:{}", row.name))?; + let readers: Vec = readers + .get(row.name.as_str()) + .map(|names| names.iter().cloned().collect()) + .unwrap_or_default(); + Some(grain(row, model_keys.clone(), readers, lineage, keys)) + }) + .collect(); + for (row, grain) in rows.iter_mut().zip(grains) { + row.json_grain = grain; + } +} + +fn grain( + row: &ModelRow, + model_keys: Vec>, + readers: Vec, + lineage: Option>, + keys: &BTreeMap>>, +) -> JsonGrain { + let mut grain: JsonGrain = JsonGrain { + model_keys, + readers, + ..JsonGrain::default() + }; + let Some(lineage) = lineage.filter(|columns| !columns.is_empty()) else { + grain.skipped = Some(NO_LINEAGE.to_owned()); + return grain; + }; + if row.inferred_columns.is_empty() { + grain.skipped = Some(NO_TYPES.to_owned()); + return grain; + } + for (name, data_type) in &row.inferred_columns { + let Some(data_type) = data_type.as_deref().filter(|kind| semi_structured(kind)) else { + continue; + }; + let Some(column) = lineage + .iter() + .find(|column| column.output.eq_ignore_ascii_case(name)) + .filter(|column| !column.aggregated) + else { + continue; + }; + let inputs: BTreeSet = column + .sources + .iter() + .filter_map(|(resource, name)| { + let kind: &str = RESOURCE_KINDS + .iter() + .find(|(known, _)| known == resource)? + .1; + Some(format!("{kind}:{name}")) + }) + .collect(); + for input in &inputs { + if let Some(found) = keys.get(input) { + grain.input_keys.insert(input.clone(), found.clone()); + } + } + grain.columns.push(GrainColumn { + name: name.clone(), + data_type: Some(data_type.to_owned()), + inputs: inputs.into_iter().collect(), + }); + } + grain +} + +/// `VARIANT`, `OBJECT` or `ARRAY`, alone or with parameters such as `ARRAY(INT)`. +fn semi_structured(data_type: &str) -> bool { + let upper: String = data_type.trim().to_ascii_uppercase(); + SEMI_STRUCTURED_TYPES.iter().any(|kind| { + upper + .strip_prefix(kind) + .is_some_and(|rest| rest.is_empty() || rest.starts_with(['(', '<', ' '])) + }) +} diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index 1237dba54e..e95f288186 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -4,6 +4,7 @@ pub mod custom_host; pub mod discard_speculative_rows; pub mod evaluate_rows; pub mod finalize_rows; +pub mod json_grains; pub mod project_model_rows; pub mod retained_request; pub mod start_speculative_rows; diff --git a/crates/sqlbuild-rules/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs index 3f069465db..b87687410b 100644 --- a/crates/sqlbuild-rules/src/engine/tests.rs +++ b/crates/sqlbuild-rules/src/engine/tests.rs @@ -8,6 +8,8 @@ mod evaluation; mod helpers; #[path = "tests/test_incremental_rules.rs"] mod incremental_rules; +#[path = "tests/test_json_grains.rs"] +mod json_grains; #[path = "tests/test_published_layers.rs"] mod published_layers; #[path = "tests/test_query_facts.rs"] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index a18c1cb885..15922fa5e8 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -645,6 +645,26 @@ pub(crate) fn query_facts_faults( request["models"][0]["enum_columns"] = json!(test_case.enum_columns); request["models"][0]["config"] = json!({"contract": "enforced"}); request["models"][0]["columns"] = test_case.columns.clone(); + evaluated_faults(&request) +} + +/// One model named `name` with `json_grain`, evaluated with `select` under `dialect`. +pub(crate) fn json_grain_faults( + test_case: &test_types::JsonGrainTestCase, +) -> Result, String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let config = json!({"select": test_case.select, "cache": {"enabled": false}}); + let mut request: Value = + serde_json::from_str(&request(&project_dir, &config)).map_err(|error| error.to_string())?; + request["dialect"] = json!(test_case.dialect); + request["models"][0]["name"] = json!(test_case.name); + request["models"][0]["query_sql"] = json!(test_case.query_sql); + request["models"][0]["authored_sql"] = json!(test_case.query_sql); + request["models"][0]["json_grain"] = test_case.json_grain.clone(); + evaluated_faults(&request) +} + +fn evaluated_faults(request: &Value) -> Result, String> { let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) .map_err(|error| error.to_string())?; let faults = result["faults"] @@ -745,7 +765,7 @@ pub(crate) fn every_field_model() -> Model { "ref:orders".to_owned(), vec![vec!["order_id".to_owned()]], )]), - backs_published_view: true, + readers: vec!["shop__mart_v__orders".to_owned()], skipped: Some("no lineage".to_owned()), }), } diff --git a/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs new file mode 100644 index 0000000000..d5174a463d --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs @@ -0,0 +1,263 @@ +use std::collections::BTreeMap; + +use serde_json::{Value, json}; + +use crate::engine::main::json_grains::json_grains; +use crate::engine::tests::helpers::json_grain_faults; +use crate::engine::tests::test_types::JsonGrainTestCase; +use crate::models::{GrainColumn, JsonGrain, LineageColumn, ModelRow, Reference}; + +const RULE: &[&str] = &["SQBRPERFORMANCE102"]; +const LINES_SQL: &str = "SELECT l.order_id, l.line_no, o.payload\nFROM __ref(\"order_lines\") l JOIN __ref(\"orders\") o USING (order_id)"; + +fn payload_grain(input_key: Value, readers: Value) -> Value { + json!({ + "model_keys": [["order_id", "line_no"]], + "columns": [{"name": "payload", "data_type": "VARIANT", "inputs": ["ref:orders"]}], + "input_keys": {"ref:orders": [input_key]}, + "readers": readers, + }) +} + +#[test] +fn given_grain_facts_when_evaluating_json_finer_grain_then_only_coarser_inputs_are_flagged() +-> Result<(), String> { + let test_cases = [ + JsonGrainTestCase { + description: "a payload keyed by order copied onto order lines is flagged at its column", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["order_id"]), json!([])), + expected_faults: &[( + "SQBRPERFORMANCE102", + 1, + 31, + "payload copies VARIANT from orders, one row per (order_id), onto rows keyed by (order_id, line_no)", + )], + }, + JsonGrainTestCase { + description: "key columns match whatever their case and order", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: json!({ + "model_keys": [["LINE_NO", "Order_Id"]], + "columns": [{"name": "PAYLOAD", "data_type": "VARIANT", "inputs": ["ref:orders"]}], + "input_keys": {"ref:orders": [["ORDER_ID"]]}, + }), + expected_faults: &[("SQBRPERFORMANCE102", 1, 31, "PAYLOAD copies VARIANT")], + }, + JsonGrainTestCase { + description: "an input on the model's own grain repeats nothing", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["line_no", "order_id"]), json!([])), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "an input keyed by another column is not coarser", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["customer_id"]), json!([])), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "a model whose only reader is the published view of its own entity is exempt", + dialect: "snowflake", + name: "shop__mart__orders", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["order_id"]), json!(["shop__mart_v__orders"])), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "a published model that another model also reads is not exempt", + dialect: "snowflake", + name: "shop__mart__orders", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain( + json!(["order_id"]), + json!(["shop__int_enriched__orders", "shop__mart_v__orders"]), + ), + expected_faults: &[("SQBRPERFORMANCE102", 1, 31, "payload copies VARIANT")], + }, + JsonGrainTestCase { + description: "a view of another entity does not exempt the model", + dialect: "snowflake", + name: "shop__mart__orders", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["order_id"]), json!(["shop__mart_v__customers"])), + expected_faults: &[("SQBRPERFORMANCE102", 1, 31, "payload copies VARIANT")], + }, + JsonGrainTestCase { + description: "a model without lineage is skipped, not flagged", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: json!({"model_keys": [["order_id", "line_no"]], "skipped": "no column lineage"}), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "the rule runs only on Snowflake", + dialect: "duckdb", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["order_id"]), json!([])), + expected_faults: &[], + }, + ]; + for test_case in test_cases { + let faults = json_grain_faults(&test_case)?; + let observed: Vec<(&str, u64, u64, bool)> = faults + .iter() + .map(|(code, line, column, _)| (code.as_str(), *line, *column, true)) + .collect(); + let expected: Vec<(&str, u64, u64, bool)> = test_case + .expected_faults + .iter() + .map(|(code, line, column, _)| (*code, *line, *column, true)) + .collect(); + assert_eq!(observed, expected, "{}", test_case.description); + for ((_, _, _, message), (_, _, _, fragment)) in + faults.iter().zip(test_case.expected_faults) + { + assert!( + message.contains(fragment), + "{}: {message}", + test_case.description + ); + } + } + Ok(()) +} + +fn row(name: &str, references: &[&str], inferred: &[(&str, &str)]) -> ModelRow { + ModelRow { + name: name.to_owned(), + references: references + .iter() + .map(|name| Reference { + ref_kind: "ref".to_owned(), + ref_name: (*name).to_owned(), + ref_package: None, + }) + .collect(), + inferred_columns: inferred + .iter() + .map(|(name, kind)| ((*name).to_owned(), Some((*kind).to_owned()))) + .collect(), + ..ModelRow::default() + } +} + +fn lineage(output: &str, sources: &[(&str, &str)], aggregated: bool) -> LineageColumn { + LineageColumn { + output: output.to_owned(), + sources: sources + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(), + aggregated, + } +} + +fn keys(key: &[&str]) -> Vec> { + vec![key.iter().map(|column| (*column).to_owned()).collect()] +} + +#[test] +fn given_rows_lineage_and_keys_when_building_grains_then_only_copied_semi_structured_columns_are_kept() + { + let mut rows = vec![ + row( + "order_lines_enriched", + &["orders", "order_lines"], + &[ + ("payload", "VARIANT"), + ("items", "ARRAY(VARCHAR)"), + ("region", "object"), + ("total", "NUMBER(38,0)"), + ("payloads", "ARRAY"), + ("arrayish", "ARRAYLIKE"), + ], + ), + row("orders", &[], &[("payload", "VARIANT")]), + row("unkeyed", &["orders"], &[("payload", "VARIANT")]), + row("untyped", &[], &[]), + ]; + let lineage = vec![ + Some(vec![ + lineage( + "PAYLOAD", + &[("model", "orders"), ("table_fn", "explode")], + false, + ), + lineage("items", &[("source", "shop.raw_orders")], false), + lineage("region", &[("seed", "countries")], false), + lineage("total", &[("model", "orders")], false), + lineage("payloads", &[("model", "orders")], true), + lineage("arrayish", &[("model", "orders")], false), + ]), + None, + Some(vec![lineage("payload", &[("model", "orders")], false)]), + Some(vec![lineage("payload", &[("model", "orders")], false)]), + ]; + let declared: BTreeMap>> = BTreeMap::from([ + ( + "ref:order_lines_enriched".to_owned(), + keys(&["order_id", "line_no"]), + ), + ("ref:orders".to_owned(), keys(&["order_id"])), + ("ref:untyped".to_owned(), keys(&["id"])), + ("seed:countries".to_owned(), keys(&["country_code"])), + ]); + json_grains(&mut rows, lineage, &declared); + let column = |name: &str, data_type: &str, inputs: &[&str]| GrainColumn { + name: name.to_owned(), + data_type: Some(data_type.to_owned()), + inputs: inputs.iter().map(|input| (*input).to_owned()).collect(), + }; + let grains: Vec> = rows.into_iter().map(|row| row.json_grain).collect(); + assert_eq!( + grains, + vec![ + Some(JsonGrain { + model_keys: keys(&["order_id", "line_no"]), + columns: vec![ + column("payload", "VARIANT", &["ref:orders"]), + column("items", "ARRAY(VARCHAR)", &["source:shop.raw_orders"]), + column("region", "object", &["seed:countries"]), + ], + input_keys: BTreeMap::from([ + ("ref:orders".to_owned(), keys(&["order_id"])), + ("seed:countries".to_owned(), keys(&["country_code"])), + ]), + readers: Vec::new(), + skipped: None, + }), + Some(JsonGrain { + model_keys: keys(&["order_id"]), + readers: vec!["order_lines_enriched".to_owned(), "unkeyed".to_owned()], + skipped: Some("no column lineage".to_owned()), + ..JsonGrain::default() + }), + None, + Some(JsonGrain { + model_keys: keys(&["id"]), + skipped: Some("no inferred column types".to_owned()), + ..JsonGrain::default() + }), + ] + ); +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index 7bd4f88f21..147bd10c48 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -282,6 +282,16 @@ pub(crate) struct QueryFactsTestCase { pub(crate) expected_faults: &'static [(&'static str, u64, u64, &'static str)], } +pub(crate) struct JsonGrainTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) name: &'static str, + pub(crate) select: &'static [&'static str], + pub(crate) query_sql: &'static str, + pub(crate) json_grain: Value, + pub(crate) expected_faults: &'static [(&'static str, u64, u64, &'static str)], +} + /// A retained request evaluated again without SQL, with or without the cached findings. pub(crate) struct RetainedRowsTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index cd1b784cd5..bd71cf5cbe 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -621,12 +621,20 @@ pub struct JsonGrain { pub columns: Vec, /// Each lineage input's keys by `kind:name`, such as `ref:orders`. pub input_keys: BTreeMap>>, - /// Whether every reader is a published view of this model's own entity, as SQBRGRAPH103 defines one. - pub backs_published_view: bool, + /// The models that read this one. + pub readers: Vec, /// Why the comparison cannot run, such as missing lineage; the model is then skipped, not flagged. pub skipped: Option, } +/// One lineage output column: the `(resource type, name)` relations it reads and whether it aggregates them. +#[derive(Clone, Debug, Default)] +pub struct LineageColumn { + pub output: String, + pub sources: Vec<(String, String)>, + pub aggregated: bool, +} + /// One output column's inferred type and the relations its lineage reads, by `kind:name`. #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] #[serde(default, deny_unknown_fields)] diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index 22ab524809..5deb14cb8c 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -224,7 +224,7 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result, rule: &RuleMetadata, faults: &FaultCollector) { +pub(crate) fn evaluate( + parsed: &ParsedModel<'_>, + config: &RulesConfig, + rule: &RuleMetadata, + faults: &FaultCollector, +) { let Some(grain) = &parsed.model.json_grain else { return; }; - if grain.skipped.is_some() || grain.backs_published_view { + if grain.skipped.is_some() || backs_published_view(&parsed.model.name, &grain.readers, config) { return; } for column in &grain.columns { - if !column.data_type.as_deref().is_some_and(|data_type| { - SEMI_STRUCTURED_TYPES - .iter() - .any(|kind| starts_with_word(data_type, kind)) - }) { - continue; - } if let Some((input, input_key, model_key)) = coarser_input(column, grain) { + let input: &str = input.split_once(':').map_or(input, |(_, name)| name); let (line, column_number) = output_start(&parsed.facts.body, &column.name) .map_or((1, 1), |offset| parsed.source.line_column(offset)); faults.push(Fault { @@ -45,14 +43,6 @@ pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &F } } -fn starts_with_word(data_type: &str, kind: &str) -> bool { - let upper: String = data_type.trim().to_ascii_uppercase(); - upper == kind - || upper - .strip_prefix(kind) - .is_some_and(|rest| rest.starts_with(['(', '<', ' '])) -} - /// An input of `column` whose key is a strict subset of one of the model's keys. fn coarser_input<'a>( column: &'a GrainColumn, diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs index 1f1b11ff0c..c6b6addfaf 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs @@ -19,6 +19,8 @@ const INTERMEDIATE_FOLDER: &str = "intermediate/enriched"; const ENTITY_SHAPE: &str = "____"; const QUALIFIED_SHAPE: &str = "______"; const MODEL_FILE_SUFFIX: &str = ".sql"; +/// The rule whose published layers and view-of-own-entity test define a published view. +const PUBLISHED_READS_RULE: &str = "SQBRGRAPH103"; pub(crate) fn explicit_model_schema(model: &Model, rule: &RuleMetadata, faults: &FaultCollector) { if model @@ -303,6 +305,22 @@ pub(crate) fn same_domain_published_reads( } } +/// Whether `readers` is not empty and each one is a published view of `model`'s own entity. +pub(crate) fn backs_published_view(model: &str, readers: &[String], config: &RulesConfig) -> bool { + let layers: Vec = published_layers(config, PUBLISHED_READS_RULE); + let Some(producer) = published_name(model).filter(|name| layers.contains(&name.layer)) else { + return false; + }; + !readers.is_empty() + && readers.iter().all(|reader| { + published_name(reader).is_some_and(|consumer| { + consumer.domain == producer.domain + && layers.contains(&consumer.layer) + && is_view_of_own_entity(&consumer, &producer) + }) + }) +} + /// `X___v__[__]` reading `X____` or the same qualifier. fn is_view_of_own_entity(consumer: &PublishedName, producer: &PublishedName) -> bool { consumer.is_view From 007ffd2561628588ffebb885201a6e0e2501bbcc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 06:14:55 +0000 Subject: [PATCH 05/17] wip: SQBRPERFORMANCE102 skipped-model note, DuckShop fixture, docs and integration test --- .../performance_rules/expected_findings.json | 7 ++ .../performance_rules/models/line_options.sql | 9 ++ .../models/line_options_trimmed.sql | 9 ++ .../models/order_exports.sql | 9 ++ .../performance_rules/models/order_lines.sql | 12 ++ .../models/order_lines_customer.sql | 13 +++ .../models/order_lines_enriched.sql | 14 +++ .../performance_rules/models/order_skus.sql | 12 ++ .../performance_rules/models/orders.sql | 10 ++ .../models/shop__mart__order_lines.sql | 13 +++ .../models/shop__mart_v__order_lines.sql | 7 ++ .../performance_rules/sqlbuild_project.toml | 9 ++ .../_helpers/compiler_records/constants.rs | 1 + .../_helpers/rule_records/constants.rs | 3 + .../src/records/constants.rs | 1 + .../_helpers/build_command/planning.rs | 1 + .../_helpers/compile_driver/analysis.rs | 4 + .../_helpers/compile_driver/command.rs | 7 ++ .../_helpers/compile_driver/plan_pipeline.rs | 8 ++ .../bindings/_helpers/plan_command/plan.rs | 3 +- .../bindings/_helpers/plan_command/rules.rs | 7 ++ .../src/bindings/_helpers/rules/constants.rs | 2 +- .../src/bindings/_helpers/rules/rows.rs | 4 +- .../bindings/_helpers/rules/rule_dispatch.rs | 1 + .../src/bindings/_helpers/rules/rule_run.rs | 1 + .../src/engine/main/json_grains.rs | 14 ++- .../main/format_skipped_models.rs | 21 ++++ .../src/rule_results/main/mod.rs | 1 + src/sqlbuild/_native.pyi | 6 + .../sqlbuild/cli/commands/main/_test_types.py | 9 ++ .../sqlbuild/cli/commands/main/test_rules.py | 57 +++++++++ .../src/content/docs/docs/concepts/rules.mdx | 7 +- .../docs/concepts/rules/performance-rules.mdx | 108 ++++++++++++++++++ website/src/sidebar.json | 1 + 34 files changed, 386 insertions(+), 5 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/expected_findings.json create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/line_options.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/order_exports.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/order_lines.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_customer.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_enriched.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/order_skus.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/orders.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart__order_lines.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart_v__order_lines.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/sqlbuild_project.toml create mode 100644 crates/sqlbuild-rules/src/rule_results/main/format_skipped_models.rs create mode 100644 website/src/content/docs/docs/concepts/rules/performance-rules.mdx diff --git a/benchmarks/duckshop/edge_cases/performance_rules/expected_findings.json b/benchmarks/duckshop/edge_cases/performance_rules/expected_findings.json new file mode 100644 index 0000000000..1370a39d58 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/expected_findings.json @@ -0,0 +1,7 @@ +{ + "findings": [ + ["SQBRPERFORMANCE101", "models/line_options.sql", 7, 3], + ["SQBRPERFORMANCE102", "models/order_lines_enriched.sql", 12, 3] + ], + "skipped_note": "SQBRPERFORMANCE102 skipped 1 model without column lineage or inferred types" +} diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/line_options.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options.sql new file mode 100644 index 0000000000..6b4d05bf54 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options.sql @@ -0,0 +1,9 @@ +MODEL ( + description "One row per line option with the whole parent line copied onto it.", +); + +SELECT + r.order_id, + o.value AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql new file mode 100644 index 0000000000..88784b8e5b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql @@ -0,0 +1,9 @@ +MODEL ( + description "One row per line option with the parent line minus its options.", +); + +SELECT + r.order_id, + OBJECT_DELETE(o.value, 'options') AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_exports.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_exports.sql new file mode 100644 index 0000000000..9eb899172a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_exports.sql @@ -0,0 +1,9 @@ +MODEL ( + description "Orders read from an export table SQLBuild cannot see.", + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT * +FROM RAW.SHOP.ORDER_EXPORTS diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines.sql new file mode 100644 index 0000000000..b8ce938855 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines.sql @@ -0,0 +1,12 @@ +MODEL ( + description "One row per order line.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, line_no], +); + +SELECT + o.order_id, + l.index AS line_no, + l.value:sku::string AS sku +FROM __ref("orders") o, LATERAL FLATTEN(input => o.payload:lines) l diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_customer.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_customer.sql new file mode 100644 index 0000000000..91b2d4e96f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_customer.sql @@ -0,0 +1,13 @@ +MODEL ( + description "Order lines with one field read from the order payload.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, line_no], +); + +SELECT + l.order_id, + l.line_no, + o.payload:customer::string AS customer_id +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_enriched.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_enriched.sql new file mode 100644 index 0000000000..34cd474d92 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_enriched.sql @@ -0,0 +1,14 @@ +MODEL ( + description "Order lines with the whole order payload copied onto every line.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, line_no], +); + +SELECT + l.order_id, + l.line_no, + l.sku, + o.payload AS order_payload +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_skus.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_skus.sql new file mode 100644 index 0000000000..5ddf490256 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_skus.sql @@ -0,0 +1,12 @@ +MODEL ( + description "One row per order with its line SKUs aggregated.", + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT + l.order_id, + ARRAY_AGG(l.sku) AS skus +FROM __ref("order_lines") l +GROUP BY l.order_id diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/orders.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/orders.sql new file mode 100644 index 0000000000..0ec4614b75 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/orders.sql @@ -0,0 +1,10 @@ +MODEL ( + description "One row per order with its raw payload.", + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT + 1 AS order_id, + PARSE_JSON('{"customer": "c1", "lines": [{"sku": "a", "options": [{"code": "gift"}]}]}') AS payload diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart__order_lines.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart__order_lines.sql new file mode 100644 index 0000000000..8388022195 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart__order_lines.sql @@ -0,0 +1,13 @@ +MODEL ( + description "Published order lines whose only reader is their own published view.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, line_no], +); + +SELECT + l.order_id, + l.line_no, + o.payload AS order_payload +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart_v__order_lines.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart_v__order_lines.sql new file mode 100644 index 0000000000..0adface38c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart_v__order_lines.sql @@ -0,0 +1,7 @@ +MODEL ( + description "The published view of order lines.", + materialized view, +); + +SELECT order_id, line_no, order_payload +FROM __ref("shop__mart__order_lines") diff --git a/benchmarks/duckshop/edge_cases/performance_rules/sqlbuild_project.toml b/benchmarks/duckshop/edge_cases/performance_rules/sqlbuild_project.toml new file mode 100644 index 0000000000..b5993670d5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/sqlbuild_project.toml @@ -0,0 +1,9 @@ +name = "orders" +adapter = "snowflake" + +[connection] +database = "ANALYTICS" +schema = "ORDERS" + +[rules] +select = ["SQBRPERFORMANCE101", "SQBRPERFORMANCE102"] diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index 42289b766d..8f2aff3869 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -394,6 +394,7 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { FieldSpec::new("external_sql_reference_resolver", FieldDefault::None), FieldSpec::new("resolve_python_run_selectors", FieldDefault::False), FieldSpec::new("no_cache", FieldDefault::False), + FieldSpec::new("verbose", FieldDefault::False), FieldSpec::new("max_microbatches", FieldDefault::None), FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("selection_diagnostics", FieldDefault::False), diff --git a/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs index b0856b5a54..989133a1cc 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs @@ -108,6 +108,7 @@ const R_RULES_RUN_RESULT: RecordSpec = RecordSpec { FieldSpec::new("cache_misses", FieldDefault::Int(0)), FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), + FieldSpec::new("skipped_models", FieldDefault::EmptyTuple), ], }; const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { @@ -121,6 +122,7 @@ const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { FieldSpec::required("cache_misses"), FieldSpec::required("built_in_ms"), FieldSpec::required("reused"), + FieldSpec::new("skipped_models", FieldDefault::EmptyTuple), ], }; const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { @@ -298,6 +300,7 @@ const R_RULES_RESULT: RecordSpec = RecordSpec { FieldSpec::new("custom_ms", FieldDefault::Int(0)), FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), + FieldSpec::new("skipped_models", FieldDefault::EmptyTuple), ], }; const R_RULE_FILE: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index 402dde4195..32bafbeb97 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -1080,6 +1080,7 @@ const COMPILE_ANALYSIS: RecordSpec = RecordSpec { FieldSpec::new("rule_cache_misses", FieldDefault::Int(0)), FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), + FieldSpec::new("skipped_models", FieldDefault::EmptyTuple), ], }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 1b9feaae1f..54e4aaeb8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -54,6 +54,7 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult( "custom_rule_timings", native_field(&rules_result, "custom_rule_timings")?, ), + ( + "skipped_models", + native_field(&rules_result, "skipped_models")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index bd7b0cb71f..6b85f69260 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -49,6 +49,7 @@ use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice; +use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models; const BYPASS_OUTCOME: &str = "bypass"; @@ -429,6 +430,12 @@ fn compiled_with_status<'py>( ) { output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?; } + if let Some(skipped_note) = format_skipped_models( + &native_field(&analysis, "skipped_models")?.extract::>()?, + false, + ) { + output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?; + } let dag_artifact_path: Bound<'py, PyAny> = if rules_failed || native_field(request, "dag_path")?.is_none() { py.None().into_bound(py) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index ecafc6ed93..15b6f23bf6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -6,6 +6,7 @@ use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice; +use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline::{ @@ -138,6 +139,13 @@ fn run_rules_gate<'py>( let elapsed: f64 = monotonic(py)? - started; let message: String = rules_progress(elapsed, &result, true)?; on_progress.call1((message,))?; + let verbose: bool = native_field(options, "verbose")?.is_truthy()?; + if let Some(note) = format_skipped_models( + &native_field(&result, "skipped_models")?.extract::>()?, + verbose, + ) { + on_progress.call1((format!("note: {note}"),))?; + } } let mut failing: Vec> = Vec::new(); for finding in native_field(&result, "findings")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 0c4c04e73b..34c25cd8ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -44,9 +44,10 @@ const DEFAULT_SECTION_ENTRIES: i64 = 50; const ERROR_SEVERITY: &str = "error"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; /// Request fields passed to the compile pipeline under the same name. -const PIPELINE_FIELDS: [&str; 11] = [ +const PIPELINE_FIELDS: [&str; 12] = [ "no_sql_validation", "no_cache", + "verbose", "defer_to", "defer_sources_to", "cursor_overrides", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index f528acb224..7098f758e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -28,6 +28,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models; const LIST_ACTION: &str = "list"; const SHOW_ACTION: &str = "show"; @@ -238,6 +239,12 @@ fn run_rule_selection( ) { print_line(py, &format!("note: {note}"), true)?; } + if let Some(note) = format_skipped_models( + &native_field(&result, "skipped_models")?.extract::>()?, + false, + ) { + print_line(py, &format!("note: {note}"), true)?; + } let fields: Bound<'_, PyDict> = PyDict::new(py); for field in RESULT_FIELDS { fields.set_item(field, native_field(&result, field)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index 56a557093d..14f0b42a54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -25,7 +25,7 @@ pub(crate) const CUSTOM_HOST_ABANDONED_SUFFIX: &str = ".abandoned"; /// Marker file whose presence tells split custom hosts the run was cancelled. pub(crate) const CUSTOM_HOST_CANCELLED_MARKER: &str = "cancelled"; /// Rules that need proven column types and are skipped without SQL analysis. -pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 1] = ["SQBRCONTRACT105"]; +pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 2] = ["SQBRCONTRACT105", "SQBRPERFORMANCE102"]; /// Runtime identity custom-rule hosts and their cache entries record. pub(crate) const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v3"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index b857d982af..8e0cdfb840 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -18,7 +18,7 @@ use sqlbuild_rules::constants::API_VERSION; use sqlbuild_rules::engine::main::build_models::build_models; use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; use sqlbuild_rules::engine::main::finalize_rows::finalize_rows; -use sqlbuild_rules::engine::main::json_grains::{json_grains, needs_json_grains}; +use sqlbuild_rules::engine::main::json_grains::{json_grains, needs_json_grains, skipped_models}; use sqlbuild_rules::engine::main::project_model_rows::project_model_rows; use sqlbuild_rules::errors::RowsError; use sqlbuild_rules::models::{ @@ -374,6 +374,7 @@ fn evaluated<'py>( early: Option<(EarlyBuiltinRules, &AtomicBool)>, ) -> PyResult> { let request: RowsRequest = taken_rows(request)?; + let skipped: Vec<(String, String)> = skipped_models(&request.request.models); let evaluation: Result = match early { None => py .compiler_detach(|| Ok(evaluate_rows(request))) @@ -416,6 +417,7 @@ fn evaluated<'py>( ), ("built_in_ms", evaluation.built_in_ms.into_bound_py_any(py)?), ("reused", evaluation.reused.into_bound_py_any(py)?), + ("skipped_models", PyTuple::new(py, skipped)?.into_any()), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 73856f15cd..8696778a65 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -555,6 +555,7 @@ fn rules_result<'py>( "custom_rule_timings", field(&custom, "custom_rule_timings")?, ), + ("skipped_models", field(&native_result, "skipped_models")?), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index a87a214f5e..390b37eb47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -137,6 +137,7 @@ pub(crate) fn finish_rule_run<'py>( "skipped_type_proof_rules", record_field(plan, "skipped_type_proof_rules")?, ), + ("skipped_models", record_field(result, "skipped_models")?), ], ) } diff --git a/crates/sqlbuild-rules/src/engine/main/json_grains.rs b/crates/sqlbuild-rules/src/engine/main/json_grains.rs index 6e0b9c595f..da2a03c3d9 100644 --- a/crates/sqlbuild-rules/src/engine/main/json_grains.rs +++ b/crates/sqlbuild-rules/src/engine/main/json_grains.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::models::{EvaluateRequest, GrainColumn, JsonGrain, LineageColumn, ModelRow}; +use crate::models::{EvaluateRequest, GrainColumn, JsonGrain, LineageColumn, Model, ModelRow}; use crate::query_facts::constants::SNOWFLAKE_DIALECT; use crate::rules::main::request_selects::request_selects; @@ -56,6 +56,18 @@ pub fn json_grains( } } +/// `(path, reason)` of each analysed model SQBRPERFORMANCE102 skipped for missing facts. +pub fn skipped_models(models: &[Model]) -> Vec<(String, String)> { + models + .iter() + .filter(|model| !model.sql_analysis_disabled) + .filter_map(|model| { + let reason: &String = model.json_grain.as_ref()?.skipped.as_ref()?; + Some((model.relative_path.clone(), reason.clone())) + }) + .collect() +} + fn grain( row: &ModelRow, model_keys: Vec>, diff --git a/crates/sqlbuild-rules/src/rule_results/main/format_skipped_models.rs b/crates/sqlbuild-rules/src/rule_results/main/format_skipped_models.rs new file mode 100644 index 0000000000..ab43ce9f51 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/format_skipped_models.rs @@ -0,0 +1,21 @@ +//! The note counting the models SQBRPERFORMANCE102 skipped, listing them when verbose. + +/// The note for `skipped` `(path, reason)` pairs, or `None` when no model was skipped. +pub fn format_skipped_models(skipped: &[(String, String)], verbose: bool) -> Option { + if skipped.is_empty() { + return None; + } + let mut note: String = format!( + "SQBRPERFORMANCE102 skipped {} model{} without column lineage or inferred types", + skipped.len(), + if skipped.len() == 1 { "" } else { "s" } + ); + if verbose { + for (path, reason) in skipped { + note.push_str(&format!("\n {path}: {reason}")); + } + } else { + note.push_str("; `sqb plan --verbose` lists them"); + } + Some(note) +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/mod.rs b/crates/sqlbuild-rules/src/rule_results/main/mod.rs index 3cb9c4ec7c..17da60c2f7 100644 --- a/crates/sqlbuild-rules/src/rule_results/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_results/main/mod.rs @@ -2,5 +2,6 @@ pub mod default_rules_notice; pub mod duplicate_literal_hints; pub mod format_result; pub mod format_rules_progress; +pub mod format_skipped_models; pub mod format_skipped_rules; pub mod group_unevaluated; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 664656b426..9f0fe7bc80 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -10195,6 +10195,7 @@ class CompilePipelineOptions(NativeRecord): external_sql_reference_resolver: ExternalSqlReferenceResolver | None resolve_python_run_selectors: bool no_cache: bool + verbose: bool max_microbatches: int | None concurrency: int | None selection_diagnostics: bool @@ -10221,6 +10222,7 @@ class CompilePipelineOptions(NativeRecord): external_sql_reference_resolver: ExternalSqlReferenceResolver | None = ..., resolve_python_run_selectors: bool = ..., no_cache: bool = ..., + verbose: bool = ..., max_microbatches: int | None = ..., concurrency: int | None = ..., selection_diagnostics: bool = ..., @@ -10960,6 +10962,7 @@ class RulesRunResult(NativeRecord): cache_misses: int skipped_type_proof_rules: tuple[str, ...] custom_rule_timings: tuple[tuple[str, int], ...] + skipped_models: tuple[tuple[str, str], ...] def __init__( self, findings: tuple[Finding, ...], @@ -10972,6 +10975,7 @@ class RulesRunResult(NativeRecord): cache_misses: int = ..., skipped_type_proof_rules: tuple[str, ...] = ..., custom_rule_timings: tuple[tuple[str, int], ...] = ..., + skipped_models: tuple[tuple[str, str], ...] = ..., ) -> None: ... class CustomRulesOutcome(NativeRecord): @@ -16115,6 +16119,7 @@ class RulesResult(NativeRecord): custom_ms: int custom_cpu_ms: int custom_rule_timings: tuple[tuple[str, int], ...] + skipped_models: tuple[tuple[str, str], ...] def __init__( self, findings: tuple[Finding, ...], @@ -16126,6 +16131,7 @@ class RulesResult(NativeRecord): custom_ms: int = ..., custom_cpu_ms: int = ..., custom_rule_timings: tuple[tuple[str, int], ...] = ..., + skipped_models: tuple[tuple[str, str], ...] = ..., ) -> None: ... class RuleFile(NativeRecord): diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 0c995342a5..e2191ebab7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1175,6 +1175,15 @@ class SqlRuleCacheSequenceTestCase: expected_hits_and_misses: tuple[tuple[int, int], ...] +@dataclass(frozen=True) +class PerformanceRulesIntegrationTestCase: + """One compile of the DuckShop performance-rules fixture, optionally with one model added.""" + + description: str + extra_models: dict[str, str] + expected_findings: tuple[tuple[str, str, int, int], ...] + + @dataclass(frozen=True) class NamedSelectorIntegrationTestCase: """One compile of the DuckShop named-selector fixture under a `[rules]` table.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d243810b9e..c446adfa85 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import shutil import sys import time from collections.abc import Callable @@ -35,6 +36,7 @@ ExplicitContractOutputRuleIntegrationTestCase, ImplicitAliasRuleIntegrationTestCase, NamedSelectorIntegrationTestCase, + PerformanceRulesIntegrationTestCase, NumericRangeDecisionIntegrationTestCase, ProjectRuleScratchStateTestCase, PythonRuleCacheSequenceTestCase, @@ -3692,3 +3694,58 @@ def test_given_orchestration_selections_when_compiling_then_named_selector_limit == test_case.expected_findings ) assert test_case.expected_text in json.dumps(diagnostics) + + +_PERFORMANCE_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/performance_rules") +_PERFORMANCE_EXPECTED: dict[str, Any] = json.loads( + (_PERFORMANCE_FIXTURE / "expected_findings.json").read_text(encoding="utf-8") +) +_PERFORMANCE_FINDINGS: tuple[tuple[str, str, int, int], ...] = tuple( + (code, path, line, column) for code, path, line, column in _PERFORMANCE_EXPECTED["findings"] +) + + +@pytest.mark.parametrize( + "test_case", + [ + PerformanceRulesIntegrationTestCase( + description="copied payloads are reported and the rest of the fixture passes", + extra_models={}, + expected_findings=_PERFORMANCE_FINDINGS, + ), + PerformanceRulesIntegrationTestCase( + description="a second reader ends the published-view exemption", + extra_models={ + "models/order_line_counts.sql": ( + 'MODEL (description "Lines per order.");\n' + "SELECT order_id, COUNT(*) AS line_count\n" + 'FROM __ref("shop__mart__order_lines")\nGROUP BY order_id\n' + ), + }, + expected_findings=( + *_PERFORMANCE_FINDINGS, + ("SQBRPERFORMANCE102", "models/shop__mart__order_lines.sql", 11, 3), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_json_copies_when_compiling_then_performance_rules_report_them( + test_case: PerformanceRulesIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Both performance Rules run end to end; models without lineage are counted, not reported.""" + shutil.copytree(_PERFORMANCE_FIXTURE, tmp_path, dirs_exist_ok=True) + for relative_path, sql in test_case.extra_models.items(): + (tmp_path / relative_path).write_text(sql, encoding="utf-8") + + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + captured: CaptureResult[str] = capsys.readouterr() + diagnostics: list[dict[str, Any]] = json.loads(captured.out)["diagnostics"] + + assert exit_code == 1 + assert tuple( + sorted((item["code"], item["path"], item["line"], item["column"]) for item in diagnostics) + ) == tuple(sorted(test_case.expected_findings)) + assert _PERFORMANCE_EXPECTED["skipped_note"] in captured.err diff --git a/website/src/content/docs/docs/concepts/rules.mdx b/website/src/content/docs/docs/concepts/rules.mdx index c590d7e61e..5509d3733d 100644 --- a/website/src/content/docs/docs/concepts/rules.mdx +++ b/website/src/content/docs/docs/concepts/rules.mdx @@ -152,7 +152,9 @@ with `SQBR`. The `SQBRPYTHON` Rules check the project's Python: annotations, imp shape and the layout of `python/`. They run only when `select` names `SQBRPYTHON` or one of its codes; see [Python Rules](/docs/concepts/rules/python-rules). `SQBRORCHESTRATION101` keeps job and sensor selections from listing nodes one by one; see -[Orchestration Rules](/docs/concepts/rules/orchestration-rules). +[Orchestration Rules](/docs/concepts/rules/orchestration-rules). The `SQBRPERFORMANCE` Rules find +Snowflake JSON payloads that repeat on every row; they run only when `select` names their codes. See +[Performance Rules](/docs/concepts/rules/performance-rules). Projects can define custom Python Rules under `rules/**/*.py`. Their codes begin with `XSQBR`, for example `XSQBRARCH001`. Custom Rules use the same selection, findings, exceptions, execution order, @@ -248,6 +250,9 @@ Rules add about 14% to a cold compile and nothing to a warm one. See Keep each domain's published layer free of consumer-specific variants. + + Catch Snowflake JSON payloads copied onto every row of a finer grain. + Learn evaluation order, focused execution, and dependency-aware reuse. diff --git a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx new file mode 100644 index 0000000000..bfc4db363a --- /dev/null +++ b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx @@ -0,0 +1,108 @@ +--- +title: "Performance Rules" +description: "Catch semi-structured payloads that repeat on every row and multiply storage and scan cost." +--- + +The `SQBRPERFORMANCE` Rules find JSON payloads copied onto rows that are finer than the payload +itself. Each copy stores and scans the same value again, so the cost grows with the row count. +Name their codes in `select` to turn them on: + +```toml +[rules] +select = ["SQBRPERFORMANCE101", "SQBRPERFORMANCE102"] +``` + +Both Rules are off by default: selecting the `SQBRPERFORMANCE` family alone does not run them. They +run on the Snowflake dialect. + +| Code | Name | Reports | +|---|---|---| +| `SQBRPERFORMANCE101` | `flatten-parent-copy` | A FLATTEN element copied whole onto the rows of a deeper FLATTEN of its own child array | +| `SQBRPERFORMANCE102` | `json-finer-grain` | A `VARIANT`, `OBJECT` or `ARRAY` column copied from an input with a coarser unique key | + +## SQBRPERFORMANCE101: a flattened parent copied onto its children + +An output column must not copy the `value` of a FLATTEN whose child array a deeper FLATTEN in the +same query reads. Every child row then carries its whole parent, children included: + +```sql +SELECT + r.order_id, + o.value AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l +``` + +```text +error[SQBRPERFORMANCE101]: line_json copies o.value whole onto every row flattened from o.value:options + --> models/line_options.sql:7:3 +``` + +Read the fields you need from the parent, such as `o.value:sku::string`, or remove the child array +before copying the rest: + +```sql +SELECT + r.order_id, + OBJECT_DELETE(o.value, 'options') AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l +``` + +## SQBRPERFORMANCE102: a payload copied onto a finer grain + +A `VARIANT`, `OBJECT` or `ARRAY` output column must not come from an input whose unique key is a +strict subset of the model's unique key. Here `orders` has one row per `order_id` and the model has +one row per `(order_id, line_no)`, so each order's payload repeats on every line: + +```sql +SELECT + l.order_id, + l.line_no, + l.sku, + o.payload AS order_payload +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id +``` + +```text +error[SQBRPERFORMANCE102]: order_payload copies VARIANT from orders, one row per (order_id), onto rows keyed by (order_id, line_no), so the same value repeats on every row + --> models/order_lines_enriched.sql:12:3 +``` + +Read the fields the model needs, such as `o.payload:customer::string`, or leave the payload on the +input and join it where it is used. + +The Rule compares the unique keys SQLBuild knows from each relation's declaration: + +| Declaration | Key | +|---|---| +| A column with `nullable false` (or a `not_null` audit) and a `unique` audit | That column | +| An incremental model with `unique_key` and the `merge` strategy | The `unique_key` columns | +| A seed whose file holds at most one data row | No columns | + +Aggregated columns, such as `ARRAY_AGG(l.sku)`, are not copies and are not reported. + +### Published views + +A model whose only readers are published views of its own entity is not reported: the view +publishes the same grain on purpose. The views follow the +[SQBRGRAPH103 allowance](/docs/concepts/rules/published-layer-rules#sqbrgraph103-no-published-reads-within-a-domain): +`shop__mart_v__order_lines` reading `shop__mart__order_lines`. Its `published_layers` option sets +which layers count. + +### Skipped models + +`SQBRPERFORMANCE102` needs column lineage and inferred column types. A model without them is +skipped, not reported. The run counts skipped models in a note: + +```text +note: SQBRPERFORMANCE102 skipped 2 models without column lineage or inferred types; `sqb plan --verbose` lists them +``` + +`sqb plan --verbose` and `sqb build --verbose` list each skipped model with the missing fact. + +## Details + +- Both Rules report the column where the root query projects it. +- `SQBRPERFORMANCE102` reports nothing for a model without a declared unique key. diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 4312a3d731..f9f4f6563e 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -77,6 +77,7 @@ "docs/concepts/rules/findings-and-exceptions", "docs/concepts/rules/python-rules", "docs/concepts/rules/orchestration-rules", + "docs/concepts/rules/performance-rules", "docs/concepts/rules/execution-and-caching" ] } From d3efc80e4badc7083d388597935a65eb508d8ede Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 06:54:44 +0000 Subject: [PATCH 06/17] wip: narrow SQBRPERFORMANCE101 copies and honour covering keys in SQBRPERFORMANCE102 --- .../models/line_options_trimmed.sql | 3 +- .../models/order_payload_dated.sql | 15 ++++++++++ .../src/engine/tests/test_json_grains.rs | 13 ++++++++ .../src/engine/tests/test_query_facts.rs | 24 +++++++++++++++ .../src/query_facts/_helpers/flatten.rs | 30 +++++++++++++++---- .../src/rules/_helpers/json_finer_grain.rs | 25 ++++++++++++---- .../docs/concepts/rules/performance-rules.mdx | 12 +++++++- 7 files changed, 109 insertions(+), 13 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/order_payload_dated.sql diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql index 88784b8e5b..f89506e696 100644 --- a/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql @@ -1,9 +1,10 @@ MODEL ( - description "One row per line option with the parent line minus its options.", + description "One row per line option with the parent line minus its options, and a hash of the whole line.", ); SELECT r.order_id, OBJECT_DELETE(o.value, 'options') AS line_json, + MD5(TO_VARCHAR(o.value)) AS line_hash, l.value:code::string AS option_code FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_payload_dated.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_payload_dated.sql new file mode 100644 index 0000000000..5b9374e1ce --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_payload_dated.sql @@ -0,0 +1,15 @@ +MODEL ( + description "One row per order: the merge key adds a date for pruning, but order_id alone is unique.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, order_date], + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT + o.order_id, + CURRENT_DATE() AS order_date, + o.payload AS order_payload +FROM __ref("orders") o diff --git a/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs index d5174a463d..e89c36a4e5 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs @@ -59,6 +59,19 @@ fn given_grain_facts_when_evaluating_json_finer_grain_then_only_coarser_inputs_a json_grain: payload_grain(json!(["line_no", "order_id"]), json!([])), expected_faults: &[], }, + JsonGrainTestCase { + description: "another declared model key on the input's grain proves one row per input row", + dialect: "snowflake", + name: "order_payload_dated", + select: RULE, + query_sql: LINES_SQL, + json_grain: json!({ + "model_keys": [["order_id", "line_no"], ["ORDER_ID"]], + "columns": [{"name": "payload", "data_type": "VARIANT", "inputs": ["ref:orders"]}], + "input_keys": {"ref:orders": [["order_id"]]}, + }), + expected_faults: &[], + }, JsonGrainTestCase { description: "an input keyed by another column is not coarser", dialect: "snowflake", diff --git a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs index ad95d7ae1b..8d99cbe043 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs @@ -374,6 +374,30 @@ fn given_dialect_sql_when_reading_query_facts_then_findings_match_the_authored_s columns: json!([]), expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "line_json copies o.value whole")], }, + QueryFactsTestCase { + description: "a function that only reads the parent, such as a hash, copies nothing", + dialect: "snowflake", + query_sql: concat!( + "SELECT MD5(TO_VARCHAR(o.value)) AS line_hash, TYPEOF(o.value) AS kind\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, + QueryFactsTestCase { + description: "the parent cast and wrapped in an array is still a whole copy", + dialect: "snowflake", + query_sql: concat!( + "SELECT ARRAY_CONSTRUCT((o.value::object)) AS line_json\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "line_json copies o.value whole")], + }, QueryFactsTestCase { description: "an element no deeper FLATTEN reads is fine", dialect: "snowflake", diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs index bbaea5adf5..2140318424 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs @@ -1,8 +1,8 @@ //! FLATTEN facts: the element each FLATTEN reads, and the whole elements a select item copies. +use sqlquack_syntax::Expression; use sqlquack_syntax::expressions::functions::models::Function; use sqlquack_syntax::expressions::names::models::Literal; -use sqlquack_syntax::{Expression, ExpressionWalk}; use crate::query_facts::models::{FlattenInput, WholeValue}; @@ -83,7 +83,19 @@ fn literal_key(expression: &Expression) -> Option { (!key.is_empty()).then(|| key.trim_matches('"').to_ascii_lowercase()) } +/// Functions whose result embeds each argument whole. +const EMBEDDING_FUNCTIONS: [&str; 5] = [ + "object_construct", + "object_construct_keep_null", + "object_insert", + "array_construct", + "array_construct_compact", +]; + /// Every `.value` `expression` copies whole, with the keys `OBJECT_DELETE` removes from it. +/// +/// Only the value itself, casts and parentheses around it, and the constructors that embed it +/// count; any other function of the value, such as a hash or a path into it, does not copy it. pub(crate) fn whole_values(expression: &Expression) -> Vec { let mut found: Vec = Vec::new(); let mut pending: Vec<(&Expression, Vec)> = vec![(expression, Vec::new())]; @@ -97,9 +109,10 @@ pub(crate) fn whole_values(expression: &Expression) -> Vec { }); } } - Expression::JSONExtract(_) | Expression::Subscript(_) if path_root(node).is_some() => {} - Expression::Function(function) - if is_path_function(function) && path_root(node).is_some() => {} + Expression::Paren(paren) => pending.push((&paren.this, removed)), + Expression::Cast(cast) | Expression::TryCast(cast) | Expression::SafeCast(cast) => { + pending.push((&cast.this, removed)); + } Expression::Function(function) if function.name.eq_ignore_ascii_case("object_delete") => { @@ -108,7 +121,14 @@ pub(crate) fn whole_values(expression: &Expression) -> Vec { pending.push((object, args.filter_map(literal_key).collect())); } } - _ => pending.extend(node.children().into_iter().map(|child| (child, Vec::new()))), + Expression::Function(function) + if EMBEDDING_FUNCTIONS + .iter() + .any(|name| function.name.eq_ignore_ascii_case(name)) => + { + pending.extend(function.args.iter().map(|arg| (arg, Vec::new()))); + } + _ => {} } } found diff --git a/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs b/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs index 68e3bf0393..e5443facb9 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs @@ -44,6 +44,9 @@ pub(crate) fn evaluate( } /// An input of `column` whose key is a strict subset of one of the model's keys. +/// +/// An input key that already covers another model key does not count: that model key proves +/// the model has one row per input row. fn coarser_input<'a>( column: &'a GrainColumn, grain: &'a JsonGrain, @@ -51,6 +54,13 @@ fn coarser_input<'a>( column.inputs.iter().find_map(|input| { let keys = grain.input_keys.get(input)?; keys.iter().find_map(|input_key| { + if grain + .model_keys + .iter() + .any(|model_key| subset(model_key, input_key)) + { + return None; + } grain .model_keys .iter() @@ -61,12 +71,15 @@ fn coarser_input<'a>( } fn strict_subset(smaller: &[String], larger: &[String]) -> bool { - smaller.len() < larger.len() - && smaller.iter().all(|column| { - larger - .iter() - .any(|other| other.eq_ignore_ascii_case(column)) - }) + smaller.len() < larger.len() && subset(smaller, larger) +} + +fn subset(smaller: &[String], larger: &[String]) -> bool { + smaller.iter().all(|column| { + larger + .iter() + .any(|other| other.eq_ignore_ascii_case(column)) + }) } /// Where the root select projects `name`, following the left branch of set operations. diff --git a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx index bfc4db363a..3863eac210 100644 --- a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx @@ -38,6 +38,10 @@ error[SQBRPERFORMANCE101]: line_json copies o.value whole onto every row flatten --> models/line_options.sql:7:3 ``` +The parent also counts as copied when a cast, `OBJECT_CONSTRUCT`, `OBJECT_INSERT` or +`ARRAY_CONSTRUCT` embeds it. Paths into it, such as `o.value:sku`, and other functions of it, such +as `MD5(TO_VARCHAR(o.value))`, are not copies. + Read the fields you need from the parent, such as `o.value:sku::string`, or remove the child array before copying the rest: @@ -81,6 +85,10 @@ The Rule compares the unique keys SQLBuild knows from each relation's declaratio | An incremental model with `unique_key` and the `merge` strategy | The `unique_key` columns | | A seed whose file holds at most one data row | No columns | +A model with another declared key that the input's key covers is not reported: that key already +proves one row per input row. An incremental model with `unique_key [order_id, order_date]` and a +unique, not-null `order_id` column has one row per order. + Aggregated columns, such as `ARRAY_AGG(l.sku)`, are not copies and are not reported. ### Published views @@ -104,5 +112,7 @@ note: SQBRPERFORMANCE102 skipped 2 models without column lineage or inferred typ ## Details -- Both Rules report the column where the root query projects it. +- `SQBRPERFORMANCE101` reports the select item that copies the parent, inside a CTE or subquery + when the copy is made there. +- `SQBRPERFORMANCE102` reports the column where the root query projects it. - `SQBRPERFORMANCE102` reports nothing for a model without a declared unique key. From d6b2d4764e2e125a13a8e7d0cbeb53cb535066a5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 06:58:25 +0000 Subject: [PATCH 07/17] wip: count conversions of a FLATTEN parent as whole copies --- .../src/engine/tests/test_query_facts.rs | 12 ++++++++++ .../src/query_facts/_helpers/flatten.rs | 23 +++++++++++++++++-- .../docs/concepts/rules/performance-rules.mdx | 4 ++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs index 8d99cbe043..aabafcf759 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs @@ -386,6 +386,18 @@ fn given_dialect_sql_when_reading_query_facts_then_findings_match_the_authored_s columns: json!([]), expected_faults: &[], }, + QueryFactsTestCase { + description: "the parent serialized to JSON text is still a whole copy", + dialect: "snowflake", + query_sql: concat!( + "SELECT TO_JSON(o.value) AS line_text, TO_VARCHAR(OBJECT_DELETE(o.value, 'lines')) AS trimmed\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "line_text copies o.value whole")], + }, QueryFactsTestCase { description: "the parent cast and wrapped in an array is still a whole copy", dialect: "snowflake", diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs index 2140318424..cc93b2275e 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs @@ -92,10 +92,20 @@ const EMBEDDING_FUNCTIONS: [&str; 5] = [ "array_construct_compact", ]; +/// Conversions whose result is their first argument whole, in another type. +const CONVERSION_FUNCTIONS: [&str; 5] = [ + "to_json", + "to_variant", + "to_object", + "to_array", + "to_varchar", +]; + /// Every `.value` `expression` copies whole, with the keys `OBJECT_DELETE` removes from it. /// -/// Only the value itself, casts and parentheses around it, and the constructors that embed it -/// count; any other function of the value, such as a hash or a path into it, does not copy it. +/// Only the value itself, casts, conversions and parentheses around it, and the constructors that +/// embed it count; any other function of the value, such as a hash or a path into it, does not +/// copy it. pub(crate) fn whole_values(expression: &Expression) -> Vec { let mut found: Vec = Vec::new(); let mut pending: Vec<(&Expression, Vec)> = vec![(expression, Vec::new())]; @@ -113,6 +123,15 @@ pub(crate) fn whole_values(expression: &Expression) -> Vec { Expression::Cast(cast) | Expression::TryCast(cast) | Expression::SafeCast(cast) => { pending.push((&cast.this, removed)); } + Expression::Function(function) + if CONVERSION_FUNCTIONS + .iter() + .any(|name| function.name.eq_ignore_ascii_case(name)) => + { + if let Some(value) = function.args.first() { + pending.push((value, removed)); + } + } Expression::Function(function) if function.name.eq_ignore_ascii_case("object_delete") => { diff --git a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx index 3863eac210..3b3fa852e5 100644 --- a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx @@ -38,8 +38,8 @@ error[SQBRPERFORMANCE101]: line_json copies o.value whole onto every row flatten --> models/line_options.sql:7:3 ``` -The parent also counts as copied when a cast, `OBJECT_CONSTRUCT`, `OBJECT_INSERT` or -`ARRAY_CONSTRUCT` embeds it. Paths into it, such as `o.value:sku`, and other functions of it, such +The parent also counts as copied through a cast, a conversion such as `TO_JSON` or `TO_VARIANT`, +or a constructor that embeds it: `OBJECT_CONSTRUCT`, `OBJECT_INSERT` or `ARRAY_CONSTRUCT`. Paths into it, such as `o.value:sku`, and other functions of it, such as `MD5(TO_VARCHAR(o.value))`, are not copies. Read the fields you need from the parent, such as `o.value:sku::string`, or remove the child array From 59a28cfad567e129a230585e6115fc7f1b0da75b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:08:09 +0000 Subject: [PATCH 08/17] wip: exempt an input when any of its keys covers a model key; count TO_CHAR copies --- .../performance_rules/models/order_headers.sql | 12 ++++++++++++ .../models/order_headers_dated.sql | 16 ++++++++++++++++ .../src/engine/tests/test_json_grains.rs | 13 +++++++++++++ .../src/engine/tests/test_query_facts.rs | 2 +- .../src/query_facts/_helpers/flatten.rs | 3 ++- .../src/rules/_helpers/json_finer_grain.rs | 15 ++++++++------- .../docs/concepts/rules/performance-rules.mdx | 7 ++++--- 7 files changed, 56 insertions(+), 12 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/order_headers.sql create mode 100644 benchmarks/duckshop/edge_cases/performance_rules/models/order_headers_dated.sql diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers.sql new file mode 100644 index 0000000000..5de9f1b4e7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers.sql @@ -0,0 +1,12 @@ +MODEL ( + description "One row per order, unique by order_id and by order_number.", + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + order_number (type VARCHAR, nullable false, audits [unique]), + ), +); + +SELECT + 1 AS order_id, + 'SO-1' AS order_number, + PARSE_JSON('{"customer": "c1"}') AS header diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers_dated.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers_dated.sql new file mode 100644 index 0000000000..6894f89aa9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers_dated.sql @@ -0,0 +1,16 @@ +MODEL ( + description "One row per order: keyed by order number and date for pruning, but order_id alone is unique.", + materialized incremental, + incremental_strategy merge, + unique_key [order_number, order_date], + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT + h.order_id, + h.order_number, + CURRENT_DATE() AS order_date, + h.header AS order_header +FROM __ref("order_headers") h diff --git a/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs index e89c36a4e5..39a00a89a9 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs @@ -72,6 +72,19 @@ fn given_grain_facts_when_evaluating_json_finer_grain_then_only_coarser_inputs_a }), expected_faults: &[], }, + JsonGrainTestCase { + description: "an input with two keys is not coarser when either covers a model key", + dialect: "snowflake", + name: "order_payload_dated", + select: RULE, + query_sql: LINES_SQL, + json_grain: json!({ + "model_keys": [["order_number", "order_date"], ["order_id"]], + "columns": [{"name": "payload", "data_type": "VARIANT", "inputs": ["ref:orders"]}], + "input_keys": {"ref:orders": [["order_id"], ["order_number"]]}, + }), + expected_faults: &[], + }, JsonGrainTestCase { description: "an input keyed by another column is not coarser", dialect: "snowflake", diff --git a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs index aabafcf759..92ec50a262 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs @@ -390,7 +390,7 @@ fn given_dialect_sql_when_reading_query_facts_then_findings_match_the_authored_s description: "the parent serialized to JSON text is still a whole copy", dialect: "snowflake", query_sql: concat!( - "SELECT TO_JSON(o.value) AS line_text, TO_VARCHAR(OBJECT_DELETE(o.value, 'lines')) AS trimmed\n", + "SELECT TO_JSON(o.value) AS line_text, TO_CHAR(OBJECT_DELETE(o.value, 'lines')) AS trimmed\n", orders_flattened!() ), select: FLATTEN_RULE, diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs index cc93b2275e..b5fdab9b3c 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs @@ -93,8 +93,9 @@ const EMBEDDING_FUNCTIONS: [&str; 5] = [ ]; /// Conversions whose result is their first argument whole, in another type. -const CONVERSION_FUNCTIONS: [&str; 5] = [ +const CONVERSION_FUNCTIONS: [&str; 6] = [ "to_json", + "to_char", "to_variant", "to_object", "to_array", diff --git a/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs b/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs index e5443facb9..92c4bce3d5 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs @@ -45,22 +45,23 @@ pub(crate) fn evaluate( /// An input of `column` whose key is a strict subset of one of the model's keys. /// -/// An input key that already covers another model key does not count: that model key proves -/// the model has one row per input row. +/// An input with any key that covers a model key does not count: that model key proves the +/// model has one row per input row. fn coarser_input<'a>( column: &'a GrainColumn, grain: &'a JsonGrain, ) -> Option<(&'a str, &'a [String], &'a [String])> { column.inputs.iter().find_map(|input| { let keys = grain.input_keys.get(input)?; - keys.iter().find_map(|input_key| { - if grain + if keys.iter().any(|input_key| { + grain .model_keys .iter() .any(|model_key| subset(model_key, input_key)) - { - return None; - } + }) { + return None; + } + keys.iter().find_map(|input_key| { grain .model_keys .iter() diff --git a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx index 3b3fa852e5..d9d2032d91 100644 --- a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx @@ -39,8 +39,9 @@ error[SQBRPERFORMANCE101]: line_json copies o.value whole onto every row flatten ``` The parent also counts as copied through a cast, a conversion such as `TO_JSON` or `TO_VARIANT`, -or a constructor that embeds it: `OBJECT_CONSTRUCT`, `OBJECT_INSERT` or `ARRAY_CONSTRUCT`. Paths into it, such as `o.value:sku`, and other functions of it, such -as `MD5(TO_VARCHAR(o.value))`, are not copies. +or a constructor that embeds it: `OBJECT_CONSTRUCT`, `OBJECT_INSERT` or `ARRAY_CONSTRUCT`. Paths +into it, such as `o.value:sku`, and other functions of it, such as `MD5(TO_VARCHAR(o.value))`, are +not copies. Read the fields you need from the parent, such as `o.value:sku::string`, or remove the child array before copying the rest: @@ -85,7 +86,7 @@ The Rule compares the unique keys SQLBuild knows from each relation's declaratio | An incremental model with `unique_key` and the `merge` strategy | The `unique_key` columns | | A seed whose file holds at most one data row | No columns | -A model with another declared key that the input's key covers is not reported: that key already +A model with another declared key that one of the input's keys covers is not reported: that key already proves one row per input row. An incremental model with `unique_key [order_id, order_date]` and a unique, not-null `order_id` column has one row per order. From 8fa82c0efec5e7d8361a738e2a37ab2f1b6916ff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:54:27 +0000 Subject: [PATCH 09/17] refactor: group rules helpers and split grain builders for the layout rules --- .../_helpers/compile_delta/rules_rows.rs | 2 +- .../_helpers/compile_driver/command.rs | 19 +- .../_helpers/compile_driver/plan_pipeline.rs | 11 +- .../src/bindings/_helpers/rules/rows.rs | 36 ++-- .../src/engine/_helpers/evaluation.rs | 2 +- .../src/engine/_helpers/json_grain.rs | 71 +++++++ .../sqlbuild-rules/src/engine/_helpers/mod.rs | 1 + crates/sqlbuild-rules/src/engine/constants.rs | 10 + .../engine/main/json_grain_skipped_models.rs | 18 ++ .../src/engine/main/json_grains.rs | 128 ++---------- crates/sqlbuild-rules/src/engine/main/mod.rs | 2 + .../src/engine/main/needs_json_grains.rs | 12 ++ crates/sqlbuild-rules/src/engine/mod.rs | 1 + .../src/engine/tests/helpers.rs | 118 ++++++++++- .../src/engine/tests/test_json_grains.rs | 193 ++++++------------ .../src/engine/tests/test_sql_scanning.rs | 2 +- .../src/engine/tests/test_types.rs | 9 + .../src/python_rules/tests/helpers.rs | 2 +- .../tests/test_python_catalogue.rs | 2 +- .../src/query_facts/_helpers/flatten.rs | 44 ++-- .../{ => contracts}/contract_name_types.rs | 0 .../{ => contracts}/dynamic_contracts.rs | 6 +- .../{ => contracts}/explicit_output_types.rs | 2 +- .../src/rules/_helpers/contracts/mod.rs | 4 + .../{ => contracts}/typed_contract_columns.rs | 0 .../_helpers/{ => dispatch}/catalogue.rs | 0 .../_helpers/{ => dispatch}/evaluation.rs | 32 +-- .../src/rules/_helpers/dispatch/mod.rs | 2 + .../_helpers/{ => layers}/domain_layout.rs | 2 +- .../src/rules/_helpers/layers/mod.rs | 2 + .../_helpers/{ => layers}/model_layers.rs | 2 +- .../sqlbuild-rules/src/rules/_helpers/mod.rs | 17 +- .../{ => performance}/flatten_parent_copy.rs | 62 +++--- .../{ => performance}/json_finer_grain.rs | 94 +++++---- .../src/rules/_helpers/performance/mod.rs | 2 + .../src/rules/_helpers/sql_checks/mod.rs | 2 + .../{ => sql_checks}/numeric_decisions.rs | 0 .../{ => sql_checks}/sql_test_rules.rs | 2 +- .../src/rules/main/assemble_catalogue.rs | 2 +- .../src/rules/main/catalogue.rs | 2 +- .../src/rules/main/compact_sql.rs | 2 +- .../sqlbuild-rules/src/rules/main/evaluate.rs | 2 +- .../src/rules/main/evaluate_project.rs | 10 +- .../src/rules/main/fingerprint.rs | 2 +- .../rules/main/resolve_threshold_overrides.rs | 2 +- .../sqlbuild-rules/src/rules/main/select.rs | 2 +- .../src/rules/main/selected_codes.rs | 2 +- .../src/rules/main/selected_sql_codes.rs | 2 +- .../src/rules/main/sql_test_coverage.rs | 2 +- .../sqlbuild/cli/commands/main/test_rules.py | 18 +- 50 files changed, 535 insertions(+), 427 deletions(-) create mode 100644 crates/sqlbuild-rules/src/engine/_helpers/json_grain.rs create mode 100644 crates/sqlbuild-rules/src/engine/constants.rs create mode 100644 crates/sqlbuild-rules/src/engine/main/json_grain_skipped_models.rs create mode 100644 crates/sqlbuild-rules/src/engine/main/needs_json_grains.rs rename crates/sqlbuild-rules/src/rules/_helpers/{ => contracts}/contract_name_types.rs (100%) rename crates/sqlbuild-rules/src/rules/_helpers/{ => contracts}/dynamic_contracts.rs (94%) rename crates/sqlbuild-rules/src/rules/_helpers/{ => contracts}/explicit_output_types.rs (99%) create mode 100644 crates/sqlbuild-rules/src/rules/_helpers/contracts/mod.rs rename crates/sqlbuild-rules/src/rules/_helpers/{ => contracts}/typed_contract_columns.rs (100%) rename crates/sqlbuild-rules/src/rules/_helpers/{ => dispatch}/catalogue.rs (100%) rename crates/sqlbuild-rules/src/rules/_helpers/{ => dispatch}/evaluation.rs (97%) create mode 100644 crates/sqlbuild-rules/src/rules/_helpers/dispatch/mod.rs rename crates/sqlbuild-rules/src/rules/_helpers/{ => layers}/domain_layout.rs (99%) create mode 100644 crates/sqlbuild-rules/src/rules/_helpers/layers/mod.rs rename crates/sqlbuild-rules/src/rules/_helpers/{ => layers}/model_layers.rs (99%) rename crates/sqlbuild-rules/src/rules/_helpers/{ => performance}/flatten_parent_copy.rs (69%) rename crates/sqlbuild-rules/src/rules/_helpers/{ => performance}/json_finer_grain.rs (54%) create mode 100644 crates/sqlbuild-rules/src/rules/_helpers/performance/mod.rs create mode 100644 crates/sqlbuild-rules/src/rules/_helpers/sql_checks/mod.rs rename crates/sqlbuild-rules/src/rules/_helpers/{ => sql_checks}/numeric_decisions.rs (100%) rename crates/sqlbuild-rules/src/rules/_helpers/{ => sql_checks}/sql_test_rules.rs (99%) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs index 1d462f372e..bb97307f01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use pyo3::PyResult; -use sqlbuild_rules::engine::main::json_grains::needs_json_grains; +use sqlbuild_rules::engine::main::needs_json_grains::needs_json_grains; use sqlbuild_rules::models::{EvaluateRequest, Model, SqlTestFact}; use crate::bindings::_helpers::compile_context::delta::BaseRules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 6b85f69260..39f3efe848 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -425,15 +425,16 @@ fn compiled_with_status<'py>( } } close_status(status)?; - if let Some(skipped_note) = format_skipped_type_proof_rules( - native_field(&analysis, "skipped_type_proof_rules")?.extract()?, - ) { - output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?; - } - if let Some(skipped_note) = format_skipped_models( - &native_field(&analysis, "skipped_models")?.extract::>()?, - false, - ) { + let skipped_notes = [ + format_skipped_type_proof_rules( + native_field(&analysis, "skipped_type_proof_rules")?.extract()?, + ), + format_skipped_models( + &native_field(&analysis, "skipped_models")?.extract::>()?, + false, + ), + ]; + for skipped_note in skipped_notes.into_iter().flatten() { output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?; } let dag_artifact_path: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index 15b6f23bf6..131bb1acfe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -138,13 +138,12 @@ fn run_rules_gate<'py>( if !on_progress.is_none() { let elapsed: f64 = monotonic(py)? - started; let message: String = rules_progress(elapsed, &result, true)?; - on_progress.call1((message,))?; - let verbose: bool = native_field(options, "verbose")?.is_truthy()?; - if let Some(note) = format_skipped_models( + let note: Option = format_skipped_models( &native_field(&result, "skipped_models")?.extract::>()?, - verbose, - ) { - on_progress.call1((format!("note: {note}"),))?; + native_field(options, "verbose")?.is_truthy()?, + ); + for line in std::iter::once(message).chain(note.map(|note| format!("note: {note}"))) { + on_progress.call1((line,))?; } } let mut failing: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 8e0cdfb840..716d9b277e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -18,7 +18,9 @@ use sqlbuild_rules::constants::API_VERSION; use sqlbuild_rules::engine::main::build_models::build_models; use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; use sqlbuild_rules::engine::main::finalize_rows::finalize_rows; -use sqlbuild_rules::engine::main::json_grains::{json_grains, needs_json_grains, skipped_models}; +use sqlbuild_rules::engine::main::json_grain_skipped_models::json_grain_skipped_models; +use sqlbuild_rules::engine::main::json_grains::json_grains; +use sqlbuild_rules::engine::main::needs_json_grains::needs_json_grains; use sqlbuild_rules::engine::main::project_model_rows::project_model_rows; use sqlbuild_rules::errors::RowsError; use sqlbuild_rules::models::{ @@ -374,7 +376,7 @@ fn evaluated<'py>( early: Option<(EarlyBuiltinRules, &AtomicBool)>, ) -> PyResult> { let request: RowsRequest = taken_rows(request)?; - let skipped: Vec<(String, String)> = skipped_models(&request.request.models); + let skipped: Vec<(String, String)> = json_grain_skipped_models(&request.request.models); let evaluation: Result = match early { None => py .compiler_detach(|| Ok(evaluate_rows(request))) @@ -538,19 +540,23 @@ fn model_lineage(model: &Bound<'_, PyAny>) -> PyResult let Some(view) = native_model(model)? else { return Ok(None); }; - Ok(view.lineage_rows(model.py())?.map(|rows| { - rows.into_iter() - .map(|row| LineageColumn { - aggregated: row.aggregated(), - output: row.output_column, - sources: row - .sources - .into_iter() - .map(|(resource, name, _)| (resource, name)) - .collect(), - }) - .collect() - })) + let Some(rows) = view.lineage_rows(model.py())? else { + return Ok(None); + }; + let mut columns: Vec = Vec::with_capacity(rows.len()); + for row in rows { + let aggregated: bool = row.aggregated(); + let mut sources: Vec<(String, String)> = Vec::with_capacity(row.sources.len()); + for (resource, name, _) in row.sources { + sources.push((resource, name)); + } + columns.push(LineageColumn { + aggregated, + output: row.output_column, + sources, + }); + } + Ok(Some(columns)) } fn declarations(rows: Vec>) -> PyResult> { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 51f5939936..724c8960b8 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -431,7 +431,7 @@ pub(crate) fn evaluate_request_with( &request.rules_cache_path, ) }); - crate::rules::_helpers::sql_test_rules::with_fixture_memo(store.as_deref(), || { + crate::rules::_helpers::sql_checks::sql_test_rules::with_fixture_memo(store.as_deref(), || { let evaluated: Result = evaluated_request(request, control); let keep: bool = match &evaluated { Ok(_) => !control.cancelled(), diff --git a/crates/sqlbuild-rules/src/engine/_helpers/json_grain.rs b/crates/sqlbuild-rules/src/engine/_helpers/json_grain.rs new file mode 100644 index 0000000000..6d7a348705 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/json_grain.rs @@ -0,0 +1,71 @@ +//! One model row's SQBRPERFORMANCE102 grain: its keys, readers and copied semi-structured columns. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::engine::constants::{NO_LINEAGE, NO_TYPES, RESOURCE_KINDS, SEMI_STRUCTURED_TYPES}; +use crate::models::{GrainColumn, JsonGrain, LineageColumn, ModelRow}; + +pub(crate) fn grain( + row: &ModelRow, + (model_keys, readers): (Vec>, Vec), + lineage: Option>, + keys: &BTreeMap>>, +) -> JsonGrain { + let mut grain: JsonGrain = JsonGrain { + model_keys, + readers, + ..JsonGrain::default() + }; + let Some(lineage) = lineage.filter(|columns| !columns.is_empty()) else { + grain.skipped = Some(NO_LINEAGE.to_owned()); + return grain; + }; + if row.inferred_columns.is_empty() { + grain.skipped = Some(NO_TYPES.to_owned()); + return grain; + } + for (name, data_type) in &row.inferred_columns { + let Some(data_type) = data_type.as_deref().filter(|kind| semi_structured(kind)) else { + continue; + }; + let Some(column) = lineage + .iter() + .find(|column| column.output.eq_ignore_ascii_case(name)) + .filter(|column| !column.aggregated) + else { + continue; + }; + let inputs: BTreeSet = column_inputs(column); + for input in &inputs { + if let Some(found) = keys.get(input) { + grain.input_keys.insert(input.clone(), found.clone()); + } + } + grain.columns.push(GrainColumn { + name: name.clone(), + data_type: Some(data_type.to_owned()), + inputs: inputs.into_iter().collect(), + }); + } + grain +} + +/// `kind:name` of each relation `column` reads whose resource type declares keys. +fn column_inputs(column: &LineageColumn) -> BTreeSet { + let mut inputs: BTreeSet = BTreeSet::new(); + for (resource, name) in &column.sources { + if let Some((_, kind)) = RESOURCE_KINDS.iter().find(|(known, _)| known == resource) { + inputs.insert(format!("{kind}:{name}")); + } + } + inputs +} + +/// `VARIANT`, `OBJECT` or `ARRAY`, alone or with parameters such as `ARRAY(INT)`. +fn semi_structured(data_type: &str) -> bool { + let upper: String = data_type.trim().to_ascii_uppercase(); + SEMI_STRUCTURED_TYPES + .iter() + .filter_map(|kind| upper.strip_prefix(kind)) + .any(|rest| rest.is_empty() || rest.starts_with(['(', '<', ' '])) +} diff --git a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs index 234f764470..a0a61194a4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs @@ -1,6 +1,7 @@ pub mod cache; pub mod custom_host; pub mod evaluation; +pub(crate) mod json_grain; pub(crate) mod memo; pub(crate) mod project_rows; pub(crate) mod retained; diff --git a/crates/sqlbuild-rules/src/engine/constants.rs b/crates/sqlbuild-rules/src/engine/constants.rs new file mode 100644 index 0000000000..282c8fc5e5 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/constants.rs @@ -0,0 +1,10 @@ +//! SQBRPERFORMANCE102 grain constants. + +pub(crate) const JSON_GRAIN_RULE_CODE: &str = "SQBRPERFORMANCE102"; +pub(crate) const REFERENCE_KIND: &str = "ref"; +pub(crate) const SEMI_STRUCTURED_TYPES: [&str; 3] = ["VARIANT", "OBJECT", "ARRAY"]; +/// Lineage resource types and the relation-key kinds they are declared under. +pub(crate) const RESOURCE_KINDS: [(&str, &str); 3] = + [("model", "ref"), ("source", "source"), ("seed", "seed")]; +pub(crate) const NO_LINEAGE: &str = "no column lineage"; +pub(crate) const NO_TYPES: &str = "no inferred column types"; diff --git a/crates/sqlbuild-rules/src/engine/main/json_grain_skipped_models.rs b/crates/sqlbuild-rules/src/engine/main/json_grain_skipped_models.rs new file mode 100644 index 0000000000..cb3dd34a62 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/json_grain_skipped_models.rs @@ -0,0 +1,18 @@ +//! The analysed models SQBRPERFORMANCE102 skipped for missing facts. + +use crate::models::Model; + +/// `(path, reason)` of each analysed model SQBRPERFORMANCE102 skipped for missing facts. +pub fn json_grain_skipped_models(models: &[Model]) -> Vec<(String, String)> { + let mut skipped: Vec<(String, String)> = Vec::new(); + for model in models.iter().filter(|model| !model.sql_analysis_disabled) { + if let Some(reason) = model + .json_grain + .as_ref() + .and_then(|grain| grain.skipped.as_ref()) + { + skipped.push((model.relative_path.clone(), reason.clone())); + } + } + skipped +} diff --git a/crates/sqlbuild-rules/src/engine/main/json_grains.rs b/crates/sqlbuild-rules/src/engine/main/json_grains.rs index da2a03c3d9..71ff2c068b 100644 --- a/crates/sqlbuild-rules/src/engine/main/json_grains.rs +++ b/crates/sqlbuild-rules/src/engine/main/json_grains.rs @@ -2,24 +2,9 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::models::{EvaluateRequest, GrainColumn, JsonGrain, LineageColumn, Model, ModelRow}; -use crate::query_facts::constants::SNOWFLAKE_DIALECT; -use crate::rules::main::request_selects::request_selects; - -const RULE_CODE: &str = "SQBRPERFORMANCE102"; -const REFERENCE_KIND: &str = "ref"; -const SEMI_STRUCTURED_TYPES: [&str; 3] = ["VARIANT", "OBJECT", "ARRAY"]; -/// Lineage resource types and the relation-key kinds they are declared under. -const RESOURCE_KINDS: [(&str, &str); 3] = - [("model", "ref"), ("source", "source"), ("seed", "seed")]; -const NO_LINEAGE: &str = "no column lineage"; -const NO_TYPES: &str = "no inferred column types"; - -/// Whether `request` selects SQBRPERFORMANCE102 for a dialect it runs on. -pub fn needs_json_grains(request: &EvaluateRequest) -> Result { - Ok(request.dialect.eq_ignore_ascii_case(SNOWFLAKE_DIALECT) - && request_selects(request, |rule| rule.code == RULE_CODE)?) -} +use crate::engine::_helpers::json_grain::grain; +use crate::engine::constants::REFERENCE_KIND; +use crate::models::{JsonGrain, LineageColumn, ModelRow}; /// Set each keyed row's grain from its lineage (`None` when unknown), in row order, and `keys` by `kind:name`. pub fn json_grains( @@ -38,98 +23,23 @@ pub fn json_grains( } } } - let grains: Vec> = rows - .iter() - .zip(lineage) - .map(|(row, lineage)| { - let model_keys: &Vec> = - keys.get(&format!("{REFERENCE_KIND}:{}", row.name))?; - let readers: Vec = readers - .get(row.name.as_str()) - .map(|names| names.iter().cloned().collect()) - .unwrap_or_default(); - Some(grain(row, model_keys.clone(), readers, lineage, keys)) - }) - .collect(); - for (row, grain) in rows.iter_mut().zip(grains) { - row.json_grain = grain; - } -} - -/// `(path, reason)` of each analysed model SQBRPERFORMANCE102 skipped for missing facts. -pub fn skipped_models(models: &[Model]) -> Vec<(String, String)> { - models - .iter() - .filter(|model| !model.sql_analysis_disabled) - .filter_map(|model| { - let reason: &String = model.json_grain.as_ref()?.skipped.as_ref()?; - Some((model.relative_path.clone(), reason.clone())) - }) - .collect() -} - -fn grain( - row: &ModelRow, - model_keys: Vec>, - readers: Vec, - lineage: Option>, - keys: &BTreeMap>>, -) -> JsonGrain { - let mut grain: JsonGrain = JsonGrain { - model_keys, - readers, - ..JsonGrain::default() - }; - let Some(lineage) = lineage.filter(|columns| !columns.is_empty()) else { - grain.skipped = Some(NO_LINEAGE.to_owned()); - return grain; - }; - if row.inferred_columns.is_empty() { - grain.skipped = Some(NO_TYPES.to_owned()); - return grain; - } - for (name, data_type) in &row.inferred_columns { - let Some(data_type) = data_type.as_deref().filter(|kind| semi_structured(kind)) else { - continue; - }; - let Some(column) = lineage - .iter() - .find(|column| column.output.eq_ignore_ascii_case(name)) - .filter(|column| !column.aggregated) - else { + let mut grains: Vec> = Vec::with_capacity(rows.len()); + for (row, lineage) in rows.iter().zip(lineage) { + let Some(model_keys) = keys.get(&format!("{REFERENCE_KIND}:{}", row.name)) else { + grains.push(None); continue; }; - let inputs: BTreeSet = column - .sources - .iter() - .filter_map(|(resource, name)| { - let kind: &str = RESOURCE_KINDS - .iter() - .find(|(known, _)| known == resource)? - .1; - Some(format!("{kind}:{name}")) - }) - .collect(); - for input in &inputs { - if let Some(found) = keys.get(input) { - grain.input_keys.insert(input.clone(), found.clone()); - } - } - grain.columns.push(GrainColumn { - name: name.clone(), - data_type: Some(data_type.to_owned()), - inputs: inputs.into_iter().collect(), - }); + let row_readers: Vec = readers + .get(row.name.as_str()) + .map_or_else(Vec::new, |names| names.iter().cloned().collect()); + grains.push(Some(grain( + row, + (model_keys.clone(), row_readers), + lineage, + keys, + ))); + } + for (row, grain) in rows.iter_mut().zip(grains) { + row.json_grain = grain; } - grain -} - -/// `VARIANT`, `OBJECT` or `ARRAY`, alone or with parameters such as `ARRAY(INT)`. -fn semi_structured(data_type: &str) -> bool { - let upper: String = data_type.trim().to_ascii_uppercase(); - SEMI_STRUCTURED_TYPES.iter().any(|kind| { - upper - .strip_prefix(kind) - .is_some_and(|rest| rest.is_empty() || rest.starts_with(['(', '<', ' '])) - }) } diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index e95f288186..1acb27f0d2 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -4,7 +4,9 @@ pub mod custom_host; pub mod discard_speculative_rows; pub mod evaluate_rows; pub mod finalize_rows; +pub mod json_grain_skipped_models; pub mod json_grains; +pub mod needs_json_grains; pub mod project_model_rows; pub mod retained_request; pub mod start_speculative_rows; diff --git a/crates/sqlbuild-rules/src/engine/main/needs_json_grains.rs b/crates/sqlbuild-rules/src/engine/main/needs_json_grains.rs new file mode 100644 index 0000000000..478b89265f --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/needs_json_grains.rs @@ -0,0 +1,12 @@ +//! Whether a rules request needs SQBRPERFORMANCE102's grain facts. + +use crate::engine::constants::JSON_GRAIN_RULE_CODE; +use crate::models::EvaluateRequest; +use crate::query_facts::constants::SNOWFLAKE_DIALECT; +use crate::rules::main::request_selects::request_selects; + +/// Whether `request` selects SQBRPERFORMANCE102 for a dialect it runs on. +pub fn needs_json_grains(request: &EvaluateRequest) -> Result { + Ok(request.dialect.eq_ignore_ascii_case(SNOWFLAKE_DIALECT) + && request_selects(request, |rule| rule.code == JSON_GRAIN_RULE_CODE)?) +} diff --git a/crates/sqlbuild-rules/src/engine/mod.rs b/crates/sqlbuild-rules/src/engine/mod.rs index d5e5e9f39a..79a6444abf 100644 --- a/crates/sqlbuild-rules/src/engine/mod.rs +++ b/crates/sqlbuild-rules/src/engine/mod.rs @@ -1,4 +1,5 @@ pub mod _helpers; +mod constants; pub mod main; #[cfg(test)] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index 15922fa5e8..878e4ba729 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -12,14 +12,15 @@ use crate::engine::main::discard_speculative_rows::discard_speculative_rows; use crate::engine::main::retained_request::retained_request; use crate::models::{ Column, Declaration, DeclarationMember, DynamicColumnFamily, EvaluateRequest, Fault, - GrainColumn, JsonGrain, Model, Reference, RetainedRows, RowsEvaluation, RowsRequest, - RulesConfig, SpeculativeAnswer, SpeculativeRows, + GrainColumn, JsonGrain, LineageColumn, Model, ModelRow, Reference, RetainedRows, + RowsEvaluation, RowsRequest, RulesConfig, SpeculativeAnswer, SpeculativeRows, }; use crate::engine::_helpers::evaluation::evaluate_json; use crate::engine::_helpers::evaluation::evaluate_request as evaluate_parsed; use crate::engine::_helpers::evaluation::parse_parts; use crate::engine::main::finalize_rows::finalize_rows; +use crate::engine::main::json_grains::json_grains; use crate::engine::tests::test_types; pub(crate) fn request(project_dir: &TempDir, config: &Value) -> String { @@ -889,3 +890,116 @@ pub(crate) fn finalize_cross_joins(exceptions: Value) -> Result, Stri findings, ) } + +/// A `json_grain` fact for a payload copied from `orders` onto order lines. +pub(crate) fn payload_grain(input_key: Value, readers: Value) -> Value { + json!({ + "model_keys": [["order_id", "line_no"]], + "columns": [{"name": "payload", "data_type": "VARIANT", "inputs": ["ref:orders"]}], + "input_keys": {"ref:orders": [input_key]}, + "readers": readers, + }) +} + +/// A model row with `references` to other models and `inferred` column types. +pub(crate) fn grain_row(name: &str, references: &[&str], inferred: &[(&str, &str)]) -> ModelRow { + ModelRow { + name: name.to_owned(), + references: references + .iter() + .map(|name| Reference { + ref_kind: "ref".to_owned(), + ref_name: (*name).to_owned(), + ref_package: None, + }) + .collect(), + inferred_columns: inferred + .iter() + .map(|(name, kind)| ((*name).to_owned(), Some((*kind).to_owned()))) + .collect(), + ..ModelRow::default() + } +} + +fn grain_lineage(output: &str, sources: &[(&str, &str)], aggregated: bool) -> LineageColumn { + LineageColumn { + output: output.to_owned(), + sources: sources + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(), + aggregated, + } +} + +/// One declared key of `key`'s columns. +pub(crate) fn grain_keys(key: &[&str]) -> Vec> { + vec![key.iter().map(|column| (*column).to_owned()).collect()] +} + +pub(crate) fn grain_column(name: &str, data_type: &str, inputs: &[&str]) -> GrainColumn { + GrainColumn { + name: name.to_owned(), + data_type: Some(data_type.to_owned()), + inputs: inputs.iter().map(|input| (*input).to_owned()).collect(), + } +} + +/// Grains `json_grains` builds for four order models, by model name. +pub(crate) fn built_json_grains() -> BTreeMap> { + let mut rows = vec![ + grain_row( + "order_lines_enriched", + &["orders", "order_lines"], + &[ + ("payload", "VARIANT"), + ("items", "ARRAY(VARCHAR)"), + ("region", "object"), + ("total", "NUMBER(38,0)"), + ("payloads", "ARRAY"), + ("arrayish", "ARRAYLIKE"), + ], + ), + grain_row("orders", &[], &[("payload", "VARIANT")]), + grain_row("unkeyed", &["orders"], &[("payload", "VARIANT")]), + grain_row("untyped", &[], &[]), + ]; + let lineage = vec![ + Some(vec![ + grain_lineage( + "PAYLOAD", + &[("model", "orders"), ("table_fn", "explode")], + false, + ), + grain_lineage("items", &[("source", "shop.raw_orders")], false), + grain_lineage("region", &[("seed", "countries")], false), + grain_lineage("total", &[("model", "orders")], false), + grain_lineage("payloads", &[("model", "orders")], true), + grain_lineage("arrayish", &[("model", "orders")], false), + ]), + None, + Some(vec![grain_lineage( + "payload", + &[("model", "orders")], + false, + )]), + Some(vec![grain_lineage( + "payload", + &[("model", "orders")], + false, + )]), + ]; + let declared: BTreeMap>> = BTreeMap::from([ + ( + "ref:order_lines_enriched".to_owned(), + grain_keys(&["order_id", "line_no"]), + ), + ("ref:orders".to_owned(), grain_keys(&["order_id"])), + ("ref:untyped".to_owned(), grain_keys(&["id"])), + ("seed:countries".to_owned(), grain_keys(&["country_code"])), + ]); + json_grains(&mut rows, lineage, &declared); + rows.into_iter() + .map(|row| (row.name, row.json_grain)) + .collect() +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs index 39a00a89a9..27687c67b3 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs @@ -1,24 +1,16 @@ use std::collections::BTreeMap; -use serde_json::{Value, json}; +use serde_json::json; -use crate::engine::main::json_grains::json_grains; -use crate::engine::tests::helpers::json_grain_faults; -use crate::engine::tests::test_types::JsonGrainTestCase; -use crate::models::{GrainColumn, JsonGrain, LineageColumn, ModelRow, Reference}; +use crate::engine::tests::helpers::{ + built_json_grains, grain_column, grain_keys, json_grain_faults, payload_grain, +}; +use crate::engine::tests::test_types::{JsonGrainTestCase, JsonGrainsBuildTestCase}; +use crate::models::JsonGrain; const RULE: &[&str] = &["SQBRPERFORMANCE102"]; const LINES_SQL: &str = "SELECT l.order_id, l.line_no, o.payload\nFROM __ref(\"order_lines\") l JOIN __ref(\"orders\") o USING (order_id)"; -fn payload_grain(input_key: Value, readers: Value) -> Value { - json!({ - "model_keys": [["order_id", "line_no"]], - "columns": [{"name": "payload", "data_type": "VARIANT", "inputs": ["ref:orders"]}], - "input_keys": {"ref:orders": [input_key]}, - "readers": readers, - }) -} - #[test] fn given_grain_facts_when_evaluating_json_finer_grain_then_only_coarser_inputs_are_flagged() -> Result<(), String> { @@ -145,145 +137,82 @@ fn given_grain_facts_when_evaluating_json_finer_grain_then_only_coarser_inputs_a ]; for test_case in test_cases { let faults = json_grain_faults(&test_case)?; - let observed: Vec<(&str, u64, u64, bool)> = faults - .iter() - .map(|(code, line, column, _)| (code.as_str(), *line, *column, true)) - .collect(); - let expected: Vec<(&str, u64, u64, bool)> = test_case - .expected_faults + let observed: Vec<(&str, u64, u64, &str)> = faults .iter() - .map(|(code, line, column, _)| (*code, *line, *column, true)) + .enumerate() + .map(|(index, (code, line, column, message))| { + let fragment = test_case + .expected_faults + .get(index) + .map(|expected| expected.3) + .filter(|fragment| message.contains(fragment)) + .unwrap_or(message.as_str()); + (code.as_str(), *line, *column, fragment) + }) .collect(); - assert_eq!(observed, expected, "{}", test_case.description); - for ((_, _, _, message), (_, _, _, fragment)) in - faults.iter().zip(test_case.expected_faults) - { - assert!( - message.contains(fragment), - "{}: {message}", - test_case.description - ); - } + assert_eq!( + observed, test_case.expected_faults, + "{}", + test_case.description + ); } Ok(()) } -fn row(name: &str, references: &[&str], inferred: &[(&str, &str)]) -> ModelRow { - ModelRow { - name: name.to_owned(), - references: references - .iter() - .map(|name| Reference { - ref_kind: "ref".to_owned(), - ref_name: (*name).to_owned(), - ref_package: None, - }) - .collect(), - inferred_columns: inferred - .iter() - .map(|(name, kind)| ((*name).to_owned(), Some((*kind).to_owned()))) - .collect(), - ..ModelRow::default() - } -} - -fn lineage(output: &str, sources: &[(&str, &str)], aggregated: bool) -> LineageColumn { - LineageColumn { - output: output.to_owned(), - sources: sources - .iter() - .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) - .collect(), - aggregated, - } -} - -fn keys(key: &[&str]) -> Vec> { - vec![key.iter().map(|column| (*column).to_owned()).collect()] -} - #[test] fn given_rows_lineage_and_keys_when_building_grains_then_only_copied_semi_structured_columns_are_kept() { - let mut rows = vec![ - row( - "order_lines_enriched", - &["orders", "order_lines"], - &[ - ("payload", "VARIANT"), - ("items", "ARRAY(VARCHAR)"), - ("region", "object"), - ("total", "NUMBER(38,0)"), - ("payloads", "ARRAY"), - ("arrayish", "ARRAYLIKE"), - ], - ), - row("orders", &[], &[("payload", "VARIANT")]), - row("unkeyed", &["orders"], &[("payload", "VARIANT")]), - row("untyped", &[], &[]), - ]; - let lineage = vec![ - Some(vec![ - lineage( - "PAYLOAD", - &[("model", "orders"), ("table_fn", "explode")], - false, - ), - lineage("items", &[("source", "shop.raw_orders")], false), - lineage("region", &[("seed", "countries")], false), - lineage("total", &[("model", "orders")], false), - lineage("payloads", &[("model", "orders")], true), - lineage("arrayish", &[("model", "orders")], false), - ]), - None, - Some(vec![lineage("payload", &[("model", "orders")], false)]), - Some(vec![lineage("payload", &[("model", "orders")], false)]), - ]; - let declared: BTreeMap>> = BTreeMap::from([ - ( - "ref:order_lines_enriched".to_owned(), - keys(&["order_id", "line_no"]), - ), - ("ref:orders".to_owned(), keys(&["order_id"])), - ("ref:untyped".to_owned(), keys(&["id"])), - ("seed:countries".to_owned(), keys(&["country_code"])), - ]); - json_grains(&mut rows, lineage, &declared); - let column = |name: &str, data_type: &str, inputs: &[&str]| GrainColumn { - name: name.to_owned(), - data_type: Some(data_type.to_owned()), - inputs: inputs.iter().map(|input| (*input).to_owned()).collect(), - }; - let grains: Vec> = rows.into_iter().map(|row| row.json_grain).collect(); - assert_eq!( - grains, - vec![ - Some(JsonGrain { - model_keys: keys(&["order_id", "line_no"]), + let test_cases = [ + JsonGrainsBuildTestCase { + description: "a keyed model keeps only copied VARIANT, OBJECT and ARRAY columns with their input keys", + row_name: "order_lines_enriched", + expected_grain: Some(JsonGrain { + model_keys: grain_keys(&["order_id", "line_no"]), columns: vec![ - column("payload", "VARIANT", &["ref:orders"]), - column("items", "ARRAY(VARCHAR)", &["source:shop.raw_orders"]), - column("region", "object", &["seed:countries"]), + grain_column("payload", "VARIANT", &["ref:orders"]), + grain_column("items", "ARRAY(VARCHAR)", &["source:shop.raw_orders"]), + grain_column("region", "object", &["seed:countries"]), ], input_keys: BTreeMap::from([ - ("ref:orders".to_owned(), keys(&["order_id"])), - ("seed:countries".to_owned(), keys(&["country_code"])), + ("ref:orders".to_owned(), grain_keys(&["order_id"])), + ("seed:countries".to_owned(), grain_keys(&["country_code"])), ]), readers: Vec::new(), skipped: None, }), - Some(JsonGrain { - model_keys: keys(&["order_id"]), + }, + JsonGrainsBuildTestCase { + description: "a model without lineage is skipped and lists its readers", + row_name: "orders", + expected_grain: Some(JsonGrain { + model_keys: grain_keys(&["order_id"]), readers: vec!["order_lines_enriched".to_owned(), "unkeyed".to_owned()], skipped: Some("no column lineage".to_owned()), ..JsonGrain::default() }), - None, - Some(JsonGrain { - model_keys: keys(&["id"]), + }, + JsonGrainsBuildTestCase { + description: "a model without a declared key has no grain", + row_name: "unkeyed", + expected_grain: None, + }, + JsonGrainsBuildTestCase { + description: "a model without inferred types is skipped", + row_name: "untyped", + expected_grain: Some(JsonGrain { + model_keys: grain_keys(&["id"]), skipped: Some("no inferred column types".to_owned()), ..JsonGrain::default() }), - ] - ); + }, + ]; + let grains: BTreeMap> = built_json_grains(); + for test_case in test_cases { + assert_eq!( + grains.get(test_case.row_name), + Some(&test_case.expected_grain), + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index 0140ad49cb..5fc59e9592 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -3,7 +3,7 @@ use crate::engine::tests::test_types::{ }; use crate::lint_text::main::preparation::prepare; use crate::query_facts::main::model_shape::model_shape; -use crate::rules::_helpers::numeric_decisions::compact_sql; +use crate::rules::_helpers::sql_checks::numeric_decisions::compact_sql; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy as rules_quote_policy; use sqlbuild_sqltext::sql_scan::models::QuotePolicy; diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index 147bd10c48..56b4ec8dbc 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -1,5 +1,7 @@ use serde_json::Value; +use crate::models::JsonGrain; + use sqlbuild_sqltext::sql_scan::models::Unclosed; pub(crate) struct NativeEvaluationTestCase { @@ -292,6 +294,13 @@ pub(crate) struct JsonGrainTestCase { pub(crate) expected_faults: &'static [(&'static str, u64, u64, &'static str)], } +/// One model's grain after `json_grains` builds the four order models. +pub(crate) struct JsonGrainsBuildTestCase { + pub(crate) description: &'static str, + pub(crate) row_name: &'static str, + pub(crate) expected_grain: Option, +} + /// A retained request evaluated again without SQL, with or without the cached findings. pub(crate) struct RetainedRowsTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs index 90f58d1765..6f943df76c 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs @@ -6,7 +6,7 @@ use fensu_native::{NativeProjectPlane, NativeRuleContext, evaluate_core_rules}; use crate::engine::_helpers::memo::{evaluate_rows, prepare}; use crate::models::{EvaluateRequest, RowsRequest, RuleMetadata, RuleScope, RulesConfig}; use crate::python_rules::tests::test_types::CachedRuns; -use crate::rules::_helpers::catalogue::{catalogue, select}; +use crate::rules::_helpers::dispatch::catalogue::{catalogue, select}; /// How many distinct `SQBRPYTHON` codes the catalogue holds. pub(super) fn distinct_python_codes(rules: &[RuleMetadata]) -> usize { diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs index 8faa13bcfb..a4ff2b7c76 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs @@ -6,7 +6,7 @@ use crate::python_rules::tests::helpers::{ use crate::python_rules::tests::test_types::{ CatalogueTestCase, CoreRuleTestCase, OptionsTestCase, SelectionTestCase, SourceKeyTestCase, }; -use crate::rules::_helpers::catalogue::catalogue; +use crate::rules::_helpers::dispatch::catalogue::catalogue; use crate::rules::main::builtin_options::builtin_options; use crate::rules::models::BuiltinOptionKind; diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs index b5fdab9b3c..7828a762ec 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs @@ -8,6 +8,25 @@ use crate::query_facts::models::{FlattenInput, WholeValue}; const VALUE_COLUMN: &str = "value"; +/// Functions whose result embeds each argument whole. +const EMBEDDING_FUNCTIONS: [&str; 5] = [ + "object_construct", + "object_construct_keep_null", + "object_insert", + "array_construct", + "array_construct_compact", +]; + +/// Conversions whose result is their first argument whole, in another type. +const CONVERSION_FUNCTIONS: [&str; 6] = [ + "to_json", + "to_char", + "to_variant", + "to_object", + "to_array", + "to_varchar", +]; + /// The `FLATTEN` call a FROM item runs, as `LATERAL FLATTEN(...)` or `TABLE(FLATTEN(...))`. pub(crate) fn flatten_call(expression: &Expression) -> Option<&Function> { match expression { @@ -83,30 +102,7 @@ fn literal_key(expression: &Expression) -> Option { (!key.is_empty()).then(|| key.trim_matches('"').to_ascii_lowercase()) } -/// Functions whose result embeds each argument whole. -const EMBEDDING_FUNCTIONS: [&str; 5] = [ - "object_construct", - "object_construct_keep_null", - "object_insert", - "array_construct", - "array_construct_compact", -]; - -/// Conversions whose result is their first argument whole, in another type. -const CONVERSION_FUNCTIONS: [&str; 6] = [ - "to_json", - "to_char", - "to_variant", - "to_object", - "to_array", - "to_varchar", -]; - -/// Every `.value` `expression` copies whole, with the keys `OBJECT_DELETE` removes from it. -/// -/// Only the value itself, casts, conversions and parentheses around it, and the constructors that -/// embed it count; any other function of the value, such as a hash or a path into it, does not -/// copy it. +/// Every `.value` that `expression` copies whole, with the keys `OBJECT_DELETE` removes. pub(crate) fn whole_values(expression: &Expression) -> Vec { let mut found: Vec = Vec::new(); let mut pending: Vec<(&Expression, Vec)> = vec![(expression, Vec::new())]; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/contract_name_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/contract_name_types.rs similarity index 100% rename from crates/sqlbuild-rules/src/rules/_helpers/contract_name_types.rs rename to crates/sqlbuild-rules/src/rules/_helpers/contracts/contract_name_types.rs diff --git a/crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/dynamic_contracts.rs similarity index 94% rename from crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs rename to crates/sqlbuild-rules/src/rules/_helpers/contracts/dynamic_contracts.rs index acf4c1e5ef..26c0781ec5 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/contracts/dynamic_contracts.rs @@ -3,8 +3,10 @@ use std::collections::BTreeMap; use crate::constants::{ENFORCED_CONTRACT, SELECT_STAR_MODEL_RULE_CODE}; use crate::models::{Fault, Model, RuleMetadata, RulesConfig}; use crate::query_facts::models::{QueryFacts, RelationFacts, SelectFacts}; -use crate::rules::_helpers::evaluation::sole_table_name; -use crate::rules::_helpers::{contract_name_types, explicit_output_types, typed_contract_columns}; +use crate::rules::_helpers::contracts::{ + contract_name_types, explicit_output_types, typed_contract_columns, +}; +use crate::rules::_helpers::dispatch::evaluation::sole_table_name; use crate::rules::models::FaultCollector; pub(crate) fn evaluate_without_query( diff --git a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/explicit_output_types.rs similarity index 99% rename from crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs rename to crates/sqlbuild-rules/src/rules/_helpers/contracts/explicit_output_types.rs index 04ada17cb7..e982824514 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/contracts/explicit_output_types.rs @@ -4,7 +4,7 @@ use sqlbuild_analysis::type_system::main::normalize_type::normalize_type; use crate::constants::ENFORCED_CONTRACT; use crate::models::{Column, Fault, Model, RuleMetadata}; use crate::query_facts::models::{ExpressionItem, ItemFacts, QueryFacts, SelectFacts, SetFacts}; -use crate::rules::_helpers::evaluation::{ +use crate::rules::_helpers::dispatch::evaluation::{ ParsedModel, plain_select, reads_dependency, sole_table_name, }; use crate::rules::models::FaultCollector; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/contracts/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/mod.rs new file mode 100644 index 0000000000..f34a766f0b --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/contracts/mod.rs @@ -0,0 +1,4 @@ +pub mod contract_name_types; +pub mod dynamic_contracts; +pub mod explicit_output_types; +pub mod typed_contract_columns; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/typed_contract_columns.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/typed_contract_columns.rs similarity index 100% rename from crates/sqlbuild-rules/src/rules/_helpers/typed_contract_columns.rs rename to crates/sqlbuild-rules/src/rules/_helpers/contracts/typed_contract_columns.rs diff --git a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/catalogue.rs similarity index 100% rename from crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs rename to crates/sqlbuild-rules/src/rules/_helpers/dispatch/catalogue.rs diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/evaluation.rs similarity index 97% rename from crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs rename to crates/sqlbuild-rules/src/rules/_helpers/dispatch/evaluation.rs index 5deb14cb8c..01bc3dc53f 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/evaluation.rs @@ -19,11 +19,13 @@ use crate::query_facts::models::{ CallKind, ColumnFact, ComparisonFacts, ItemFacts, ModelCall, ModelSource, QueryFacts, RelationFacts, SelectFacts, }; -use crate::rules::_helpers::domain_layout::folder_layer_details; -use crate::rules::_helpers::{ - contract_name_types, dynamic_contracts, explicit_output_types, flatten_parent_copy, - json_finer_grain, model_layers, numeric_decisions, sql_test_rules, typed_contract_columns, +use crate::rules::_helpers::contracts::{ + contract_name_types, dynamic_contracts, explicit_output_types, typed_contract_columns, }; +use crate::rules::_helpers::layers::domain_layout::folder_layer_details; +use crate::rules::_helpers::layers::model_layers; +use crate::rules::_helpers::performance::{flatten_parent_copy, json_finer_grain}; +use crate::rules::_helpers::sql_checks::{numeric_decisions, sql_test_rules}; use crate::rules::models::{ FaultCollector, ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride, }; @@ -272,7 +274,7 @@ macro_rules! custom_fault { }; } -pub(super) fn path_fault( +pub(in crate::rules::_helpers) fn path_fault( rule: &RuleMetadata, path: &str, message: String, @@ -362,7 +364,7 @@ fn requires_model_query(rule: &RuleMetadata) -> bool { )) } -pub(super) fn plain_select(select: &SelectFacts, allow_star: bool) -> bool { +pub(in crate::rules::_helpers) fn plain_select(select: &SelectFacts, allow_star: bool) -> bool { if select.items.is_empty() || select.filtered || !select.joins.is_empty() { return false; } @@ -390,7 +392,7 @@ fn dependency_kind(relation: &RelationFacts) -> Option { } } -pub(super) fn reads_dependency(select: &SelectFacts) -> bool { +pub(in crate::rules::_helpers) fn reads_dependency(select: &SelectFacts) -> bool { select.from.len() == 1 && select.joins.is_empty() && dependency_kind(&select.from[0]).is_some() } @@ -398,7 +400,7 @@ fn dependency_import(query: &QueryFacts) -> bool { query.root_select().is_some_and(reads_dependency) && plain_projection(query, true) } -pub(super) fn sole_table_name(query: &QueryFacts) -> Option<&str> { +pub(in crate::rules::_helpers) fn sole_table_name(query: &QueryFacts) -> Option<&str> { let select = query.root_select()?; if select.from.len() != 1 || !select.joins.is_empty() { return None; @@ -591,15 +593,15 @@ fn select_star( } #[derive(Clone)] -pub(super) struct NameParts { - pub(super) domain: String, - pub(super) layer: String, - pub(super) subject: String, - pub(super) source: Option, - pub(super) is_view: bool, +pub(in crate::rules::_helpers) struct NameParts { + pub(in crate::rules::_helpers) domain: String, + pub(in crate::rules::_helpers) layer: String, + pub(in crate::rules::_helpers) subject: String, + pub(in crate::rules::_helpers) source: Option, + pub(in crate::rules::_helpers) is_view: bool, } -pub(super) fn parse_name(name: &str) -> Option { +pub(in crate::rules::_helpers) fn parse_name(name: &str) -> Option { let parts: Vec<&str> = name.split("__").collect(); if !(3..=4).contains(&parts.len()) { return None; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/dispatch/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/mod.rs new file mode 100644 index 0000000000..508c91acbc --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/mod.rs @@ -0,0 +1,2 @@ +pub mod catalogue; +pub mod evaluation; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/domain_layout.rs b/crates/sqlbuild-rules/src/rules/_helpers/layers/domain_layout.rs similarity index 99% rename from crates/sqlbuild-rules/src/rules/_helpers/domain_layout.rs rename to crates/sqlbuild-rules/src/rules/_helpers/layers/domain_layout.rs index a39aa84b97..9871b1b1b3 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/domain_layout.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/layers/domain_layout.rs @@ -1,6 +1,6 @@ use crate::configuration::main::setting_help::setting_help; use crate::models::{DeclarationKind, Fault, Model, RuleMetadata, ScopeDeclaration, ScopeKind}; -use crate::rules::_helpers::evaluation::path_fault; +use crate::rules::_helpers::dispatch::evaluation::path_fault; use crate::rules::models::ProjectEvaluationRequest; use std::collections::{BTreeMap, BTreeSet}; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/layers/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/layers/mod.rs new file mode 100644 index 0000000000..7a9531f2f4 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/layers/mod.rs @@ -0,0 +1,2 @@ +pub mod domain_layout; +pub mod model_layers; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/layers/model_layers.rs similarity index 99% rename from crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs rename to crates/sqlbuild-rules/src/rules/_helpers/layers/model_layers.rs index c6b6addfaf..4132f1953d 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/layers/model_layers.rs @@ -8,7 +8,7 @@ use crate::constants::{ PUBLISHED_LAYERS_OPTION, REFERENCE_KIND, STAGING_LAYER_DIRECTORY, }; use crate::models::{EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; -use crate::rules::_helpers::evaluation::{parse_name, path_fault}; +use crate::rules::_helpers::dispatch::evaluation::{parse_name, path_fault}; use crate::rules::models::{FaultCollector, ProjectEvaluationRequest}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::text::main::glob_match::glob_match; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs index 9a76967985..1253a56fe9 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs @@ -1,12 +1,5 @@ -pub mod catalogue; -pub mod contract_name_types; -pub mod domain_layout; -pub mod dynamic_contracts; -pub mod evaluation; -pub mod explicit_output_types; -pub mod flatten_parent_copy; -pub mod json_finer_grain; -pub mod model_layers; -pub mod numeric_decisions; -pub mod sql_test_rules; -pub mod typed_contract_columns; +pub mod contracts; +pub mod dispatch; +pub mod layers; +pub mod performance; +pub mod sql_checks; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/flatten_parent_copy.rs b/crates/sqlbuild-rules/src/rules/_helpers/performance/flatten_parent_copy.rs similarity index 69% rename from crates/sqlbuild-rules/src/rules/_helpers/flatten_parent_copy.rs rename to crates/sqlbuild-rules/src/rules/_helpers/performance/flatten_parent_copy.rs index 7f16054252..bd5a878ccf 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/flatten_parent_copy.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/performance/flatten_parent_copy.rs @@ -6,43 +6,45 @@ use crate::models::{Fault, RuleMetadata}; use crate::query_facts::models::{ ExpressionItem, FlattenInput, ItemFacts, QueryFacts, RelationFacts, SelectFacts, SetFacts, }; -use crate::rules::_helpers::evaluation::ParsedModel; +use crate::rules::_helpers::dispatch::evaluation::ParsedModel; use crate::rules::models::FaultCollector; const VALUE_COLUMN: &str = "value"; pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &FaultCollector) { - let mut selects: Vec<&SelectFacts> = Vec::new(); - collect_query(parsed.facts, &mut selects); - for select in selects { + for select in query_selects(parsed.facts) { let children: BTreeMap<&str, Vec>> = flattened_parents(select); for item in &select.items { let ItemFacts::Expression(expression) = item else { continue; }; - if let Some((parent, key)) = copied_parent(expression, &children) { - push(parsed, rule, faults, expression, parent, key); + if let Some(copied) = copied_parent(expression, &children) { + faults.push(fault(parsed, rule, expression, copied)); } } } } -fn collect_query<'a>(query: &'a QueryFacts, selects: &mut Vec<&'a SelectFacts>) { +/// Every SELECT in `query`, its CTEs first. +fn query_selects(query: &QueryFacts) -> Vec<&SelectFacts> { + let mut selects: Vec<&SelectFacts> = Vec::new(); for cte in &query.ctes { - collect_query(&cte.query, selects); + selects.extend(query_selects(&cte.query)); } - collect_set(&query.body, selects); + selects.extend(set_selects(&query.body)); + selects } -fn collect_set<'a>(set: &'a SetFacts, selects: &mut Vec<&'a SelectFacts>) { +fn set_selects(set: &SetFacts) -> Vec<&SelectFacts> { match set { - SetFacts::Select(select) => selects.push(select), - SetFacts::Query(query) => collect_query(query, selects), + SetFacts::Select(select) => vec![select], + SetFacts::Query(query) => query_selects(query), SetFacts::SetOperation { left, right, .. } => { - collect_set(left, selects); - collect_set(right, selects); + let mut selects: Vec<&SelectFacts> = set_selects(left); + selects.extend(set_selects(right)); + selects } - SetFacts::Other => {} + SetFacts::Other => Vec::new(), } } @@ -87,21 +89,29 @@ fn copied_parent<'a>( ) -> Option<(&'a str, Option<&'a str>)> { expression.whole_values.iter().find_map(|whole| { let keys = parents.get(whole.qualifier.as_str())?; - let kept = keys.iter().find(|key| { - key.is_none_or(|key| !whole.removed.iter().any(|removed| removed == key)) - })?; - Some((whole.qualifier.as_str(), *kept)) + let kept: Option<&str> = kept_key(keys, &whole.removed)?; + Some((whole.qualifier.as_str(), kept)) }) } -fn push( +/// The first child key that `removed` does not take out of the parent; `None` reads it all. +fn kept_key<'a>(keys: &[Option<&'a str>], removed: &[String]) -> Option> { + for key in keys { + match key { + None => return Some(None), + Some(key) if !removed.iter().any(|name| name == key) => return Some(Some(key)), + Some(_) => {} + } + } + None +} + +fn fault( parsed: &ParsedModel<'_>, rule: &RuleMetadata, - faults: &FaultCollector, expression: &ExpressionItem, - parent: &str, - key: Option<&str>, -) { + (parent, key): (&str, Option<&str>), +) -> Fault { let (line, column) = expression .start .map_or((1, 1), |offset| parsed.source.line_column(offset)); @@ -110,7 +120,7 @@ fn push( || "its child array".to_owned(), |key| format!("{parent}.value:{key}"), ); - faults.push(Fault { + Fault { unevaluated: false, code: rule.code.clone(), path: parsed.model.relative_path.clone(), @@ -120,5 +130,5 @@ fn push( "{output} copies {parent}.value whole onto every row flattened from {child}" ), remediation: rule.remediation.clone(), - }); + } } diff --git a/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs b/crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs similarity index 54% rename from crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs rename to crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs index 92c4bce3d5..ce0c46a5d6 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/json_finer_grain.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs @@ -2,8 +2,8 @@ use crate::models::{Fault, GrainColumn, JsonGrain, RuleMetadata, RulesConfig}; use crate::query_facts::models::{ItemFacts, SetFacts}; -use crate::rules::_helpers::evaluation::ParsedModel; -use crate::rules::_helpers::model_layers::backs_published_view; +use crate::rules::_helpers::dispatch::evaluation::ParsedModel; +use crate::rules::_helpers::layers::model_layers::backs_published_view; use crate::rules::models::FaultCollector; pub(crate) fn evaluate( @@ -43,60 +43,74 @@ pub(crate) fn evaluate( } } -/// An input of `column` whose key is a strict subset of one of the model's keys. -/// -/// An input with any key that covers a model key does not count: that model key proves the -/// model has one row per input row. +/// An input whose key is a strict subset of a model key, unless another of its keys covers one. fn coarser_input<'a>( column: &'a GrainColumn, grain: &'a JsonGrain, ) -> Option<(&'a str, &'a [String], &'a [String])> { - column.inputs.iter().find_map(|input| { - let keys = grain.input_keys.get(input)?; - if keys.iter().any(|input_key| { - grain - .model_keys - .iter() - .any(|model_key| subset(model_key, input_key)) - }) { - return None; + for input in &column.inputs { + let Some(keys) = grain.input_keys.get(input) else { + continue; + }; + let one_model_row_per_input_row = keys + .iter() + .any(|input_key| covers_model_key(input_key, &grain.model_keys)); + if one_model_row_per_input_row { + continue; + } + for input_key in keys { + if let Some(model_key) = finer_model_key(input_key, &grain.model_keys) { + return Some((input.as_str(), input_key.as_slice(), model_key)); + } } - keys.iter().find_map(|input_key| { - grain - .model_keys - .iter() - .find(|model_key| strict_subset(input_key, model_key)) - .map(|model_key| (input.as_str(), input_key.as_slice(), model_key.as_slice())) - }) - }) + } + None +} + +fn covers_model_key(input_key: &[String], model_keys: &[Vec]) -> bool { + model_keys + .iter() + .any(|model_key| subset(model_key, input_key)) } -fn strict_subset(smaller: &[String], larger: &[String]) -> bool { - smaller.len() < larger.len() && subset(smaller, larger) +fn finer_model_key<'a>( + input_key: &[String], + model_keys: &'a [Vec], +) -> Option<&'a [String]> { + model_keys + .iter() + .find(|model_key| input_key.len() < model_key.len() && subset(input_key, model_key)) + .map(Vec::as_slice) } fn subset(smaller: &[String], larger: &[String]) -> bool { - smaller.iter().all(|column| { - larger - .iter() - .any(|other| other.eq_ignore_ascii_case(column)) - }) + smaller + .iter() + .all(|column| contains_ignoring_case(larger, column)) +} + +fn contains_ignoring_case(columns: &[String], name: &str) -> bool { + columns + .iter() + .any(|column| column.eq_ignore_ascii_case(name)) } /// Where the root select projects `name`, following the left branch of set operations. fn output_start(body: &SetFacts, name: &str) -> Option { match body { - SetFacts::Select(select) => select.items.iter().find_map(|item| match item { - ItemFacts::Expression(expression) - if expression - .output_name - .as_deref() - .is_some_and(|output| output.eq_ignore_ascii_case(name)) => - { - expression.start + SetFacts::Select(select) => { + for item in &select.items { + if let ItemFacts::Expression(expression) = item + && expression + .output_name + .as_deref() + .is_some_and(|output| output.eq_ignore_ascii_case(name)) + { + return expression.start; + } } - _ => None, - }), + None + } SetFacts::Query(query) => output_start(&query.body, name), SetFacts::SetOperation { left, .. } => output_start(left, name), SetFacts::Other => None, diff --git a/crates/sqlbuild-rules/src/rules/_helpers/performance/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/performance/mod.rs new file mode 100644 index 0000000000..0fd5391a33 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/performance/mod.rs @@ -0,0 +1,2 @@ +pub mod flatten_parent_copy; +pub mod json_finer_grain; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/mod.rs new file mode 100644 index 0000000000..5957b15086 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/mod.rs @@ -0,0 +1,2 @@ +pub mod numeric_decisions; +pub mod sql_test_rules; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/numeric_decisions.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/numeric_decisions.rs similarity index 100% rename from crates/sqlbuild-rules/src/rules/_helpers/numeric_decisions.rs rename to crates/sqlbuild-rules/src/rules/_helpers/sql_checks/numeric_decisions.rs diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/sql_test_rules.rs similarity index 99% rename from crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs rename to crates/sqlbuild-rules/src/rules/_helpers/sql_checks/sql_test_rules.rs index f53e86786c..d8ac16edd4 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/sql_test_rules.rs @@ -7,7 +7,7 @@ use crate::query_facts::main::model_shape::model_shape; use crate::query_facts::main::parse_fixture::parse_fixture; use crate::query_facts::main::stub_call::stub_call; use crate::query_facts::models::{CallKind, ParsedFixture}; -use crate::rules::_helpers::evaluation::{path_fault, rules_unavailable}; +use crate::rules::_helpers::dispatch::evaluation::{path_fault, rules_unavailable}; use crate::rules::models::ProjectEvaluationRequest; use serde::{Deserialize, Serialize}; use sqlbuild_analysis::compiler::main::sql_test_fixture_facts::fixture_facts; diff --git a/crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs b/crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs index 3ddf9d2bc3..6125301736 100644 --- a/crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs @@ -1,5 +1,5 @@ use crate::models::{CustomRule, RuleMetadata}; pub(crate) fn assemble_catalogue(custom: &[CustomRule]) -> Result, String> { - crate::rules::_helpers::catalogue::with_custom(custom) + crate::rules::_helpers::dispatch::catalogue::with_custom(custom) } diff --git a/crates/sqlbuild-rules/src/rules/main/catalogue.rs b/crates/sqlbuild-rules/src/rules/main/catalogue.rs index 88c4b1aeac..c3646042fa 100644 --- a/crates/sqlbuild-rules/src/rules/main/catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/main/catalogue.rs @@ -1,5 +1,5 @@ use crate::models::RuleMetadata; pub fn catalogue() -> Vec { - crate::rules::_helpers::catalogue::catalogue() + crate::rules::_helpers::dispatch::catalogue::catalogue() } diff --git a/crates/sqlbuild-rules/src/rules/main/compact_sql.rs b/crates/sqlbuild-rules/src/rules/main/compact_sql.rs index a1fa2a7c57..5f7909a1cb 100644 --- a/crates/sqlbuild-rules/src/rules/main/compact_sql.rs +++ b/crates/sqlbuild-rules/src/rules/main/compact_sql.rs @@ -2,5 +2,5 @@ use sqlbuild_sqltext::sql_scan::models::QuotePolicy; /// `sql` lowercased without whitespace or comments outside quoted text. pub(crate) fn compact_sql(sql: &str, policy: QuotePolicy) -> String { - crate::rules::_helpers::numeric_decisions::compact_sql(sql, policy) + crate::rules::_helpers::sql_checks::numeric_decisions::compact_sql(sql, policy) } diff --git a/crates/sqlbuild-rules/src/rules/main/evaluate.rs b/crates/sqlbuild-rules/src/rules/main/evaluate.rs index 5b8dfe173a..3d87e61504 100644 --- a/crates/sqlbuild-rules/src/rules/main/evaluate.rs +++ b/crates/sqlbuild-rules/src/rules/main/evaluate.rs @@ -2,5 +2,5 @@ use crate::models::Fault; use crate::rules::models::ModelEvaluationRequest; pub(crate) fn evaluate_model(request: ModelEvaluationRequest<'_>) -> Result, String> { - crate::rules::_helpers::evaluation::evaluate_model(request) + crate::rules::_helpers::dispatch::evaluation::evaluate_model(request) } diff --git a/crates/sqlbuild-rules/src/rules/main/evaluate_project.rs b/crates/sqlbuild-rules/src/rules/main/evaluate_project.rs index 0154a81dba..a689f19a9f 100644 --- a/crates/sqlbuild-rules/src/rules/main/evaluate_project.rs +++ b/crates/sqlbuild-rules/src/rules/main/evaluate_project.rs @@ -6,15 +6,11 @@ use crate::rules::models::ProjectEvaluationRequest; pub(crate) fn evaluate_project( request: ProjectEvaluationRequest<'_>, ) -> Result, String> { - let mut faults = crate::rules::_helpers::domain_layout::evaluate_project(&request); - faults.extend(crate::rules::_helpers::evaluation::evaluate_project_rules( - &request, - )); + let mut faults = crate::rules::_helpers::layers::domain_layout::evaluate_project(&request); + faults.extend(crate::rules::_helpers::dispatch::evaluation::evaluate_project_rules(&request)); if let Some(rule) = request.selected.get(NAMED_SELECTOR_LIMIT_CODE) { faults.extend(evaluate_orchestration_rules(request.request, rule)); } - faults.extend(crate::rules::_helpers::sql_test_rules::evaluate_project( - request, - )?); + faults.extend(crate::rules::_helpers::sql_checks::sql_test_rules::evaluate_project(request)?); Ok(faults) } diff --git a/crates/sqlbuild-rules/src/rules/main/fingerprint.rs b/crates/sqlbuild-rules/src/rules/main/fingerprint.rs index 8799a90c04..feea9f4275 100644 --- a/crates/sqlbuild-rules/src/rules/main/fingerprint.rs +++ b/crates/sqlbuild-rules/src/rules/main/fingerprint.rs @@ -5,5 +5,5 @@ pub(crate) fn fingerprint( config: &RulesConfig, dialect: &str, ) -> Result { - crate::rules::_helpers::catalogue::fingerprint(rules, config, dialect) + crate::rules::_helpers::dispatch::catalogue::fingerprint(rules, config, dialect) } diff --git a/crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs b/crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs index a48c151ed0..d04072582f 100644 --- a/crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs +++ b/crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs @@ -4,5 +4,5 @@ use crate::rules::models::ResolvedThresholdOverride; pub(crate) fn resolve_threshold_overrides( config: &RulesConfig, ) -> Result, String> { - crate::rules::_helpers::evaluation::resolve_threshold_overrides(config) + crate::rules::_helpers::dispatch::evaluation::resolve_threshold_overrides(config) } diff --git a/crates/sqlbuild-rules/src/rules/main/select.rs b/crates/sqlbuild-rules/src/rules/main/select.rs index 45f8eef747..8fb6cc208e 100644 --- a/crates/sqlbuild-rules/src/rules/main/select.rs +++ b/crates/sqlbuild-rules/src/rules/main/select.rs @@ -5,5 +5,5 @@ pub(crate) fn select<'a>( selected: Option<&[String]>, ignored: &[String], ) -> Result, String> { - crate::rules::_helpers::catalogue::select(catalogue, selected, ignored) + crate::rules::_helpers::dispatch::catalogue::select(catalogue, selected, ignored) } diff --git a/crates/sqlbuild-rules/src/rules/main/selected_codes.rs b/crates/sqlbuild-rules/src/rules/main/selected_codes.rs index d2c4ea82e5..6a4604beae 100644 --- a/crates/sqlbuild-rules/src/rules/main/selected_codes.rs +++ b/crates/sqlbuild-rules/src/rules/main/selected_codes.rs @@ -1,3 +1,3 @@ pub fn selected_codes_json(request_json: &str) -> Result { - crate::rules::_helpers::catalogue::selected_codes_json(request_json) + crate::rules::_helpers::dispatch::catalogue::selected_codes_json(request_json) } diff --git a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs index fe88b7198c..e52ad46cb9 100644 --- a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs +++ b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs @@ -1,7 +1,7 @@ //! `selected_sql_codes`: the built-in SQL Rule codes a `select`/`ignore` pair enables. use crate::models::RulesCodeGrammar; -use crate::rules::_helpers::catalogue::{matches_any_selector, sql_lint_selection}; +use crate::rules::_helpers::dispatch::catalogue::{matches_any_selector, sql_lint_selection}; use crate::rules::main::catalogue::catalogue; /// SQL Rule codes `select` enables (SQLClean's defaults when it is `None`) that no `ignore` selector matches, in catalogue order. diff --git a/crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs b/crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs index 8f79e2c170..90bd8903a2 100644 --- a/crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs +++ b/crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs @@ -1,5 +1,5 @@ use crate::models::EvaluateRequest; pub(crate) fn annotate(request: EvaluateRequest) -> EvaluateRequest { - crate::rules::_helpers::sql_test_rules::annotate_empty_input_only_tests(request) + crate::rules::_helpers::sql_checks::sql_test_rules::annotate_empty_input_only_tests(request) } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index c446adfa85..6d854b9e56 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -220,6 +220,15 @@ def project_facts(*, project: Project, ctx: RuleContext) -> list[Finding]: ) +_PERFORMANCE_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/performance_rules") +_PERFORMANCE_EXPECTED: dict[str, Any] = json.loads( + (_PERFORMANCE_FIXTURE / "expected_findings.json").read_text(encoding="utf-8") +) +_PERFORMANCE_FINDINGS: tuple[tuple[str, str, int, int], ...] = tuple( + (code, path, line, column) for code, path, line, column in _PERFORMANCE_EXPECTED["findings"] +) + + @pytest.mark.parametrize( "test_case", [ @@ -3696,15 +3705,6 @@ def test_given_orchestration_selections_when_compiling_then_named_selector_limit assert test_case.expected_text in json.dumps(diagnostics) -_PERFORMANCE_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/performance_rules") -_PERFORMANCE_EXPECTED: dict[str, Any] = json.loads( - (_PERFORMANCE_FIXTURE / "expected_findings.json").read_text(encoding="utf-8") -) -_PERFORMANCE_FINDINGS: tuple[tuple[str, str, int, int], ...] = tuple( - (code, path, line, column) for code, path, line, column in _PERFORMANCE_EXPECTED["findings"] -) - - @pytest.mark.parametrize( "test_case", [ From 0a8b45300c698aa188c060dd49c494c275bd10aa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:28:59 +0000 Subject: [PATCH 10/17] chore: raise the bindings crate cap for the JSON grain rule plumbing --- fensu.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fensu.toml b/fensu.toml index dd9592d42d..98dc6b1454 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1703,7 +1703,8 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 206367 } +# Raised from 206,367: SQBRPERFORMANCE102 bindings plumbing, +77 with sqlbuild-python-base. +thresholds = { max_crate_code_lines = 206439 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] From d1ecf847ceafe930d8f175494575859878c3ad69 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:37:02 +0000 Subject: [PATCH 11/17] docs: regenerate the docs skill for performance rules --- .../sqlbuild/references/docs/CONTENTS.md | 1 + .../references/docs/concepts/rules.md | 5 +- .../docs/concepts/rules/performance-rules.md | 128 ++++++++++++++++++ 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 6073b2a6e5..12ccec2ff4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -50,6 +50,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Findings and exceptions](concepts/rules/findings-and-exceptions.md) (`concepts/rules/findings-and-exceptions`) - Understand Rule diagnostics and record intentional exceptions safely. - [Python Rules](concepts/rules/python-rules.md) (`concepts/rules/python-rules`) - Check the project's Python for annotations, imports, function shape and the layout of python/. - [Orchestration Rules](concepts/rules/orchestration-rules.md) (`concepts/rules/orchestration-rules`) - Keep job and sensor selections from listing nodes one by one. +- [Performance Rules](concepts/rules/performance-rules.md) (`concepts/rules/performance-rules`) - Catch semi-structured payloads that repeat on every row and multiply storage and scan cost. - [Execution and caching](concepts/rules/execution-and-caching.md) (`concepts/rules/execution-and-caching`) - Understand Rule enforcement order, focused runs, and dependency-aware reuse. ## Ship diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md index 89a8c77fc8..95b637cf5e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md @@ -165,7 +165,9 @@ with `SQBR`. The `SQBRPYTHON` Rules check the project's Python: annotations, imp shape and the layout of `python/`. They run only when `select` names `SQBRPYTHON` or one of its codes; see [Python Rules](rules/python-rules.md). `SQBRORCHESTRATION101` keeps job and sensor selections from listing nodes one by one; see -[Orchestration Rules](rules/orchestration-rules.md). +[Orchestration Rules](rules/orchestration-rules.md). The `SQBRPERFORMANCE` Rules find +Snowflake JSON payloads that repeat on every row; they run only when `select` names their codes. See +[Performance Rules](rules/performance-rules.md). Projects can define custom Python Rules under `rules/**/*.py`. Their codes begin with `XSQBR`, for example `XSQBRARCH001`. Custom Rules use the same selection, findings, exceptions, execution order, @@ -250,5 +252,6 @@ Rules add about 14% to a cold compile and nothing to a warm one. See Understand diagnostics, exact exceptions, path ignores, and stale checks. Check the project's Python for annotations, imports, function shape and the layout of `python/`. Keep each domain's published layer free of consumer-specific variants. + Catch Snowflake JSON payloads copied onto every row of a finer grain. Learn evaluation order, focused execution, and dependency-aware reuse. List, inspect, run, and generate guidance for configured Rules. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md new file mode 100644 index 0000000000..7cdea22dde --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md @@ -0,0 +1,128 @@ + + +# Performance Rules + +> Catch semi-structured payloads that repeat on every row and multiply storage and scan cost. + +Online: https://sqlbuild.com/docs/concepts/rules/performance-rules/ + +## Contents + +- SQBRPERFORMANCE101: a flattened parent copied onto its children +- SQBRPERFORMANCE102: a payload copied onto a finer grain +- Details + +The `SQBRPERFORMANCE` Rules find JSON payloads copied onto rows that are finer than the payload +itself. Each copy stores and scans the same value again, so the cost grows with the row count. +Name their codes in `select` to turn them on: + +```toml +[rules] +select = ["SQBRPERFORMANCE101", "SQBRPERFORMANCE102"] +``` + +Both Rules are off by default: selecting the `SQBRPERFORMANCE` family alone does not run them. They +run on the Snowflake dialect. + +| Code | Name | Reports | +|---|---|---| +| `SQBRPERFORMANCE101` | `flatten-parent-copy` | A FLATTEN element copied whole onto the rows of a deeper FLATTEN of its own child array | +| `SQBRPERFORMANCE102` | `json-finer-grain` | A `VARIANT`, `OBJECT` or `ARRAY` column copied from an input with a coarser unique key | + +## SQBRPERFORMANCE101: a flattened parent copied onto its children + +An output column must not copy the `value` of a FLATTEN whose child array a deeper FLATTEN in the +same query reads. Every child row then carries its whole parent, children included: + +```sql +SELECT + r.order_id, + o.value AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l +``` + +```text +error[SQBRPERFORMANCE101]: line_json copies o.value whole onto every row flattened from o.value:options + --> models/line_options.sql:7:3 +``` + +The parent also counts as copied through a cast, a conversion such as `TO_JSON` or `TO_VARIANT`, +or a constructor that embeds it: `OBJECT_CONSTRUCT`, `OBJECT_INSERT` or `ARRAY_CONSTRUCT`. Paths +into it, such as `o.value:sku`, and other functions of it, such as `MD5(TO_VARCHAR(o.value))`, are +not copies. + +Read the fields you need from the parent, such as `o.value:sku::string`, or remove the child array +before copying the rest: + +```sql +SELECT + r.order_id, + OBJECT_DELETE(o.value, 'options') AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l +``` + +## SQBRPERFORMANCE102: a payload copied onto a finer grain + +A `VARIANT`, `OBJECT` or `ARRAY` output column must not come from an input whose unique key is a +strict subset of the model's unique key. Here `orders` has one row per `order_id` and the model has +one row per `(order_id, line_no)`, so each order's payload repeats on every line: + +```sql +SELECT + l.order_id, + l.line_no, + l.sku, + o.payload AS order_payload +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id +``` + +```text +error[SQBRPERFORMANCE102]: order_payload copies VARIANT from orders, one row per (order_id), onto rows keyed by (order_id, line_no), so the same value repeats on every row + --> models/order_lines_enriched.sql:12:3 +``` + +Read the fields the model needs, such as `o.payload:customer::string`, or leave the payload on the +input and join it where it is used. + +The Rule compares the unique keys SQLBuild knows from each relation's declaration: + +| Declaration | Key | +|---|---| +| A column with `nullable false` (or a `not_null` audit) and a `unique` audit | That column | +| An incremental model with `unique_key` and the `merge` strategy | The `unique_key` columns | +| A seed whose file holds at most one data row | No columns | + +A model with another declared key that one of the input's keys covers is not reported: that key already +proves one row per input row. An incremental model with `unique_key [order_id, order_date]` and a +unique, not-null `order_id` column has one row per order. + +Aggregated columns, such as `ARRAY_AGG(l.sku)`, are not copies and are not reported. + +### Published views + +A model whose only readers are published views of its own entity is not reported: the view +publishes the same grain on purpose. The views follow the +[SQBRGRAPH103 allowance](published-layer-rules.md#sqbrgraph103-no-published-reads-within-a-domain): +`shop__mart_v__order_lines` reading `shop__mart__order_lines`. Its `published_layers` option sets +which layers count. + +### Skipped models + +`SQBRPERFORMANCE102` needs column lineage and inferred column types. A model without them is +skipped, not reported. The run counts skipped models in a note: + +```text +note: SQBRPERFORMANCE102 skipped 2 models without column lineage or inferred types; `sqb plan --verbose` lists them +``` + +`sqb plan --verbose` and `sqb build --verbose` list each skipped model with the missing fact. + +## Details + +- `SQBRPERFORMANCE101` reports the select item that copies the parent, inside a CTE or subquery + when the copy is made there. +- `SQBRPERFORMANCE102` reports the column where the root query projects it. +- `SQBRPERFORMANCE102` reports nothing for a model without a declared unique key. From c1243a0ce960e6763e43056063f0d4a1892ce837 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:40:10 +0000 Subject: [PATCH 12/17] test: sort the performance rules test case import --- tests/integration/src/sqlbuild/cli/commands/main/test_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 6d854b9e56..fce0de2a8f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -36,8 +36,8 @@ ExplicitContractOutputRuleIntegrationTestCase, ImplicitAliasRuleIntegrationTestCase, NamedSelectorIntegrationTestCase, - PerformanceRulesIntegrationTestCase, NumericRangeDecisionIntegrationTestCase, + PerformanceRulesIntegrationTestCase, ProjectRuleScratchStateTestCase, PythonRuleCacheSequenceTestCase, PythonRulesIntegrationTestCase, From cd1f9145d46e375457fbfa8e2e1f02aec4570891 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:50:33 +0000 Subject: [PATCH 13/17] fix: report a JSON grain finding at the first positioned output column --- .../src/rules/_helpers/performance/json_finer_grain.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs b/crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs index ce0c46a5d6..c3ac43fd87 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs @@ -105,6 +105,7 @@ fn output_start(body: &SetFacts, name: &str) -> Option { .output_name .as_deref() .is_some_and(|output| output.eq_ignore_ascii_case(name)) + && expression.start.is_some() { return expression.start; } From ffb1e3150a57988d69dd38bb4d7039a981ced6ca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:28:16 +0000 Subject: [PATCH 14/17] test: select the opt-in performance rules in the dense compile benchmark --- scripts/cold_compile_performance/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/cold_compile_performance/constants.py b/scripts/cold_compile_performance/constants.py index 63ed73b060..24f5e5f145 100644 --- a/scripts/cold_compile_performance/constants.py +++ b/scripts/cold_compile_performance/constants.py @@ -4,7 +4,8 @@ COMPILE_MEMORY_LIMIT_ENV_VAR: str = "SQLBUILD_COMPILE_MEMORY_LIMIT_BYTES" COMPILE_MEASUREMENT_FIELD_COUNT: int = 6 DENSE_RULE_SELECT: str = ( - 'select = ["SQBR", "SQBRPYTHON", "SQBRPYTHON306", "SQBRPYTHON507", "SQLCLEAN", "XSQBR"]' + 'select = ["SQBR", "SQBRPERFORMANCE101", "SQBRPERFORMANCE102", "SQBRPYTHON", ' + '"SQBRPYTHON306", "SQBRPYTHON507", "SQLCLEAN", "XSQBR"]' ) DENSE_JOIN_THRESHOLDS: tuple[int, ...] = (55, 75, 90, 99) From fd90a00b346cdcfbd8dcc9916f1da37d573134cf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:10:30 +0000 Subject: [PATCH 15/17] test: keep the opt-in performance rules out of the dense edit benchmarks SQBRPERFORMANCE102 compares models across the project, so a Snowflake compile that selects it never takes the delta path. Selecting it in the shared dense project made the delta edit loop and the base/head edit ratio measure full compiles. The default dense selection is back to the one main uses, and only the all-rules cold benchmark and the varied benchmark select both performance rules. --- .../cold_compile_performance/_helpers/dense_project.py | 6 ++++-- .../cold_compile_performance/_helpers/varied_project.py | 5 ++++- scripts/cold_compile_performance/constants.py | 9 +++++++-- .../main/compile/test_dense_compile_performance.py | 7 ++++++- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/scripts/cold_compile_performance/_helpers/dense_project.py b/scripts/cold_compile_performance/_helpers/dense_project.py index 608daf5468..abdd22580b 100644 --- a/scripts/cold_compile_performance/_helpers/dense_project.py +++ b/scripts/cold_compile_performance/_helpers/dense_project.py @@ -67,7 +67,9 @@ def dense_column_count(index: int) -> int: return 256 + (rank - 990) * 24 -def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: +def write_dense_compile_project( + *, project_dir: Path, model_count: int, rule_select: str = DENSE_RULE_SELECT +) -> None: """Write a deterministic query-heavy project without external inputs or services.""" project_dir.mkdir(parents=True) @@ -77,7 +79,7 @@ def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: (project_dir / "sqlbuild_project.toml").write_text( 'name = "dense_orders"\nadapter = "snowflake"\n' '[defaults]\ndatabase = "warehouse"\nschema = "analytics"\n' - f"[rules]\n{DENSE_RULE_SELECT}\ndomains = [{domains}]\n", + f"[rules]\n{rule_select}\ndomains = [{domains}]\n", encoding="utf-8", ) source_count: int = max(1, model_count * 24 // 100) diff --git a/scripts/cold_compile_performance/_helpers/varied_project.py b/scripts/cold_compile_performance/_helpers/varied_project.py index 7ef058e0e3..90be85a03a 100644 --- a/scripts/cold_compile_performance/_helpers/varied_project.py +++ b/scripts/cold_compile_performance/_helpers/varied_project.py @@ -8,6 +8,7 @@ write_dense_compile_project, ) from scripts.cold_compile_performance.constants import ( + DENSE_ALL_RULE_SELECT, VARIED_ARRAY_VARIANT, VARIED_CASE_SHARE, VARIED_GROUP_SHARE, @@ -32,7 +33,9 @@ def write_varied_compile_project(*, project_dir: Path, model_count: int) -> None: """Write independent neutral SQL with diverse expressions and shared ancestors.""" - write_dense_compile_project(project_dir=project_dir, model_count=model_count) + write_dense_compile_project( + project_dir=project_dir, model_count=model_count, rule_select=DENSE_ALL_RULE_SELECT + ) for index, model in enumerate(sorted((project_dir / "models").rglob("*.sql"))): contents: str = model.read_text(encoding="utf-8") replacements: dict[str, str] = _dependency_replacements( diff --git a/scripts/cold_compile_performance/constants.py b/scripts/cold_compile_performance/constants.py index 24f5e5f145..583592fd47 100644 --- a/scripts/cold_compile_performance/constants.py +++ b/scripts/cold_compile_performance/constants.py @@ -4,8 +4,13 @@ COMPILE_MEMORY_LIMIT_ENV_VAR: str = "SQLBUILD_COMPILE_MEMORY_LIMIT_BYTES" COMPILE_MEASUREMENT_FIELD_COUNT: int = 6 DENSE_RULE_SELECT: str = ( - 'select = ["SQBR", "SQBRPERFORMANCE101", "SQBRPERFORMANCE102", "SQBRPYTHON", ' - '"SQBRPYTHON306", "SQBRPYTHON507", "SQLCLEAN", "XSQBR"]' + 'select = ["SQBR", "SQBRPYTHON", "SQBRPYTHON306", "SQBRPYTHON507", "SQLCLEAN", "XSQBR"]' +) +# SQBRPERFORMANCE102 compares models across the project, so a Snowflake compile that selects it +# never takes the delta path. Edit and base/head benchmarks leave the opt-in performance rules +# out; the all-rules benchmarks select them. +DENSE_ALL_RULE_SELECT: str = DENSE_RULE_SELECT.replace( + '"SQBR", ', '"SQBR", "SQBRPERFORMANCE101", "SQBRPERFORMANCE102", ' ) DENSE_JOIN_THRESHOLDS: tuple[int, ...] = (55, 75, 90, 99) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index a6f4866ae2..bc10c8923f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -12,6 +12,7 @@ dense_test_path, write_dense_compile_project, ) +from scripts.cold_compile_performance.constants import DENSE_ALL_RULE_SELECT from scripts.cold_compile_performance.main.require_compile_memory_scope import ( require_compile_memory_scope, ) @@ -97,7 +98,11 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ wall_budget: float = test_case.expected_max_wall_seconds * measure_runner_calibration().factor print(f"dense cold models={test_case.model_count} wall_budget={wall_budget:.2f}s", flush=True) project_dir: Path = tmp_path / "dense_orders" - write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) + write_dense_compile_project( + project_dir=project_dir, + model_count=test_case.model_count, + rule_select=DENSE_ALL_RULE_SELECT, + ) config: RulesConfig = load_rules_config(project_dir=project_dir) catalogue: tuple[Rule, ...] = build_rule_catalogue( config=config, project_dir=project_dir, include_custom=True From 2b132220e060ae10c28340c478a9b48864ecaae0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:17:40 +0000 Subject: [PATCH 16/17] docs: say that SQBRPERFORMANCE102 turns off delta compiles on Snowflake Selecting the rule on the Snowflake dialect makes every edit compile the whole project. The compile page lists it among the full-compile cases and the performance rules page says so where it explains selecting the rules. A delta compile test covers both sides: the rules selected on DuckDB keep the delta, and on Snowflake the fallback names SQBRPERFORMANCE102. --- .../sqlbuild/references/docs/cli/compile.md | 1 + .../docs/concepts/rules/performance-rules.md | 4 ++- .../main/compile/test_compile_delta.py | 29 +++++++++++++++++++ website/src/content/docs/docs/cli/compile.mdx | 1 + .../docs/concepts/rules/performance-rules.mdx | 4 ++- 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index e99ccba08a..c777522ee9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -310,6 +310,7 @@ Compile runs in full instead when: a model-subject Rule must run again for a model that was not edited (for example because it reads an edited upstream file), or reads more for an edited model than that model's SQL, columns, contract and grain, project files and the source list; +- the project selects `SQBRPERFORMANCE102` on the Snowflake dialect; - the previous compile failed its Rules, since that compile writes no SQL tests, functions or audits to `target/compiled/`, or it reported other errors or warnings in files the edit didn't change; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md index 7cdea22dde..f1f4d02b2d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md @@ -22,7 +22,9 @@ select = ["SQBRPERFORMANCE101", "SQBRPERFORMANCE102"] ``` Both Rules are off by default: selecting the `SQBRPERFORMANCE` family alone does not run them. They -run on the Snowflake dialect. +run on the Snowflake dialect. There, selecting `SQBRPERFORMANCE102` makes every edit compile the +whole project, because a [delta compile](../../cli/compile.md#compiling-after-an-edit) cannot recheck +the models that read or are read by the edited ones. | Code | Name | Reports | |---|---|---| diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 87b79f8819..f560599079 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -210,6 +210,19 @@ def order_folder_siblings(*, model: Model, ctx: RuleContext) -> list[Finding]: 'select = ["XSQBR"]', ) _RULE_READ_MORE: str = "delta compile fell back: a custom rule read more than the changed models" +_PERFORMANCE_RULES_ON: tuple[str, str, str] = ( + "sqlbuild_project.toml", + 'select = ["SQBR", "SQLCLEAN", "XSQBR"]', + 'select = ["SQBR", "SQBRPERFORMANCE101", "SQBRPERFORMANCE102", "SQLCLEAN", "XSQBR"]', +) +_SNOWFLAKE_ADAPTER: tuple[str, str, str] = ( + "sqlbuild_project.toml", + 'adapter = "duckdb"', + 'adapter = "snowflake"', +) +_JSON_GRAINS_READ_ACROSS: str = ( + "delta compile fell back: SQBRPERFORMANCE102 compares keys and readers across models" +) _FAMILY_TYPE_EDIT: tuple[str, str, str] = (_CATEGORY_PIVOT_PATH, "type DOUBLE", "type VARCHAR") _FAMILIES_CHANGED: str = "delta compile fell back: a model's dynamic column families changed" _EXAMPLE_TRACKED_VARIABLE: str = "SQLBUILD_ORDERS_EXAMPLE" @@ -535,6 +548,22 @@ def test_given_project_shape_when_compiling_an_edit_then_it_prints_what_a_full_c env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, expected_stderr_line=_RULE_READ_MORE, ), + DeltaCompileVerifyTestCase( + description="the performance rules off Snowflake keep the delta", + files=(), + stored_edits=(_PERFORMANCE_RULES_ON,), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=DELTA_VERIFIED_LINE, + ), + DeltaCompileVerifyTestCase( + description="SQBRPERFORMANCE102 on Snowflake names why it fell back", + files=(), + stored_edits=(_PERFORMANCE_RULES_ON, _SNOWFLAKE_ADAPTER), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=_JSON_GRAINS_READ_ACROSS, + ), DeltaCompileVerifyTestCase( description="a new tracked variable names what changed", files=(), diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 37ad52ab87..27f042fca0 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -294,6 +294,7 @@ Compile runs in full instead when: a model-subject Rule must run again for a model that was not edited (for example because it reads an edited upstream file), or reads more for an edited model than that model's SQL, columns, contract and grain, project files and the source list; +- the project selects `SQBRPERFORMANCE102` on the Snowflake dialect; - the previous compile failed its Rules, since that compile writes no SQL tests, functions or audits to `target/compiled/`, or it reported other errors or warnings in files the edit didn't change; diff --git a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx index d9d2032d91..f005c838de 100644 --- a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx @@ -13,7 +13,9 @@ select = ["SQBRPERFORMANCE101", "SQBRPERFORMANCE102"] ``` Both Rules are off by default: selecting the `SQBRPERFORMANCE` family alone does not run them. They -run on the Snowflake dialect. +run on the Snowflake dialect. There, selecting `SQBRPERFORMANCE102` makes every edit compile the +whole project, because a [delta compile](/docs/cli/compile#compiling-after-an-edit) cannot recheck +the models that read or are read by the edited ones. | Code | Name | Reports | |---|---|---| From cd02645fd198d310f20c89d394976217a06b2ce9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:21:54 +0000 Subject: [PATCH 17/17] test: describe the dense rule selection without a standalone comment --- scripts/cold_compile_performance/_helpers/dense_project.py | 2 +- scripts/cold_compile_performance/constants.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/cold_compile_performance/_helpers/dense_project.py b/scripts/cold_compile_performance/_helpers/dense_project.py index abdd22580b..1c7a08422d 100644 --- a/scripts/cold_compile_performance/_helpers/dense_project.py +++ b/scripts/cold_compile_performance/_helpers/dense_project.py @@ -70,7 +70,7 @@ def dense_column_count(index: int) -> int: def write_dense_compile_project( *, project_dir: Path, model_count: int, rule_select: str = DENSE_RULE_SELECT ) -> None: - """Write a deterministic query-heavy project without external inputs or services.""" + """Write a deterministic query-heavy project; the default rules leave delta compiles on.""" project_dir.mkdir(parents=True) domains: str = ", ".join( diff --git a/scripts/cold_compile_performance/constants.py b/scripts/cold_compile_performance/constants.py index 583592fd47..5f4da9c988 100644 --- a/scripts/cold_compile_performance/constants.py +++ b/scripts/cold_compile_performance/constants.py @@ -6,9 +6,6 @@ DENSE_RULE_SELECT: str = ( 'select = ["SQBR", "SQBRPYTHON", "SQBRPYTHON306", "SQBRPYTHON507", "SQLCLEAN", "XSQBR"]' ) -# SQBRPERFORMANCE102 compares models across the project, so a Snowflake compile that selects it -# never takes the delta path. Edit and base/head benchmarks leave the opt-in performance rules -# out; the all-rules benchmarks select them. DENSE_ALL_RULE_SELECT: str = DENSE_RULE_SELECT.replace( '"SQBR", ', '"SQBR", "SQBRPERFORMANCE101", "SQBRPERFORMANCE102", ' )