diff --git a/benchmarks/duckshop/edge_cases/performance_rules/expected_findings.json b/benchmarks/duckshop/edge_cases/performance_rules/expected_findings.json new file mode 100644 index 0000000000..1370a39d58 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/expected_findings.json @@ -0,0 +1,7 @@ +{ + "findings": [ + ["SQBRPERFORMANCE101", "models/line_options.sql", 7, 3], + ["SQBRPERFORMANCE102", "models/order_lines_enriched.sql", 12, 3] + ], + "skipped_note": "SQBRPERFORMANCE102 skipped 1 model without column lineage or inferred types" +} diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/line_options.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options.sql new file mode 100644 index 0000000000..6b4d05bf54 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options.sql @@ -0,0 +1,9 @@ +MODEL ( + description "One row per line option with the whole parent line copied onto it.", +); + +SELECT + r.order_id, + o.value AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql new file mode 100644 index 0000000000..f89506e696 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/line_options_trimmed.sql @@ -0,0 +1,10 @@ +MODEL ( + description "One row per line option with the parent line minus its options, and a hash of the whole line.", +); + +SELECT + r.order_id, + OBJECT_DELETE(o.value, 'options') AS line_json, + MD5(TO_VARCHAR(o.value)) AS line_hash, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_exports.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_exports.sql new file mode 100644 index 0000000000..9eb899172a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_exports.sql @@ -0,0 +1,9 @@ +MODEL ( + description "Orders read from an export table SQLBuild cannot see.", + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT * +FROM RAW.SHOP.ORDER_EXPORTS diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers.sql new file mode 100644 index 0000000000..5de9f1b4e7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers.sql @@ -0,0 +1,12 @@ +MODEL ( + description "One row per order, unique by order_id and by order_number.", + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + order_number (type VARCHAR, nullable false, audits [unique]), + ), +); + +SELECT + 1 AS order_id, + 'SO-1' AS order_number, + PARSE_JSON('{"customer": "c1"}') AS header diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers_dated.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers_dated.sql new file mode 100644 index 0000000000..6894f89aa9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_headers_dated.sql @@ -0,0 +1,16 @@ +MODEL ( + description "One row per order: keyed by order number and date for pruning, but order_id alone is unique.", + materialized incremental, + incremental_strategy merge, + unique_key [order_number, order_date], + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT + h.order_id, + h.order_number, + CURRENT_DATE() AS order_date, + h.header AS order_header +FROM __ref("order_headers") h diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines.sql new file mode 100644 index 0000000000..b8ce938855 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines.sql @@ -0,0 +1,12 @@ +MODEL ( + description "One row per order line.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, line_no], +); + +SELECT + o.order_id, + l.index AS line_no, + l.value:sku::string AS sku +FROM __ref("orders") o, LATERAL FLATTEN(input => o.payload:lines) l diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_customer.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_customer.sql new file mode 100644 index 0000000000..91b2d4e96f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_customer.sql @@ -0,0 +1,13 @@ +MODEL ( + description "Order lines with one field read from the order payload.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, line_no], +); + +SELECT + l.order_id, + l.line_no, + o.payload:customer::string AS customer_id +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_enriched.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_enriched.sql new file mode 100644 index 0000000000..34cd474d92 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_lines_enriched.sql @@ -0,0 +1,14 @@ +MODEL ( + description "Order lines with the whole order payload copied onto every line.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, line_no], +); + +SELECT + l.order_id, + l.line_no, + l.sku, + o.payload AS order_payload +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_payload_dated.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_payload_dated.sql new file mode 100644 index 0000000000..5b9374e1ce --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_payload_dated.sql @@ -0,0 +1,15 @@ +MODEL ( + description "One row per order: the merge key adds a date for pruning, but order_id alone is unique.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, order_date], + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT + o.order_id, + CURRENT_DATE() AS order_date, + o.payload AS order_payload +FROM __ref("orders") o diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/order_skus.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/order_skus.sql new file mode 100644 index 0000000000..5ddf490256 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/order_skus.sql @@ -0,0 +1,12 @@ +MODEL ( + description "One row per order with its line SKUs aggregated.", + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT + l.order_id, + ARRAY_AGG(l.sku) AS skus +FROM __ref("order_lines") l +GROUP BY l.order_id diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/orders.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/orders.sql new file mode 100644 index 0000000000..0ec4614b75 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/orders.sql @@ -0,0 +1,10 @@ +MODEL ( + description "One row per order with its raw payload.", + columns ( + order_id (type NUMBER, nullable false, audits [unique]), + ), +); + +SELECT + 1 AS order_id, + PARSE_JSON('{"customer": "c1", "lines": [{"sku": "a", "options": [{"code": "gift"}]}]}') AS payload diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart__order_lines.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart__order_lines.sql new file mode 100644 index 0000000000..8388022195 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart__order_lines.sql @@ -0,0 +1,13 @@ +MODEL ( + description "Published order lines whose only reader is their own published view.", + materialized incremental, + incremental_strategy merge, + unique_key [order_id, line_no], +); + +SELECT + l.order_id, + l.line_no, + o.payload AS order_payload +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id diff --git a/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart_v__order_lines.sql b/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart_v__order_lines.sql new file mode 100644 index 0000000000..0adface38c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/models/shop__mart_v__order_lines.sql @@ -0,0 +1,7 @@ +MODEL ( + description "The published view of order lines.", + materialized view, +); + +SELECT order_id, line_no, order_payload +FROM __ref("shop__mart__order_lines") diff --git a/benchmarks/duckshop/edge_cases/performance_rules/sqlbuild_project.toml b/benchmarks/duckshop/edge_cases/performance_rules/sqlbuild_project.toml new file mode 100644 index 0000000000..b5993670d5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/performance_rules/sqlbuild_project.toml @@ -0,0 +1,9 @@ +name = "orders" +adapter = "snowflake" + +[connection] +database = "ANALYTICS" +schema = "ORDERS" + +[rules] +select = ["SQBRPERFORMANCE101", "SQBRPERFORMANCE102"] diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index b8a426f348..928a8cb371 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -241,6 +241,13 @@ pub struct LineageRow { pub expression_range: Option<(usize, usize)>, } +impl LineageRow { + /// Whether the column aggregates its sources rather than carrying one source row's value. + pub fn aggregated(&self) -> bool { + self.transform_code == crate::assembly::analysis_session::constants::TRANSFORM_AGGREGATION + } +} + /// Where a model's lineage facts live. #[derive(Debug, Clone, PartialEq, Eq)] pub enum LineageFacts { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index 42289b766d..8f2aff3869 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -394,6 +394,7 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { FieldSpec::new("external_sql_reference_resolver", FieldDefault::None), FieldSpec::new("resolve_python_run_selectors", FieldDefault::False), FieldSpec::new("no_cache", FieldDefault::False), + FieldSpec::new("verbose", FieldDefault::False), FieldSpec::new("max_microbatches", FieldDefault::None), FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("selection_diagnostics", FieldDefault::False), diff --git a/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs index b0856b5a54..989133a1cc 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs @@ -108,6 +108,7 @@ const R_RULES_RUN_RESULT: RecordSpec = RecordSpec { FieldSpec::new("cache_misses", FieldDefault::Int(0)), FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), + FieldSpec::new("skipped_models", FieldDefault::EmptyTuple), ], }; const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { @@ -121,6 +122,7 @@ const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { FieldSpec::required("cache_misses"), FieldSpec::required("built_in_ms"), FieldSpec::required("reused"), + FieldSpec::new("skipped_models", FieldDefault::EmptyTuple), ], }; const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { @@ -298,6 +300,7 @@ const R_RULES_RESULT: RecordSpec = RecordSpec { FieldSpec::new("custom_ms", FieldDefault::Int(0)), FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), + FieldSpec::new("skipped_models", FieldDefault::EmptyTuple), ], }; const R_RULE_FILE: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index 402dde4195..32bafbeb97 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -1080,6 +1080,7 @@ const COMPILE_ANALYSIS: RecordSpec = RecordSpec { FieldSpec::new("rule_cache_misses", FieldDefault::Int(0)), FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), + FieldSpec::new("skipped_models", FieldDefault::EmptyTuple), ], }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index b230110d6b..fb29fddc1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -56,6 +56,7 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult PyResult = injection .rules .lock() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 883111a1b8..6fab69d476 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -306,6 +306,10 @@ fn analyzed<'py>( "custom_rule_timings", native_field(&rules_result, "custom_rule_timings")?, ), + ( + "skipped_models", + native_field(&rules_result, "skipped_models")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 088bac0238..d76f3ae913 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -49,6 +49,7 @@ use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice; +use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models; const BYPASS_OUTCOME: &str = "bypass"; @@ -424,9 +425,16 @@ fn compiled_with_status<'py>( } } close_status(status)?; - if let Some(skipped_note) = format_skipped_type_proof_rules( - native_field(&analysis, "skipped_type_proof_rules")?.extract()?, - ) { + let skipped_notes = [ + format_skipped_type_proof_rules( + native_field(&analysis, "skipped_type_proof_rules")?.extract()?, + ), + format_skipped_models( + &native_field(&analysis, "skipped_models")?.extract::>()?, + false, + ), + ]; + for skipped_note in skipped_notes.into_iter().flatten() { output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?; } let dag_artifact_path: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index ecafc6ed93..131bb1acfe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -6,6 +6,7 @@ use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice; +use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline::{ @@ -137,7 +138,13 @@ fn run_rules_gate<'py>( if !on_progress.is_none() { let elapsed: f64 = monotonic(py)? - started; let message: String = rules_progress(elapsed, &result, true)?; - on_progress.call1((message,))?; + let note: Option = format_skipped_models( + &native_field(&result, "skipped_models")?.extract::>()?, + native_field(options, "verbose")?.is_truthy()?, + ); + for line in std::iter::once(message).chain(note.map(|note| format!("note: {note}"))) { + on_progress.call1((line,))?; + } } let mut failing: Vec> = Vec::new(); for finding in native_field(&result, "findings")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 88b9602038..9b4c92d489 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -44,9 +44,10 @@ const DEFAULT_SECTION_ENTRIES: i64 = 50; const ERROR_SEVERITY: &str = "error"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; /// Request fields passed to the compile pipeline under the same name. -const PIPELINE_FIELDS: [&str; 11] = [ +const PIPELINE_FIELDS: [&str; 12] = [ "no_sql_validation", "no_cache", + "verbose", "defer_to", "defer_sources_to", "cursor_overrides", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index 389a15a7f6..40e05c5b09 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -28,6 +28,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models; const LIST_ACTION: &str = "list"; const SHOW_ACTION: &str = "show"; @@ -238,6 +239,12 @@ fn run_rule_selection( ) { print_line(py, &format!("note: {note}"), true)?; } + if let Some(note) = format_skipped_models( + &native_field(&result, "skipped_models")?.extract::>()?, + false, + ) { + print_line(py, &format!("note: {note}"), true)?; + } let fields: Bound<'_, PyDict> = PyDict::new(py); for field in RESULT_FIELDS { fields.set_item(field, native_field(&result, field)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index 56a557093d..14f0b42a54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -25,7 +25,7 @@ pub(crate) const CUSTOM_HOST_ABANDONED_SUFFIX: &str = ".abandoned"; /// Marker file whose presence tells split custom hosts the run was cancelled. pub(crate) const CUSTOM_HOST_CANCELLED_MARKER: &str = "cancelled"; /// Rules that need proven column types and are skipped without SQL analysis. -pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 1] = ["SQBRCONTRACT105"]; +pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 2] = ["SQBRCONTRACT105", "SQBRPERFORMANCE102"]; /// Runtime identity custom-rule hosts and their cache entries record. pub(crate) const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v3"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 314da78b83..359cb75556 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -1,6 +1,6 @@ //! Build and evaluate the built-in rules request natively from compiled-project rows. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::path::PathBuf; use std::sync::Mutex; use std::sync::atomic::AtomicBool; @@ -18,18 +18,23 @@ use sqlbuild_rules::constants::API_VERSION; use sqlbuild_rules::engine::main::build_models::build_models; use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; use sqlbuild_rules::engine::main::finalize_rows::finalize_rows; +use sqlbuild_rules::engine::main::json_grain_skipped_models::json_grain_skipped_models; +use sqlbuild_rules::engine::main::json_grains::json_grains; +use sqlbuild_rules::engine::main::needs_json_grains::needs_json_grains; use sqlbuild_rules::engine::main::project_model_rows::project_model_rows; use sqlbuild_rules::errors::RowsError; use sqlbuild_rules::models::{ - Declaration, DeclarationMember, EvaluateRequest, Fault, Model, ModelRow, ModelRowsContext, - RowsEvaluation, RowsRequest, RulesConfig, + Declaration, DeclarationMember, EvaluateRequest, Fault, LineageColumn, Model, ModelRow, + ModelRowsContext, RowsEvaluation, RowsRequest, RulesConfig, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; +use crate::bindings::_helpers::compiled_views::model_reads::native_model; use crate::bindings::_helpers::compiled_views::records::compiled_project_facts; use crate::bindings::_helpers::compiled_views::records::declaration_rows; +use crate::bindings::_helpers::compiled_views::relation_keys::project_relation_keys; use crate::bindings::_helpers::rules::config_json::asdict_json; use crate::bindings::_helpers::rules::early_builtin_rules::EarlyBuiltinRules; use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; @@ -293,10 +298,8 @@ fn built_request( Some(facts) => facts.clone(), None => compiled_project_facts(&project)?, }; - let names: Vec = project_models(&project)? - .into_iter() - .map(|model| model.name) - .collect(); + let models = project_models(&project)?; + let names: Vec = models.iter().map(|model| model.name.clone()).collect(); let sql_tests: Vec> = project_attribute(&project, "sql_tests")? .try_iter()? .collect::>()?; @@ -320,8 +323,17 @@ fn built_request( .iter() .map(finding_fault) .collect::>()?; - let model_rows: Vec = + let mut model_rows: Vec = project_model_rows(facts.borrow().facts(), &names).map_err(value_error)?; + if needs_json_grains(&request).map_err(value_error)? { + let lineage: Vec>> = models + .iter() + .map(|entry| model_lineage(&entry.model)) + .collect::>()?; + let keys: BTreeMap>> = + project_relation_keys(&project)?.into_iter().collect(); + json_grains(&mut model_rows, lineage, &keys); + } request.models = built_models(&rows, &request, model_rows)?; let request: EvaluateRequest = crate::bindings::_helpers::compile_delta::rules_rows::whole_project_request(request)?; @@ -365,6 +377,7 @@ fn evaluated<'py>( early: Option<(EarlyBuiltinRules, &AtomicBool)>, ) -> PyResult> { let request: RowsRequest = taken_rows(request)?; + let skipped: Vec<(String, String)> = json_grain_skipped_models(&request.request.models); let evaluation: Result = match early { None => py .compiler_detach(|| Ok(evaluate_rows(request))) @@ -407,6 +420,7 @@ fn evaluated<'py>( ), ("built_in_ms", evaluation.built_in_ms.into_bound_py_any(py)?), ("reused", evaluation.reused.into_bound_py_any(py)?), + ("skipped_models", PyTuple::new(py, skipped)?.into_any()), ], ) } @@ -522,6 +536,30 @@ fn built_models( built.map_err(|message| python_error.take().unwrap_or_else(|| value_error(message))) } +/// The model's lineage columns, or `None` when it has no kept lineage. +fn model_lineage(model: &Bound<'_, PyAny>) -> PyResult>> { + let Some(view) = native_model(model)? else { + return Ok(None); + }; + let Some(rows) = view.lineage_rows(model.py())? else { + return Ok(None); + }; + let mut columns: Vec = Vec::with_capacity(rows.len()); + for row in rows { + let aggregated: bool = row.aggregated(); + let mut sources: Vec<(String, String)> = Vec::with_capacity(row.sources.len()); + for (resource, name, _) in row.sources { + sources.push((resource, name)); + } + columns.push(LineageColumn { + aggregated, + output: row.output_column, + sources, + }); + } + Ok(Some(columns)) +} + fn declarations(rows: Vec>) -> PyResult> { rows.into_iter().map(declaration).collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 5a0e5b77ef..99c2f570e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -555,6 +555,7 @@ fn rules_result<'py>( "custom_rule_timings", field(&custom, "custom_rule_timings")?, ), + ("skipped_models", field(&native_result, "skipped_models")?), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index a87a214f5e..390b37eb47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -137,6 +137,7 @@ pub(crate) fn finish_rule_run<'py>( "skipped_type_proof_rules", record_field(plan, "skipped_type_proof_rules")?, ), + ("skipped_models", record_field(result, "skipped_models")?), ], ) } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index bd33ed3e7a..8a2fad67e4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -431,7 +431,7 @@ pub(crate) fn evaluate_request_with( &request.rules_cache_path, ) }); - crate::rules::_helpers::sql_test_rules::with_fixture_memo(store.as_deref(), || { + crate::rules::_helpers::sql_checks::sql_test_rules::with_fixture_memo(store.as_deref(), || { let evaluated: Result = evaluated_request(request, control); let keep: bool = match &evaluated { Ok(_) => !control.cancelled(), diff --git a/crates/sqlbuild-rules/src/engine/_helpers/json_grain.rs b/crates/sqlbuild-rules/src/engine/_helpers/json_grain.rs new file mode 100644 index 0000000000..6d7a348705 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/json_grain.rs @@ -0,0 +1,71 @@ +//! One model row's SQBRPERFORMANCE102 grain: its keys, readers and copied semi-structured columns. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::engine::constants::{NO_LINEAGE, NO_TYPES, RESOURCE_KINDS, SEMI_STRUCTURED_TYPES}; +use crate::models::{GrainColumn, JsonGrain, LineageColumn, ModelRow}; + +pub(crate) fn grain( + row: &ModelRow, + (model_keys, readers): (Vec>, Vec), + lineage: Option>, + keys: &BTreeMap>>, +) -> JsonGrain { + let mut grain: JsonGrain = JsonGrain { + model_keys, + readers, + ..JsonGrain::default() + }; + let Some(lineage) = lineage.filter(|columns| !columns.is_empty()) else { + grain.skipped = Some(NO_LINEAGE.to_owned()); + return grain; + }; + if row.inferred_columns.is_empty() { + grain.skipped = Some(NO_TYPES.to_owned()); + return grain; + } + for (name, data_type) in &row.inferred_columns { + let Some(data_type) = data_type.as_deref().filter(|kind| semi_structured(kind)) else { + continue; + }; + let Some(column) = lineage + .iter() + .find(|column| column.output.eq_ignore_ascii_case(name)) + .filter(|column| !column.aggregated) + else { + continue; + }; + let inputs: BTreeSet = column_inputs(column); + for input in &inputs { + if let Some(found) = keys.get(input) { + grain.input_keys.insert(input.clone(), found.clone()); + } + } + grain.columns.push(GrainColumn { + name: name.clone(), + data_type: Some(data_type.to_owned()), + inputs: inputs.into_iter().collect(), + }); + } + grain +} + +/// `kind:name` of each relation `column` reads whose resource type declares keys. +fn column_inputs(column: &LineageColumn) -> BTreeSet { + let mut inputs: BTreeSet = BTreeSet::new(); + for (resource, name) in &column.sources { + if let Some((_, kind)) = RESOURCE_KINDS.iter().find(|(known, _)| known == resource) { + inputs.insert(format!("{kind}:{name}")); + } + } + inputs +} + +/// `VARIANT`, `OBJECT` or `ARRAY`, alone or with parameters such as `ARRAY(INT)`. +fn semi_structured(data_type: &str) -> bool { + let upper: String = data_type.trim().to_ascii_uppercase(); + SEMI_STRUCTURED_TYPES + .iter() + .filter_map(|kind| upper.strip_prefix(kind)) + .any(|rest| rest.is_empty() || rest.starts_with(['(', '<', ' '])) +} diff --git a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs index 234f764470..a0a61194a4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs @@ -1,6 +1,7 @@ pub mod cache; pub mod custom_host; pub mod evaluation; +pub(crate) mod json_grain; pub(crate) mod memo; pub(crate) mod project_rows; pub(crate) mod retained; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index 46e814d073..cffac3626c 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -65,6 +65,7 @@ pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result .collect(), typed_calls: facts.typed_calls.clone(), sql_analysis_skipped: facts.sql_analysis_skipped, + json_grain: None, }) } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs index 006c1474f4..db58807940 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs @@ -152,6 +152,7 @@ impl RowWriter { empty_input_only_test_count: _, payload_digest: _, stripped: _, + json_grain, } = model; self.flag(*sql_analysis_disabled); self.text(name); @@ -217,6 +218,10 @@ impl RowWriter { self.declarations(constant_declarations); self.record.put_u64(u64::from(*declared_audit_count)); self.record.put_u64(u64::from(*targeting_test_count)); + let grain: Option = json_grain + .as_ref() + .map(|grain| serde_json::to_value(grain).unwrap_or(Value::Null)); + self.optional_value(grain.as_ref()); } } @@ -386,6 +391,11 @@ impl RowReader<'_> { empty_input_only_test_count: 0, payload_digest: None, stripped: false, + json_grain: self + .optional_value()? + .map(serde_json::from_value) + .transpose() + .map_err(|_| StoreDecodeError)?, }) } } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index c7a9a6bdee..b31ddf2e3d 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -104,6 +104,7 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result Vec<(String, String)> { + let mut skipped: Vec<(String, String)> = Vec::new(); + for model in models.iter().filter(|model| !model.sql_analysis_disabled) { + if let Some(reason) = model + .json_grain + .as_ref() + .and_then(|grain| grain.skipped.as_ref()) + { + skipped.push((model.relative_path.clone(), reason.clone())); + } + } + skipped +} diff --git a/crates/sqlbuild-rules/src/engine/main/json_grains.rs b/crates/sqlbuild-rules/src/engine/main/json_grains.rs new file mode 100644 index 0000000000..71ff2c068b --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/json_grains.rs @@ -0,0 +1,45 @@ +//! SQBRPERFORMANCE102's grain facts on each model row, built only when the request selects it. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::engine::_helpers::json_grain::grain; +use crate::engine::constants::REFERENCE_KIND; +use crate::models::{JsonGrain, LineageColumn, ModelRow}; + +/// Set each keyed row's grain from its lineage (`None` when unknown), in row order, and `keys` by `kind:name`. +pub fn json_grains( + rows: &mut [ModelRow], + lineage: Vec>>, + keys: &BTreeMap>>, +) { + let mut readers: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + for row in rows.iter() { + for reference in &row.references { + if reference.ref_kind == REFERENCE_KIND { + readers + .entry(reference.ref_name.as_str()) + .or_default() + .insert(row.name.clone()); + } + } + } + let mut grains: Vec> = Vec::with_capacity(rows.len()); + for (row, lineage) in rows.iter().zip(lineage) { + let Some(model_keys) = keys.get(&format!("{REFERENCE_KIND}:{}", row.name)) else { + grains.push(None); + continue; + }; + let row_readers: Vec = readers + .get(row.name.as_str()) + .map_or_else(Vec::new, |names| names.iter().cloned().collect()); + grains.push(Some(grain( + row, + (model_keys.clone(), row_readers), + lineage, + keys, + ))); + } + for (row, grain) in rows.iter_mut().zip(grains) { + row.json_grain = grain; + } +} diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index 1237dba54e..1acb27f0d2 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -4,6 +4,9 @@ pub mod custom_host; pub mod discard_speculative_rows; pub mod evaluate_rows; pub mod finalize_rows; +pub mod json_grain_skipped_models; +pub mod json_grains; +pub mod needs_json_grains; pub mod project_model_rows; pub mod retained_request; pub mod start_speculative_rows; diff --git a/crates/sqlbuild-rules/src/engine/main/needs_json_grains.rs b/crates/sqlbuild-rules/src/engine/main/needs_json_grains.rs new file mode 100644 index 0000000000..478b89265f --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/needs_json_grains.rs @@ -0,0 +1,12 @@ +//! Whether a rules request needs SQBRPERFORMANCE102's grain facts. + +use crate::engine::constants::JSON_GRAIN_RULE_CODE; +use crate::models::EvaluateRequest; +use crate::query_facts::constants::SNOWFLAKE_DIALECT; +use crate::rules::main::request_selects::request_selects; + +/// Whether `request` selects SQBRPERFORMANCE102 for a dialect it runs on. +pub fn needs_json_grains(request: &EvaluateRequest) -> Result { + Ok(request.dialect.eq_ignore_ascii_case(SNOWFLAKE_DIALECT) + && request_selects(request, |rule| rule.code == JSON_GRAIN_RULE_CODE)?) +} diff --git a/crates/sqlbuild-rules/src/engine/mod.rs b/crates/sqlbuild-rules/src/engine/mod.rs index d5e5e9f39a..79a6444abf 100644 --- a/crates/sqlbuild-rules/src/engine/mod.rs +++ b/crates/sqlbuild-rules/src/engine/mod.rs @@ -1,4 +1,5 @@ pub mod _helpers; +mod constants; pub mod main; #[cfg(test)] diff --git a/crates/sqlbuild-rules/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs index 3f069465db..b87687410b 100644 --- a/crates/sqlbuild-rules/src/engine/tests.rs +++ b/crates/sqlbuild-rules/src/engine/tests.rs @@ -8,6 +8,8 @@ mod evaluation; mod helpers; #[path = "tests/test_incremental_rules.rs"] mod incremental_rules; +#[path = "tests/test_json_grains.rs"] +mod json_grains; #[path = "tests/test_published_layers.rs"] mod published_layers; #[path = "tests/test_query_facts.rs"] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index 798aa9f1c3..878e4ba729 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -11,15 +11,16 @@ use crate::engine::main::answer_speculative_rows::answer_speculative_rows; use crate::engine::main::discard_speculative_rows::discard_speculative_rows; use crate::engine::main::retained_request::retained_request; use crate::models::{ - Column, Declaration, DeclarationMember, DynamicColumnFamily, EvaluateRequest, Fault, Model, - Reference, RetainedRows, RowsEvaluation, RowsRequest, RulesConfig, SpeculativeAnswer, - SpeculativeRows, + Column, Declaration, DeclarationMember, DynamicColumnFamily, EvaluateRequest, Fault, + GrainColumn, JsonGrain, LineageColumn, Model, ModelRow, Reference, RetainedRows, + RowsEvaluation, RowsRequest, RulesConfig, SpeculativeAnswer, SpeculativeRows, }; use crate::engine::_helpers::evaluation::evaluate_json; use crate::engine::_helpers::evaluation::evaluate_request as evaluate_parsed; use crate::engine::_helpers::evaluation::parse_parts; use crate::engine::main::finalize_rows::finalize_rows; +use crate::engine::main::json_grains::json_grains; use crate::engine::tests::test_types; pub(crate) fn request(project_dir: &TempDir, config: &Value) -> String { @@ -645,6 +646,26 @@ pub(crate) fn query_facts_faults( request["models"][0]["enum_columns"] = json!(test_case.enum_columns); request["models"][0]["config"] = json!({"contract": "enforced"}); request["models"][0]["columns"] = test_case.columns.clone(); + evaluated_faults(&request) +} + +/// One model named `name` with `json_grain`, evaluated with `select` under `dialect`. +pub(crate) fn json_grain_faults( + test_case: &test_types::JsonGrainTestCase, +) -> Result, String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let config = json!({"select": test_case.select, "cache": {"enabled": false}}); + let mut request: Value = + serde_json::from_str(&request(&project_dir, &config)).map_err(|error| error.to_string())?; + request["dialect"] = json!(test_case.dialect); + request["models"][0]["name"] = json!(test_case.name); + request["models"][0]["query_sql"] = json!(test_case.query_sql); + request["models"][0]["authored_sql"] = json!(test_case.query_sql); + request["models"][0]["json_grain"] = test_case.json_grain.clone(); + evaluated_faults(&request) +} + +fn evaluated_faults(request: &Value) -> Result, String> { let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) .map_err(|error| error.to_string())?; let faults = result["faults"] @@ -734,6 +755,20 @@ pub(crate) fn every_field_model() -> Model { empty_input_only_test_count: 0, payload_digest: Some("digest".to_owned()), stripped: false, + json_grain: Some(JsonGrain { + model_keys: vec![vec!["order_id".to_owned(), "line_id".to_owned()]], + columns: vec![GrainColumn { + name: "order_json".to_owned(), + data_type: Some("VARIANT".to_owned()), + inputs: vec!["ref:orders".to_owned()], + }], + input_keys: BTreeMap::from([( + "ref:orders".to_owned(), + vec![vec!["order_id".to_owned()]], + )]), + readers: vec!["shop__mart_v__orders".to_owned()], + skipped: Some("no lineage".to_owned()), + }), } } @@ -855,3 +890,116 @@ pub(crate) fn finalize_cross_joins(exceptions: Value) -> Result, Stri findings, ) } + +/// A `json_grain` fact for a payload copied from `orders` onto order lines. +pub(crate) fn payload_grain(input_key: Value, readers: Value) -> Value { + json!({ + "model_keys": [["order_id", "line_no"]], + "columns": [{"name": "payload", "data_type": "VARIANT", "inputs": ["ref:orders"]}], + "input_keys": {"ref:orders": [input_key]}, + "readers": readers, + }) +} + +/// A model row with `references` to other models and `inferred` column types. +pub(crate) fn grain_row(name: &str, references: &[&str], inferred: &[(&str, &str)]) -> ModelRow { + ModelRow { + name: name.to_owned(), + references: references + .iter() + .map(|name| Reference { + ref_kind: "ref".to_owned(), + ref_name: (*name).to_owned(), + ref_package: None, + }) + .collect(), + inferred_columns: inferred + .iter() + .map(|(name, kind)| ((*name).to_owned(), Some((*kind).to_owned()))) + .collect(), + ..ModelRow::default() + } +} + +fn grain_lineage(output: &str, sources: &[(&str, &str)], aggregated: bool) -> LineageColumn { + LineageColumn { + output: output.to_owned(), + sources: sources + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(), + aggregated, + } +} + +/// One declared key of `key`'s columns. +pub(crate) fn grain_keys(key: &[&str]) -> Vec> { + vec![key.iter().map(|column| (*column).to_owned()).collect()] +} + +pub(crate) fn grain_column(name: &str, data_type: &str, inputs: &[&str]) -> GrainColumn { + GrainColumn { + name: name.to_owned(), + data_type: Some(data_type.to_owned()), + inputs: inputs.iter().map(|input| (*input).to_owned()).collect(), + } +} + +/// Grains `json_grains` builds for four order models, by model name. +pub(crate) fn built_json_grains() -> BTreeMap> { + let mut rows = vec![ + grain_row( + "order_lines_enriched", + &["orders", "order_lines"], + &[ + ("payload", "VARIANT"), + ("items", "ARRAY(VARCHAR)"), + ("region", "object"), + ("total", "NUMBER(38,0)"), + ("payloads", "ARRAY"), + ("arrayish", "ARRAYLIKE"), + ], + ), + grain_row("orders", &[], &[("payload", "VARIANT")]), + grain_row("unkeyed", &["orders"], &[("payload", "VARIANT")]), + grain_row("untyped", &[], &[]), + ]; + let lineage = vec![ + Some(vec![ + grain_lineage( + "PAYLOAD", + &[("model", "orders"), ("table_fn", "explode")], + false, + ), + grain_lineage("items", &[("source", "shop.raw_orders")], false), + grain_lineage("region", &[("seed", "countries")], false), + grain_lineage("total", &[("model", "orders")], false), + grain_lineage("payloads", &[("model", "orders")], true), + grain_lineage("arrayish", &[("model", "orders")], false), + ]), + None, + Some(vec![grain_lineage( + "payload", + &[("model", "orders")], + false, + )]), + Some(vec![grain_lineage( + "payload", + &[("model", "orders")], + false, + )]), + ]; + let declared: BTreeMap>> = BTreeMap::from([ + ( + "ref:order_lines_enriched".to_owned(), + grain_keys(&["order_id", "line_no"]), + ), + ("ref:orders".to_owned(), grain_keys(&["order_id"])), + ("ref:untyped".to_owned(), grain_keys(&["id"])), + ("seed:countries".to_owned(), grain_keys(&["country_code"])), + ]); + json_grains(&mut rows, lineage, &declared); + rows.into_iter() + .map(|row| (row.name, row.json_grain)) + .collect() +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index b4c7d5645d..e8ed79e46d 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -37,6 +37,8 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRMODEL103", RuleScope::Model), ("SQBRMODEL104", RuleScope::Model), ("SQBRORCHESTRATION101", RuleScope::Project), + ("SQBRPERFORMANCE101", RuleScope::Model), + ("SQBRPERFORMANCE102", RuleScope::Model), ("SQBRPROJECT101", RuleScope::Model), ("SQBRPROJECT102", RuleScope::Model), ("SQBRPROJECT103", RuleScope::Model), diff --git a/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs new file mode 100644 index 0000000000..27687c67b3 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_json_grains.rs @@ -0,0 +1,218 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::engine::tests::helpers::{ + built_json_grains, grain_column, grain_keys, json_grain_faults, payload_grain, +}; +use crate::engine::tests::test_types::{JsonGrainTestCase, JsonGrainsBuildTestCase}; +use crate::models::JsonGrain; + +const RULE: &[&str] = &["SQBRPERFORMANCE102"]; +const LINES_SQL: &str = "SELECT l.order_id, l.line_no, o.payload\nFROM __ref(\"order_lines\") l JOIN __ref(\"orders\") o USING (order_id)"; + +#[test] +fn given_grain_facts_when_evaluating_json_finer_grain_then_only_coarser_inputs_are_flagged() +-> Result<(), String> { + let test_cases = [ + JsonGrainTestCase { + description: "a payload keyed by order copied onto order lines is flagged at its column", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["order_id"]), json!([])), + expected_faults: &[( + "SQBRPERFORMANCE102", + 1, + 31, + "payload copies VARIANT from orders, one row per (order_id), onto rows keyed by (order_id, line_no)", + )], + }, + JsonGrainTestCase { + description: "key columns match whatever their case and order", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: json!({ + "model_keys": [["LINE_NO", "Order_Id"]], + "columns": [{"name": "PAYLOAD", "data_type": "VARIANT", "inputs": ["ref:orders"]}], + "input_keys": {"ref:orders": [["ORDER_ID"]]}, + }), + expected_faults: &[("SQBRPERFORMANCE102", 1, 31, "PAYLOAD copies VARIANT")], + }, + JsonGrainTestCase { + description: "an input on the model's own grain repeats nothing", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["line_no", "order_id"]), json!([])), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "another declared model key on the input's grain proves one row per input row", + dialect: "snowflake", + name: "order_payload_dated", + select: RULE, + query_sql: LINES_SQL, + json_grain: json!({ + "model_keys": [["order_id", "line_no"], ["ORDER_ID"]], + "columns": [{"name": "payload", "data_type": "VARIANT", "inputs": ["ref:orders"]}], + "input_keys": {"ref:orders": [["order_id"]]}, + }), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "an input with two keys is not coarser when either covers a model key", + dialect: "snowflake", + name: "order_payload_dated", + select: RULE, + query_sql: LINES_SQL, + json_grain: json!({ + "model_keys": [["order_number", "order_date"], ["order_id"]], + "columns": [{"name": "payload", "data_type": "VARIANT", "inputs": ["ref:orders"]}], + "input_keys": {"ref:orders": [["order_id"], ["order_number"]]}, + }), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "an input keyed by another column is not coarser", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["customer_id"]), json!([])), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "a model whose only reader is the published view of its own entity is exempt", + dialect: "snowflake", + name: "shop__mart__orders", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["order_id"]), json!(["shop__mart_v__orders"])), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "a published model that another model also reads is not exempt", + dialect: "snowflake", + name: "shop__mart__orders", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain( + json!(["order_id"]), + json!(["shop__int_enriched__orders", "shop__mart_v__orders"]), + ), + expected_faults: &[("SQBRPERFORMANCE102", 1, 31, "payload copies VARIANT")], + }, + JsonGrainTestCase { + description: "a view of another entity does not exempt the model", + dialect: "snowflake", + name: "shop__mart__orders", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["order_id"]), json!(["shop__mart_v__customers"])), + expected_faults: &[("SQBRPERFORMANCE102", 1, 31, "payload copies VARIANT")], + }, + JsonGrainTestCase { + description: "a model without lineage is skipped, not flagged", + dialect: "snowflake", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: json!({"model_keys": [["order_id", "line_no"]], "skipped": "no column lineage"}), + expected_faults: &[], + }, + JsonGrainTestCase { + description: "the rule runs only on Snowflake", + dialect: "duckdb", + name: "order_lines_enriched", + select: RULE, + query_sql: LINES_SQL, + json_grain: payload_grain(json!(["order_id"]), json!([])), + expected_faults: &[], + }, + ]; + for test_case in test_cases { + let faults = json_grain_faults(&test_case)?; + let observed: Vec<(&str, u64, u64, &str)> = faults + .iter() + .enumerate() + .map(|(index, (code, line, column, message))| { + let fragment = test_case + .expected_faults + .get(index) + .map(|expected| expected.3) + .filter(|fragment| message.contains(fragment)) + .unwrap_or(message.as_str()); + (code.as_str(), *line, *column, fragment) + }) + .collect(); + assert_eq!( + observed, test_case.expected_faults, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_rows_lineage_and_keys_when_building_grains_then_only_copied_semi_structured_columns_are_kept() + { + let test_cases = [ + JsonGrainsBuildTestCase { + description: "a keyed model keeps only copied VARIANT, OBJECT and ARRAY columns with their input keys", + row_name: "order_lines_enriched", + expected_grain: Some(JsonGrain { + model_keys: grain_keys(&["order_id", "line_no"]), + columns: vec![ + grain_column("payload", "VARIANT", &["ref:orders"]), + grain_column("items", "ARRAY(VARCHAR)", &["source:shop.raw_orders"]), + grain_column("region", "object", &["seed:countries"]), + ], + input_keys: BTreeMap::from([ + ("ref:orders".to_owned(), grain_keys(&["order_id"])), + ("seed:countries".to_owned(), grain_keys(&["country_code"])), + ]), + readers: Vec::new(), + skipped: None, + }), + }, + JsonGrainsBuildTestCase { + description: "a model without lineage is skipped and lists its readers", + row_name: "orders", + expected_grain: Some(JsonGrain { + model_keys: grain_keys(&["order_id"]), + readers: vec!["order_lines_enriched".to_owned(), "unkeyed".to_owned()], + skipped: Some("no column lineage".to_owned()), + ..JsonGrain::default() + }), + }, + JsonGrainsBuildTestCase { + description: "a model without a declared key has no grain", + row_name: "unkeyed", + expected_grain: None, + }, + JsonGrainsBuildTestCase { + description: "a model without inferred types is skipped", + row_name: "untyped", + expected_grain: Some(JsonGrain { + model_keys: grain_keys(&["id"]), + skipped: Some("no inferred column types".to_owned()), + ..JsonGrain::default() + }), + }, + ]; + let grains: BTreeMap> = built_json_grains(); + for test_case in test_cases { + assert_eq!( + grains.get(test_case.row_name), + Some(&test_case.expected_grain), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs index f93bcf2a37..7c7e403dd8 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs @@ -9,6 +9,12 @@ const ENUM_RULE: &[&str] = &["SQBRDECLARATION101"]; const NUMERIC_RULE: &[&str] = &["SQBRDECLARATION102"]; const OUTPUT_TYPE_RULE: &[&str] = &["SQBRCONTRACT105"]; const STATUS: &[&str] = &["status"]; +const FLATTEN_RULE: &[&str] = &["SQBRPERFORMANCE101"]; +macro_rules! orders_flattened { + () => { + "FROM __ref(\"raw_orders\") r, LATERAL FLATTEN(input => r.payload:orders) o, LATERAL FLATTEN(o.value:lines) l" + }; +} #[test] fn given_dialect_sql_when_reading_query_facts_then_findings_match_the_authored_sql() @@ -305,6 +311,147 @@ fn given_dialect_sql_when_reading_query_facts_then_findings_match_the_authored_s columns: json!([]), expected_faults: &[("SQBRDECLARATION102", 1, 64, "amount < 9")], }, + QueryFactsTestCase { + description: "the parent element copied whole onto each line is flagged at the column", + dialect: "snowflake", + query_sql: concat!( + "SELECT l.value:sku::string AS sku, o.value AS order_json\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[( + "SQBRPERFORMANCE101", + 1, + 36, + "order_json copies o.value whole onto every row flattened from o.value:lines", + )], + }, + QueryFactsTestCase { + description: "a field read from the parent is fine", + dialect: "snowflake", + query_sql: concat!( + "SELECT o.value:customer_id::string AS customer_id, l.value:sku::string AS sku\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, + QueryFactsTestCase { + description: "the parent without its flattened child array is fine", + dialect: "snowflake", + query_sql: concat!( + "SELECT OBJECT_DELETE(o.value, 'lines') AS order_json, l.value:sku AS sku\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, + QueryFactsTestCase { + description: "removing a different key still copies the child array", + dialect: "snowflake", + query_sql: concat!( + "SELECT OBJECT_DELETE(o.value, 'notes') AS order_json\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "copies o.value whole")], + }, + QueryFactsTestCase { + description: "the parent embedded in a built object is flagged", + dialect: "snowflake", + query_sql: concat!( + "SELECT OBJECT_CONSTRUCT('order', o.value, 'sku', l.value:sku) AS line_json\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "line_json copies o.value whole")], + }, + QueryFactsTestCase { + description: "a function that only reads the parent, such as a hash, copies nothing", + dialect: "snowflake", + query_sql: concat!( + "SELECT MD5(TO_VARCHAR(o.value)) AS line_hash, TYPEOF(o.value) AS kind\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, + QueryFactsTestCase { + description: "the parent serialized to JSON text is still a whole copy", + dialect: "snowflake", + query_sql: concat!( + "SELECT TO_JSON(o.value) AS line_text, TO_CHAR(OBJECT_DELETE(o.value, 'lines')) AS trimmed\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "line_text copies o.value whole")], + }, + QueryFactsTestCase { + description: "the parent cast and wrapped in an array is still a whole copy", + dialect: "snowflake", + query_sql: concat!( + "SELECT ARRAY_CONSTRUCT((o.value::object)) AS line_json\n", + orders_flattened!() + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "line_json copies o.value whole")], + }, + QueryFactsTestCase { + description: "an element no deeper FLATTEN reads is fine", + dialect: "snowflake", + query_sql: "SELECT o.value AS order_json FROM __ref(\"raw_orders\") r, LATERAL FLATTEN(input => r.payload:orders) o", + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, + QueryFactsTestCase { + description: "TABLE(FLATTEN(...)) and GET paths are read the same way", + dialect: "snowflake", + query_sql: "SELECT o.value AS order_json\nFROM __ref(\"raw_orders\") r, TABLE(FLATTEN(r.payload:orders)) o, TABLE(FLATTEN(GET(o.value, 'lines'))) l", + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 1, 8, "flattened from o.value:lines")], + }, + QueryFactsTestCase { + description: "a copy inside a CTE is flagged where it is made", + dialect: "snowflake", + query_sql: concat!( + "WITH lines AS (\n SELECT o.value AS order_json, l.value AS line_json\n ", + orders_flattened!(), + "\n)\nSELECT line_json FROM lines" + ), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRPERFORMANCE101", 2, 10, "order_json copies o.value")], + }, + QueryFactsTestCase { + description: "the child element itself is not a parent copy", + dialect: "snowflake", + query_sql: concat!("SELECT l.value AS line_json\n", orders_flattened!()), + select: FLATTEN_RULE, + enum_columns: &[], + columns: json!([]), + expected_faults: &[], + }, ]; for test_case in test_cases { let faults = helpers::query_facts_faults(&test_case)?; diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index 0140ad49cb..5fc59e9592 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -3,7 +3,7 @@ use crate::engine::tests::test_types::{ }; use crate::lint_text::main::preparation::prepare; use crate::query_facts::main::model_shape::model_shape; -use crate::rules::_helpers::numeric_decisions::compact_sql; +use crate::rules::_helpers::sql_checks::numeric_decisions::compact_sql; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy as rules_quote_policy; use sqlbuild_sqltext::sql_scan::models::QuotePolicy; diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index 47e67ffbdc..9b3d0a944c 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -1,5 +1,7 @@ use serde_json::Value; +use crate::models::JsonGrain; + use sqlbuild_sqltext::sql_scan::models::Unclosed; pub(crate) struct NativeEvaluationTestCase { @@ -271,6 +273,23 @@ pub(crate) struct QueryFactsTestCase { pub(crate) expected_faults: &'static [(&'static str, u64, u64, &'static str)], } +pub(crate) struct JsonGrainTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) name: &'static str, + pub(crate) select: &'static [&'static str], + pub(crate) query_sql: &'static str, + pub(crate) json_grain: Value, + pub(crate) expected_faults: &'static [(&'static str, u64, u64, &'static str)], +} + +/// One model's grain after `json_grains` builds the four order models. +pub(crate) struct JsonGrainsBuildTestCase { + pub(crate) description: &'static str, + pub(crate) row_name: &'static str, + pub(crate) expected_grain: Option, +} + /// A retained request evaluated again without SQL, with or without the cached findings. pub(crate) struct RetainedRowsTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index dfcdb43bf5..bd71cf5cbe 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -608,6 +608,40 @@ pub struct Model { /// Whether this row's SQL text was dropped, so only a cached result may stand for it. #[serde(skip)] pub stripped: bool, + /// SQBRPERFORMANCE102's grain facts, present only when that rule is selected. + pub json_grain: Option, +} + +/// What SQBRPERFORMANCE102 compares: the model's keys, each output's type and lineage inputs, and the inputs' keys. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct JsonGrain { + /// Column sets each proving one row per value in this model. + pub model_keys: Vec>, + pub columns: Vec, + /// Each lineage input's keys by `kind:name`, such as `ref:orders`. + pub input_keys: BTreeMap>>, + /// The models that read this one. + pub readers: Vec, + /// Why the comparison cannot run, such as missing lineage; the model is then skipped, not flagged. + pub skipped: Option, +} + +/// One lineage output column: the `(resource type, name)` relations it reads and whether it aggregates them. +#[derive(Clone, Debug, Default)] +pub struct LineageColumn { + pub output: String, + pub sources: Vec<(String, String)>, + pub aggregated: bool, +} + +/// One output column's inferred type and the relations its lineage reads, by `kind:name`. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct GrainColumn { + pub name: String, + pub data_type: Option, + pub inputs: Vec, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -957,6 +991,8 @@ pub struct ModelRow { pub typed_calls: Vec, /// Whether analysis was skipped because the dialect's native parser cannot read the model. pub sql_analysis_skipped: bool, + /// SQBRPERFORMANCE102's grain facts, present only when that rule is selected. + pub json_grain: Option, } /// A model's declared schema entry. diff --git a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs index 90f58d1765..6f943df76c 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs @@ -6,7 +6,7 @@ use fensu_native::{NativeProjectPlane, NativeRuleContext, evaluate_core_rules}; use crate::engine::_helpers::memo::{evaluate_rows, prepare}; use crate::models::{EvaluateRequest, RowsRequest, RuleMetadata, RuleScope, RulesConfig}; use crate::python_rules::tests::test_types::CachedRuns; -use crate::rules::_helpers::catalogue::{catalogue, select}; +use crate::rules::_helpers::dispatch::catalogue::{catalogue, select}; /// How many distinct `SQBRPYTHON` codes the catalogue holds. pub(super) fn distinct_python_codes(rules: &[RuleMetadata]) -> usize { diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs index d12291234a..7eead951e7 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs @@ -7,7 +7,7 @@ use crate::python_rules::tests::helpers::{ use crate::python_rules::tests::test_types::{ CatalogueTestCase, CoreRuleTestCase, OptionsTestCase, SelectionTestCase, SourceKeyTestCase, }; -use crate::rules::_helpers::catalogue::catalogue; +use crate::rules::_helpers::dispatch::catalogue::catalogue; use crate::rules::main::builtin_options::builtin_options; use crate::rules::models::BuiltinOptionKind; diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs index a70b2a101a..46ca4fa8af 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs @@ -1,3 +1,4 @@ +use crate::query_facts::_helpers::flatten::{flatten_call, flatten_input, whole_values}; use crate::query_facts::_helpers::shape::stub_call; use crate::query_facts::_helpers::tokens::{ComparisonOperator, QueryTokens, SelectTokens}; use crate::query_facts::constants::{ @@ -340,6 +341,7 @@ impl Builder<'_> { ) }), start: range.and_then(|range| self.tokens.leaf_start(&range)), + whole_values: whole_values(value), }) } @@ -599,10 +601,25 @@ fn relation(expression: &Expression, tokens: &QueryTokens<'_>) -> RelationFacts last, alias: name, }, + RelationFacts::Flatten { input, .. } => RelationFacts::Flatten { + alias: name.map(|name| name.to_ascii_lowercase()), + input, + }, other => other, } } Expression::Table(table) => table_relation(table, tokens), + Expression::Lateral(lateral) => match flatten_call(expression) { + Some(function) => RelationFacts::Flatten { + alias: lateral.alias.as_ref().map(|alias| alias.to_ascii_lowercase()), + input: flatten_input(function), + }, + None => RelationFacts::Other, + }, + Expression::Function(_) if flatten_call(expression).is_some() => RelationFacts::Flatten { + alias: None, + input: flatten_call(expression).and_then(flatten_input), + }, Expression::Pivot(pivot) if !pivot.unpivot => { if pivot.fields.iter().any(|field| { matches!(field, Expression::In(list) diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs new file mode 100644 index 0000000000..7828a762ec --- /dev/null +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs @@ -0,0 +1,151 @@ +//! FLATTEN facts: the element each FLATTEN reads, and the whole elements a select item copies. + +use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::functions::models::Function; +use sqlquack_syntax::expressions::names::models::Literal; + +use crate::query_facts::models::{FlattenInput, WholeValue}; + +const VALUE_COLUMN: &str = "value"; + +/// Functions whose result embeds each argument whole. +const EMBEDDING_FUNCTIONS: [&str; 5] = [ + "object_construct", + "object_construct_keep_null", + "object_insert", + "array_construct", + "array_construct_compact", +]; + +/// Conversions whose result is their first argument whole, in another type. +const CONVERSION_FUNCTIONS: [&str; 6] = [ + "to_json", + "to_char", + "to_variant", + "to_object", + "to_array", + "to_varchar", +]; + +/// The `FLATTEN` call a FROM item runs, as `LATERAL FLATTEN(...)` or `TABLE(FLATTEN(...))`. +pub(crate) fn flatten_call(expression: &Expression) -> Option<&Function> { + match expression { + Expression::Lateral(lateral) => flatten_call(&lateral.this), + Expression::Function(function) if function.name.eq_ignore_ascii_case("table") => { + function.args.first().and_then(flatten_call) + } + Expression::Function(function) if function.name.eq_ignore_ascii_case("flatten") => { + Some(function) + } + _ => None, + } +} + +/// The column path a `FLATTEN` reads, such as `o.value:lines`, when its input is one. +pub(crate) fn flatten_input(function: &Function) -> Option { + let input: &Expression = function.args.iter().find_map(|arg| match arg { + Expression::NamedArgument(named) => named + .name + .name + .eq_ignore_ascii_case("input") + .then_some(&named.value), + positional => Some(positional), + })?; + path_root(input) +} + +/// The column `expression` reads a path of, with the path's first key. +fn path_root(expression: &Expression) -> Option { + let (base, key): (&Expression, Option) = match expression { + Expression::Column(column) => { + return Some(FlattenInput { + qualifier: column.table.as_ref()?.name.to_ascii_lowercase(), + column: column.name.name.to_ascii_lowercase(), + key: None, + }); + } + Expression::Cast(cast) => return path_root(&cast.this), + Expression::JSONExtract(extract) => (&extract.this, literal_key(&extract.expression)), + Expression::Subscript(subscript) => (&subscript.this, literal_key(&subscript.index)), + Expression::Function(function) if is_path_function(function) => ( + function.args.first()?, + function.args.get(1).and_then(literal_key), + ), + _ => return None, + }; + let mut root: FlattenInput = path_root(base)?; + if root.key.is_none() { + root.key = key; + } + Some(root) +} + +fn is_path_function(function: &Function) -> bool { + ["get", "get_path"] + .iter() + .any(|name| function.name.eq_ignore_ascii_case(name)) +} + +/// The first key of a path literal such as `'lines.sku'` or `'lines'`. +fn literal_key(expression: &Expression) -> Option { + let Expression::Literal(literal) = expression else { + return None; + }; + let Literal::String(path) = literal.as_ref() else { + return None; + }; + let key: &str = path + .trim_start_matches('$') + .trim_start_matches('.') + .split(['.', '[']) + .next()?; + (!key.is_empty()).then(|| key.trim_matches('"').to_ascii_lowercase()) +} + +/// Every `.value` that `expression` copies whole, with the keys `OBJECT_DELETE` removes. +pub(crate) fn whole_values(expression: &Expression) -> Vec { + let mut found: Vec = Vec::new(); + let mut pending: Vec<(&Expression, Vec)> = vec![(expression, Vec::new())]; + while let Some((node, removed)) = pending.pop() { + match node { + Expression::Column(column) if column.name.name.eq_ignore_ascii_case(VALUE_COLUMN) => { + if let Some(table) = &column.table { + found.push(WholeValue { + qualifier: table.name.to_ascii_lowercase(), + removed, + }); + } + } + Expression::Paren(paren) => pending.push((&paren.this, removed)), + Expression::Cast(cast) | Expression::TryCast(cast) | Expression::SafeCast(cast) => { + pending.push((&cast.this, removed)); + } + Expression::Function(function) + if CONVERSION_FUNCTIONS + .iter() + .any(|name| function.name.eq_ignore_ascii_case(name)) => + { + if let Some(value) = function.args.first() { + pending.push((value, removed)); + } + } + Expression::Function(function) + if function.name.eq_ignore_ascii_case("object_delete") => + { + let mut args = function.args.iter(); + if let Some(object) = args.next() { + pending.push((object, args.filter_map(literal_key).collect())); + } + } + Expression::Function(function) + if EMBEDDING_FUNCTIONS + .iter() + .any(|name| function.name.eq_ignore_ascii_case(name)) => + { + pending.extend(function.args.iter().map(|arg| (arg, Vec::new()))); + } + _ => {} + } + } + found +} diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/mod.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/mod.rs index 90c1051a7c..a6eda9b1fa 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/mod.rs @@ -1,3 +1,4 @@ pub mod facts; +pub mod flatten; pub mod shape; pub mod tokens; diff --git a/crates/sqlbuild-rules/src/query_facts/models.rs b/crates/sqlbuild-rules/src/query_facts/models.rs index a5688f1e06..bd64b65dd8 100644 --- a/crates/sqlbuild-rules/src/query_facts/models.rs +++ b/crates/sqlbuild-rules/src/query_facts/models.rs @@ -95,6 +95,8 @@ pub(crate) struct ExpressionItem { pub(crate) start: Option, /// The normalized characters from the expression's first token to its last, alias excluded. pub(crate) span: Option<(usize, usize)>, + /// The FLATTEN elements the expression copies whole. + pub(crate) whole_values: Vec, } #[derive(Debug)] @@ -112,9 +114,29 @@ pub(crate) enum RelationFacts { alias: Option, }, DynamicPivot, + /// A `LATERAL FLATTEN(...)` or `TABLE(FLATTEN(...))` item and the column path it reads. + Flatten { + alias: Option, + input: Option, + }, Other, } +/// The column path a FLATTEN reads: `o.value:lines` is qualifier `o`, column `value`, key `lines`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct FlattenInput { + pub(crate) qualifier: String, + pub(crate) column: String, + pub(crate) key: Option, +} + +/// An `.value` a select item copies whole, less the keys `OBJECT_DELETE` removes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct WholeValue { + pub(crate) qualifier: String, + pub(crate) removed: Vec, +} + #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub(crate) struct ColumnFact { pub(crate) name: String, diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs index 222049ed26..f098f35a22 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -33,7 +33,15 @@ pub(crate) const LAYOUT_RULE_PREFIXES: [&str; 2] = ["SQBRPROJECT2", "SQBRDECLARA pub(crate) const TEST_RULE_PREFIX: &str = "SQBRTEST"; pub(crate) const GRAPH_EDGE_RULE_CODE: &str = "SQBRGRAPH101"; pub(crate) const REPLACEABLE_SKILL_STATES: [&str; 3] = ["fresh", "missing", "stale"]; -pub(crate) const RULE_EXAMPLES: [(&str, &str); 28] = [ +pub(crate) const RULE_EXAMPLES: [(&str, &str); 30] = [ + ( + "SQBRPERFORMANCE101", + "SELECT o.value:customer_id, l.value:sku FROM raw_orders r, LATERAL FLATTEN(r.payload:orders) o, LATERAL FLATTEN(o.value:lines) l.", + ), + ( + "SQBRPERFORMANCE102", + "order_lines (order_id, line_no) selects o.payload:status, not the whole orders payload keyed by order_id.", + ), ( "SQLCLEAN040", "Compute category_key in an input CTE, then JOIN categories AS c ON o.category_key = c.category_key.", diff --git a/crates/sqlbuild-rules/src/rule_results/main/format_skipped_models.rs b/crates/sqlbuild-rules/src/rule_results/main/format_skipped_models.rs new file mode 100644 index 0000000000..ab43ce9f51 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/format_skipped_models.rs @@ -0,0 +1,21 @@ +//! The note counting the models SQBRPERFORMANCE102 skipped, listing them when verbose. + +/// The note for `skipped` `(path, reason)` pairs, or `None` when no model was skipped. +pub fn format_skipped_models(skipped: &[(String, String)], verbose: bool) -> Option { + if skipped.is_empty() { + return None; + } + let mut note: String = format!( + "SQBRPERFORMANCE102 skipped {} model{} without column lineage or inferred types", + skipped.len(), + if skipped.len() == 1 { "" } else { "s" } + ); + if verbose { + for (path, reason) in skipped { + note.push_str(&format!("\n {path}: {reason}")); + } + } else { + note.push_str("; `sqb plan --verbose` lists them"); + } + Some(note) +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/mod.rs b/crates/sqlbuild-rules/src/rule_results/main/mod.rs index 3cb9c4ec7c..17da60c2f7 100644 --- a/crates/sqlbuild-rules/src/rule_results/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_results/main/mod.rs @@ -2,5 +2,6 @@ pub mod default_rules_notice; pub mod duplicate_literal_hints; pub mod format_result; pub mod format_rules_progress; +pub mod format_skipped_models; pub mod format_skipped_rules; pub mod group_unevaluated; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/contract_name_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/contract_name_types.rs similarity index 100% rename from crates/sqlbuild-rules/src/rules/_helpers/contract_name_types.rs rename to crates/sqlbuild-rules/src/rules/_helpers/contracts/contract_name_types.rs diff --git a/crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/dynamic_contracts.rs similarity index 94% rename from crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs rename to crates/sqlbuild-rules/src/rules/_helpers/contracts/dynamic_contracts.rs index acf4c1e5ef..26c0781ec5 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/contracts/dynamic_contracts.rs @@ -3,8 +3,10 @@ use std::collections::BTreeMap; use crate::constants::{ENFORCED_CONTRACT, SELECT_STAR_MODEL_RULE_CODE}; use crate::models::{Fault, Model, RuleMetadata, RulesConfig}; use crate::query_facts::models::{QueryFacts, RelationFacts, SelectFacts}; -use crate::rules::_helpers::evaluation::sole_table_name; -use crate::rules::_helpers::{contract_name_types, explicit_output_types, typed_contract_columns}; +use crate::rules::_helpers::contracts::{ + contract_name_types, explicit_output_types, typed_contract_columns, +}; +use crate::rules::_helpers::dispatch::evaluation::sole_table_name; use crate::rules::models::FaultCollector; pub(crate) fn evaluate_without_query( diff --git a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/explicit_output_types.rs similarity index 99% rename from crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs rename to crates/sqlbuild-rules/src/rules/_helpers/contracts/explicit_output_types.rs index 04ada17cb7..e982824514 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/contracts/explicit_output_types.rs @@ -4,7 +4,7 @@ use sqlbuild_analysis::type_system::main::normalize_type::normalize_type; use crate::constants::ENFORCED_CONTRACT; use crate::models::{Column, Fault, Model, RuleMetadata}; use crate::query_facts::models::{ExpressionItem, ItemFacts, QueryFacts, SelectFacts, SetFacts}; -use crate::rules::_helpers::evaluation::{ +use crate::rules::_helpers::dispatch::evaluation::{ ParsedModel, plain_select, reads_dependency, sole_table_name, }; use crate::rules::models::FaultCollector; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/contracts/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/mod.rs new file mode 100644 index 0000000000..f34a766f0b --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/contracts/mod.rs @@ -0,0 +1,4 @@ +pub mod contract_name_types; +pub mod dynamic_contracts; +pub mod explicit_output_types; +pub mod typed_contract_columns; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/typed_contract_columns.rs b/crates/sqlbuild-rules/src/rules/_helpers/contracts/typed_contract_columns.rs similarity index 100% rename from crates/sqlbuild-rules/src/rules/_helpers/typed_contract_columns.rs rename to crates/sqlbuild-rules/src/rules/_helpers/contracts/typed_contract_columns.rs diff --git a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/catalogue.rs similarity index 96% rename from crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs rename to crates/sqlbuild-rules/src/rules/_helpers/dispatch/catalogue.rs index 6242dc9e10..c36166b7b9 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/catalogue.rs @@ -21,7 +21,7 @@ macro_rules! rule { remediation: $remediation.into(), guidance: None, implementation_fingerprint: native_build_identity().into(), - enabled_by_default: true, + enabled_by_default: !matches!($family, "performance"), project_wide: matches!( $code, "SQBRDECLARATION201" @@ -375,6 +375,22 @@ pub(crate) fn catalogue() -> Vec { "SQL unit tests must not only prove that empty inputs produce no rows", "Mock representative input rows and assert concrete transformed output in __expected__ or a targeted __assert__ CTE, or delete this test. An empty-input test is legitimate only when it asserts concrete output, for example a global aggregate that must return one zero-valued summary row compared against expected rows. List a reviewed exception by test name in [rules.rule_options.SQBRTEST203] allowed_tests.", ), + rule!( + "SQBRPERFORMANCE101", + RuleScope::Model, + "performance", + "flatten-parent-copy", + "flattened rows must not copy the parent element whose child array they flatten", + "Read the fields you need from the parent (o.value:customer_id), or remove the flattened child array first with OBJECT_DELETE(o.value, 'lines').", + ), + rule!( + "SQBRPERFORMANCE102", + RuleScope::Model, + "performance", + "json-finer-grain", + "semi-structured columns must not repeat on a finer grain than their input", + "Keep the payload on the coarser model and join to it when needed, or select only the fields this grain needs.", + ), rule!( "SQBRTEST301", RuleScope::Project, diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/evaluation.rs similarity index 96% rename from crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs rename to crates/sqlbuild-rules/src/rules/_helpers/dispatch/evaluation.rs index 4e3256ff5e..01bc3dc53f 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/evaluation.rs @@ -10,6 +10,7 @@ use crate::constants::{ }; use crate::lint_text::constants::LINT_UNAVAILABLE_CODE; use crate::models::{Declaration, EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; +use crate::query_facts::constants::SNOWFLAKE_DIALECT; use crate::query_facts::main::dependency_texts::dependency_texts; use crate::query_facts::main::facts_available::facts_available; use crate::query_facts::main::model_shape::model_shape; @@ -18,11 +19,13 @@ use crate::query_facts::models::{ CallKind, ColumnFact, ComparisonFacts, ItemFacts, ModelCall, ModelSource, QueryFacts, RelationFacts, SelectFacts, }; -use crate::rules::_helpers::domain_layout::folder_layer_details; -use crate::rules::_helpers::{ - contract_name_types, dynamic_contracts, explicit_output_types, model_layers, numeric_decisions, - sql_test_rules, typed_contract_columns, +use crate::rules::_helpers::contracts::{ + contract_name_types, dynamic_contracts, explicit_output_types, typed_contract_columns, }; +use crate::rules::_helpers::layers::domain_layout::folder_layer_details; +use crate::rules::_helpers::layers::model_layers; +use crate::rules::_helpers::performance::{flatten_parent_copy, json_finer_grain}; +use crate::rules::_helpers::sql_checks::{numeric_decisions, sql_test_rules}; use crate::rules::models::{ FaultCollector, ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride, }; @@ -215,6 +218,16 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result bool { !rule.custom && (rule.code.starts_with("SQBRMODEL") || rule.code.starts_with("SQBRCONTRACT") + || rule.code.starts_with("SQBRPERFORMANCE") || matches!( rule.code.as_str(), "SQBRGRAPH102" @@ -350,7 +364,7 @@ fn requires_model_query(rule: &RuleMetadata) -> bool { )) } -pub(super) fn plain_select(select: &SelectFacts, allow_star: bool) -> bool { +pub(in crate::rules::_helpers) fn plain_select(select: &SelectFacts, allow_star: bool) -> bool { if select.items.is_empty() || select.filtered || !select.joins.is_empty() { return false; } @@ -378,7 +392,7 @@ fn dependency_kind(relation: &RelationFacts) -> Option { } } -pub(super) fn reads_dependency(select: &SelectFacts) -> bool { +pub(in crate::rules::_helpers) fn reads_dependency(select: &SelectFacts) -> bool { select.from.len() == 1 && select.joins.is_empty() && dependency_kind(&select.from[0]).is_some() } @@ -386,7 +400,7 @@ fn dependency_import(query: &QueryFacts) -> bool { query.root_select().is_some_and(reads_dependency) && plain_projection(query, true) } -pub(super) fn sole_table_name(query: &QueryFacts) -> Option<&str> { +pub(in crate::rules::_helpers) fn sole_table_name(query: &QueryFacts) -> Option<&str> { let select = query.root_select()?; if select.from.len() != 1 || !select.joins.is_empty() { return None; @@ -579,15 +593,15 @@ fn select_star( } #[derive(Clone)] -pub(super) struct NameParts { - pub(super) domain: String, - pub(super) layer: String, - pub(super) subject: String, - pub(super) source: Option, - pub(super) is_view: bool, +pub(in crate::rules::_helpers) struct NameParts { + pub(in crate::rules::_helpers) domain: String, + pub(in crate::rules::_helpers) layer: String, + pub(in crate::rules::_helpers) subject: String, + pub(in crate::rules::_helpers) source: Option, + pub(in crate::rules::_helpers) is_view: bool, } -pub(super) fn parse_name(name: &str) -> Option { +pub(in crate::rules::_helpers) fn parse_name(name: &str) -> Option { let parts: Vec<&str> = name.split("__").collect(); if !(3..=4).contains(&parts.len()) { return None; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/dispatch/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/mod.rs new file mode 100644 index 0000000000..508c91acbc --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/dispatch/mod.rs @@ -0,0 +1,2 @@ +pub mod catalogue; +pub mod evaluation; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/domain_layout.rs b/crates/sqlbuild-rules/src/rules/_helpers/layers/domain_layout.rs similarity index 99% rename from crates/sqlbuild-rules/src/rules/_helpers/domain_layout.rs rename to crates/sqlbuild-rules/src/rules/_helpers/layers/domain_layout.rs index a39aa84b97..9871b1b1b3 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/domain_layout.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/layers/domain_layout.rs @@ -1,6 +1,6 @@ use crate::configuration::main::setting_help::setting_help; use crate::models::{DeclarationKind, Fault, Model, RuleMetadata, ScopeDeclaration, ScopeKind}; -use crate::rules::_helpers::evaluation::path_fault; +use crate::rules::_helpers::dispatch::evaluation::path_fault; use crate::rules::models::ProjectEvaluationRequest; use std::collections::{BTreeMap, BTreeSet}; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/layers/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/layers/mod.rs new file mode 100644 index 0000000000..7a9531f2f4 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/layers/mod.rs @@ -0,0 +1,2 @@ +pub mod domain_layout; +pub mod model_layers; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/layers/model_layers.rs similarity index 95% rename from crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs rename to crates/sqlbuild-rules/src/rules/_helpers/layers/model_layers.rs index 1f1b11ff0c..4132f1953d 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/layers/model_layers.rs @@ -8,7 +8,7 @@ use crate::constants::{ PUBLISHED_LAYERS_OPTION, REFERENCE_KIND, STAGING_LAYER_DIRECTORY, }; use crate::models::{EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; -use crate::rules::_helpers::evaluation::{parse_name, path_fault}; +use crate::rules::_helpers::dispatch::evaluation::{parse_name, path_fault}; use crate::rules::models::{FaultCollector, ProjectEvaluationRequest}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::text::main::glob_match::glob_match; @@ -19,6 +19,8 @@ const INTERMEDIATE_FOLDER: &str = "intermediate/enriched"; const ENTITY_SHAPE: &str = "____"; const QUALIFIED_SHAPE: &str = "______"; const MODEL_FILE_SUFFIX: &str = ".sql"; +/// The rule whose published layers and view-of-own-entity test define a published view. +const PUBLISHED_READS_RULE: &str = "SQBRGRAPH103"; pub(crate) fn explicit_model_schema(model: &Model, rule: &RuleMetadata, faults: &FaultCollector) { if model @@ -303,6 +305,22 @@ pub(crate) fn same_domain_published_reads( } } +/// Whether `readers` is not empty and each one is a published view of `model`'s own entity. +pub(crate) fn backs_published_view(model: &str, readers: &[String], config: &RulesConfig) -> bool { + let layers: Vec = published_layers(config, PUBLISHED_READS_RULE); + let Some(producer) = published_name(model).filter(|name| layers.contains(&name.layer)) else { + return false; + }; + !readers.is_empty() + && readers.iter().all(|reader| { + published_name(reader).is_some_and(|consumer| { + consumer.domain == producer.domain + && layers.contains(&consumer.layer) + && is_view_of_own_entity(&consumer, &producer) + }) + }) +} + /// `X___v__[__]` reading `X____` or the same qualifier. fn is_view_of_own_entity(consumer: &PublishedName, producer: &PublishedName) -> bool { consumer.is_view diff --git a/crates/sqlbuild-rules/src/rules/_helpers/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs index 18833e9d3d..1253a56fe9 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs @@ -1,10 +1,5 @@ -pub mod catalogue; -pub mod contract_name_types; -pub mod domain_layout; -pub mod dynamic_contracts; -pub mod evaluation; -pub mod explicit_output_types; -pub mod model_layers; -pub mod numeric_decisions; -pub mod sql_test_rules; -pub mod typed_contract_columns; +pub mod contracts; +pub mod dispatch; +pub mod layers; +pub mod performance; +pub mod sql_checks; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/performance/flatten_parent_copy.rs b/crates/sqlbuild-rules/src/rules/_helpers/performance/flatten_parent_copy.rs new file mode 100644 index 0000000000..bd5a878ccf --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/performance/flatten_parent_copy.rs @@ -0,0 +1,134 @@ +//! SQBRPERFORMANCE101: a flattened row must not copy the parent element whose child array it flattens. + +use std::collections::BTreeMap; + +use crate::models::{Fault, RuleMetadata}; +use crate::query_facts::models::{ + ExpressionItem, FlattenInput, ItemFacts, QueryFacts, RelationFacts, SelectFacts, SetFacts, +}; +use crate::rules::_helpers::dispatch::evaluation::ParsedModel; +use crate::rules::models::FaultCollector; + +const VALUE_COLUMN: &str = "value"; + +pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &FaultCollector) { + for select in query_selects(parsed.facts) { + let children: BTreeMap<&str, Vec>> = flattened_parents(select); + for item in &select.items { + let ItemFacts::Expression(expression) = item else { + continue; + }; + if let Some(copied) = copied_parent(expression, &children) { + faults.push(fault(parsed, rule, expression, copied)); + } + } + } +} + +/// Every SELECT in `query`, its CTEs first. +fn query_selects(query: &QueryFacts) -> Vec<&SelectFacts> { + let mut selects: Vec<&SelectFacts> = Vec::new(); + for cte in &query.ctes { + selects.extend(query_selects(&cte.query)); + } + selects.extend(set_selects(&query.body)); + selects +} + +fn set_selects(set: &SetFacts) -> Vec<&SelectFacts> { + match set { + SetFacts::Select(select) => vec![select], + SetFacts::Query(query) => query_selects(query), + SetFacts::SetOperation { left, right, .. } => { + let mut selects: Vec<&SelectFacts> = set_selects(left); + selects.extend(set_selects(right)); + selects + } + SetFacts::Other => Vec::new(), + } +} + +/// Each FLATTEN alias in `select` that a deeper FLATTEN reads, with the child keys read from it. +fn flattened_parents(select: &SelectFacts) -> BTreeMap<&str, Vec>> { + let relations = || select.from.iter().chain(&select.joins); + let aliases: Vec<&str> = relations() + .filter_map(|relation| match relation { + RelationFacts::Flatten { + alias: Some(alias), .. + } => Some(alias.as_str()), + _ => None, + }) + .collect(); + let mut parents: BTreeMap<&str, Vec>> = BTreeMap::new(); + for relation in relations() { + if let RelationFacts::Flatten { + input: + Some(FlattenInput { + qualifier, + column, + key, + }), + .. + } = relation + && column == VALUE_COLUMN + && aliases.contains(&qualifier.as_str()) + { + parents + .entry(qualifier.as_str()) + .or_default() + .push(key.as_deref()); + } + } + parents +} + +/// The parent `expression` copies whole and a child key it keeps, unless `OBJECT_DELETE` removes them all. +fn copied_parent<'a>( + expression: &'a ExpressionItem, + parents: &BTreeMap<&str, Vec>>, +) -> Option<(&'a str, Option<&'a str>)> { + expression.whole_values.iter().find_map(|whole| { + let keys = parents.get(whole.qualifier.as_str())?; + let kept: Option<&str> = kept_key(keys, &whole.removed)?; + Some((whole.qualifier.as_str(), kept)) + }) +} + +/// The first child key that `removed` does not take out of the parent; `None` reads it all. +fn kept_key<'a>(keys: &[Option<&'a str>], removed: &[String]) -> Option> { + for key in keys { + match key { + None => return Some(None), + Some(key) if !removed.iter().any(|name| name == key) => return Some(Some(key)), + Some(_) => {} + } + } + None +} + +fn fault( + parsed: &ParsedModel<'_>, + rule: &RuleMetadata, + expression: &ExpressionItem, + (parent, key): (&str, Option<&str>), +) -> Fault { + let (line, column) = expression + .start + .map_or((1, 1), |offset| parsed.source.line_column(offset)); + let output: &str = expression.output_name.as_deref().unwrap_or("this column"); + let child: String = key.map_or_else( + || "its child array".to_owned(), + |key| format!("{parent}.value:{key}"), + ); + Fault { + unevaluated: false, + code: rule.code.clone(), + path: parsed.model.relative_path.clone(), + line, + column, + message: format!( + "{output} copies {parent}.value whole onto every row flattened from {child}" + ), + remediation: rule.remediation.clone(), + } +} diff --git a/crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs b/crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs new file mode 100644 index 0000000000..c3ac43fd87 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/performance/json_finer_grain.rs @@ -0,0 +1,119 @@ +//! SQBRPERFORMANCE102: a semi-structured column must not repeat on a finer grain than its input. + +use crate::models::{Fault, GrainColumn, JsonGrain, RuleMetadata, RulesConfig}; +use crate::query_facts::models::{ItemFacts, SetFacts}; +use crate::rules::_helpers::dispatch::evaluation::ParsedModel; +use crate::rules::_helpers::layers::model_layers::backs_published_view; +use crate::rules::models::FaultCollector; + +pub(crate) fn evaluate( + parsed: &ParsedModel<'_>, + config: &RulesConfig, + rule: &RuleMetadata, + faults: &FaultCollector, +) { + let Some(grain) = &parsed.model.json_grain else { + return; + }; + if grain.skipped.is_some() || backs_published_view(&parsed.model.name, &grain.readers, config) { + return; + } + for column in &grain.columns { + if let Some((input, input_key, model_key)) = coarser_input(column, grain) { + let input: &str = input.split_once(':').map_or(input, |(_, name)| name); + let (line, column_number) = output_start(&parsed.facts.body, &column.name) + .map_or((1, 1), |offset| parsed.source.line_column(offset)); + faults.push(Fault { + unevaluated: false, + code: rule.code.clone(), + path: parsed.model.relative_path.clone(), + line, + column: column_number, + message: format!( + "{} copies {} from {input}, one row per ({}), onto rows keyed by ({}), so the \ + same value repeats on every row", + column.name, + column.data_type.as_deref().unwrap_or_default(), + input_key.join(", "), + model_key.join(", "), + ), + remediation: rule.remediation.clone(), + }); + } + } +} + +/// An input whose key is a strict subset of a model key, unless another of its keys covers one. +fn coarser_input<'a>( + column: &'a GrainColumn, + grain: &'a JsonGrain, +) -> Option<(&'a str, &'a [String], &'a [String])> { + for input in &column.inputs { + let Some(keys) = grain.input_keys.get(input) else { + continue; + }; + let one_model_row_per_input_row = keys + .iter() + .any(|input_key| covers_model_key(input_key, &grain.model_keys)); + if one_model_row_per_input_row { + continue; + } + for input_key in keys { + if let Some(model_key) = finer_model_key(input_key, &grain.model_keys) { + return Some((input.as_str(), input_key.as_slice(), model_key)); + } + } + } + None +} + +fn covers_model_key(input_key: &[String], model_keys: &[Vec]) -> bool { + model_keys + .iter() + .any(|model_key| subset(model_key, input_key)) +} + +fn finer_model_key<'a>( + input_key: &[String], + model_keys: &'a [Vec], +) -> Option<&'a [String]> { + model_keys + .iter() + .find(|model_key| input_key.len() < model_key.len() && subset(input_key, model_key)) + .map(Vec::as_slice) +} + +fn subset(smaller: &[String], larger: &[String]) -> bool { + smaller + .iter() + .all(|column| contains_ignoring_case(larger, column)) +} + +fn contains_ignoring_case(columns: &[String], name: &str) -> bool { + columns + .iter() + .any(|column| column.eq_ignore_ascii_case(name)) +} + +/// Where the root select projects `name`, following the left branch of set operations. +fn output_start(body: &SetFacts, name: &str) -> Option { + match body { + SetFacts::Select(select) => { + for item in &select.items { + if let ItemFacts::Expression(expression) = item + && expression + .output_name + .as_deref() + .is_some_and(|output| output.eq_ignore_ascii_case(name)) + && expression.start.is_some() + { + return expression.start; + } + } + None + } + SetFacts::Query(query) => output_start(&query.body, name), + SetFacts::SetOperation { left, .. } => output_start(left, name), + SetFacts::Other => None, + } +} diff --git a/crates/sqlbuild-rules/src/rules/_helpers/performance/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/performance/mod.rs new file mode 100644 index 0000000000..0fd5391a33 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/performance/mod.rs @@ -0,0 +1,2 @@ +pub mod flatten_parent_copy; +pub mod json_finer_grain; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/mod.rs new file mode 100644 index 0000000000..5957b15086 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/mod.rs @@ -0,0 +1,2 @@ +pub mod numeric_decisions; +pub mod sql_test_rules; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/numeric_decisions.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/numeric_decisions.rs similarity index 100% rename from crates/sqlbuild-rules/src/rules/_helpers/numeric_decisions.rs rename to crates/sqlbuild-rules/src/rules/_helpers/sql_checks/numeric_decisions.rs diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/sql_test_rules.rs similarity index 99% rename from crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs rename to crates/sqlbuild-rules/src/rules/_helpers/sql_checks/sql_test_rules.rs index 1faa38d2e7..608e2ba36a 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_checks/sql_test_rules.rs @@ -8,7 +8,7 @@ use crate::query_facts::main::model_shape::model_shape; use crate::query_facts::main::parse_fixture::parse_fixture; use crate::query_facts::main::stub_call::stub_call; use crate::query_facts::models::{CallKind, ParsedFixture}; -use crate::rules::_helpers::evaluation::{path_fault, rules_unavailable}; +use crate::rules::_helpers::dispatch::evaluation::{path_fault, rules_unavailable}; use crate::rules::models::ProjectEvaluationRequest; use serde::{Deserialize, Serialize}; use sqlbuild_analysis::compiler::main::sql_test_fixture_facts::fixture_facts; diff --git a/crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs b/crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs index 3ddf9d2bc3..6125301736 100644 --- a/crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs @@ -1,5 +1,5 @@ use crate::models::{CustomRule, RuleMetadata}; pub(crate) fn assemble_catalogue(custom: &[CustomRule]) -> Result, String> { - crate::rules::_helpers::catalogue::with_custom(custom) + crate::rules::_helpers::dispatch::catalogue::with_custom(custom) } diff --git a/crates/sqlbuild-rules/src/rules/main/catalogue.rs b/crates/sqlbuild-rules/src/rules/main/catalogue.rs index 88c4b1aeac..c3646042fa 100644 --- a/crates/sqlbuild-rules/src/rules/main/catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/main/catalogue.rs @@ -1,5 +1,5 @@ use crate::models::RuleMetadata; pub fn catalogue() -> Vec { - crate::rules::_helpers::catalogue::catalogue() + crate::rules::_helpers::dispatch::catalogue::catalogue() } diff --git a/crates/sqlbuild-rules/src/rules/main/compact_sql.rs b/crates/sqlbuild-rules/src/rules/main/compact_sql.rs index a1fa2a7c57..5f7909a1cb 100644 --- a/crates/sqlbuild-rules/src/rules/main/compact_sql.rs +++ b/crates/sqlbuild-rules/src/rules/main/compact_sql.rs @@ -2,5 +2,5 @@ use sqlbuild_sqltext::sql_scan::models::QuotePolicy; /// `sql` lowercased without whitespace or comments outside quoted text. pub(crate) fn compact_sql(sql: &str, policy: QuotePolicy) -> String { - crate::rules::_helpers::numeric_decisions::compact_sql(sql, policy) + crate::rules::_helpers::sql_checks::numeric_decisions::compact_sql(sql, policy) } diff --git a/crates/sqlbuild-rules/src/rules/main/evaluate.rs b/crates/sqlbuild-rules/src/rules/main/evaluate.rs index 5b8dfe173a..3d87e61504 100644 --- a/crates/sqlbuild-rules/src/rules/main/evaluate.rs +++ b/crates/sqlbuild-rules/src/rules/main/evaluate.rs @@ -2,5 +2,5 @@ use crate::models::Fault; use crate::rules::models::ModelEvaluationRequest; pub(crate) fn evaluate_model(request: ModelEvaluationRequest<'_>) -> Result, String> { - crate::rules::_helpers::evaluation::evaluate_model(request) + crate::rules::_helpers::dispatch::evaluation::evaluate_model(request) } diff --git a/crates/sqlbuild-rules/src/rules/main/evaluate_project.rs b/crates/sqlbuild-rules/src/rules/main/evaluate_project.rs index 0154a81dba..a689f19a9f 100644 --- a/crates/sqlbuild-rules/src/rules/main/evaluate_project.rs +++ b/crates/sqlbuild-rules/src/rules/main/evaluate_project.rs @@ -6,15 +6,11 @@ use crate::rules::models::ProjectEvaluationRequest; pub(crate) fn evaluate_project( request: ProjectEvaluationRequest<'_>, ) -> Result, String> { - let mut faults = crate::rules::_helpers::domain_layout::evaluate_project(&request); - faults.extend(crate::rules::_helpers::evaluation::evaluate_project_rules( - &request, - )); + let mut faults = crate::rules::_helpers::layers::domain_layout::evaluate_project(&request); + faults.extend(crate::rules::_helpers::dispatch::evaluation::evaluate_project_rules(&request)); if let Some(rule) = request.selected.get(NAMED_SELECTOR_LIMIT_CODE) { faults.extend(evaluate_orchestration_rules(request.request, rule)); } - faults.extend(crate::rules::_helpers::sql_test_rules::evaluate_project( - request, - )?); + faults.extend(crate::rules::_helpers::sql_checks::sql_test_rules::evaluate_project(request)?); Ok(faults) } diff --git a/crates/sqlbuild-rules/src/rules/main/fingerprint.rs b/crates/sqlbuild-rules/src/rules/main/fingerprint.rs index 8799a90c04..feea9f4275 100644 --- a/crates/sqlbuild-rules/src/rules/main/fingerprint.rs +++ b/crates/sqlbuild-rules/src/rules/main/fingerprint.rs @@ -5,5 +5,5 @@ pub(crate) fn fingerprint( config: &RulesConfig, dialect: &str, ) -> Result { - crate::rules::_helpers::catalogue::fingerprint(rules, config, dialect) + crate::rules::_helpers::dispatch::catalogue::fingerprint(rules, config, dialect) } diff --git a/crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs b/crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs index a48c151ed0..d04072582f 100644 --- a/crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs +++ b/crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs @@ -4,5 +4,5 @@ use crate::rules::models::ResolvedThresholdOverride; pub(crate) fn resolve_threshold_overrides( config: &RulesConfig, ) -> Result, String> { - crate::rules::_helpers::evaluation::resolve_threshold_overrides(config) + crate::rules::_helpers::dispatch::evaluation::resolve_threshold_overrides(config) } diff --git a/crates/sqlbuild-rules/src/rules/main/select.rs b/crates/sqlbuild-rules/src/rules/main/select.rs index 45f8eef747..8fb6cc208e 100644 --- a/crates/sqlbuild-rules/src/rules/main/select.rs +++ b/crates/sqlbuild-rules/src/rules/main/select.rs @@ -5,5 +5,5 @@ pub(crate) fn select<'a>( selected: Option<&[String]>, ignored: &[String], ) -> Result, String> { - crate::rules::_helpers::catalogue::select(catalogue, selected, ignored) + crate::rules::_helpers::dispatch::catalogue::select(catalogue, selected, ignored) } diff --git a/crates/sqlbuild-rules/src/rules/main/selected_codes.rs b/crates/sqlbuild-rules/src/rules/main/selected_codes.rs index d2c4ea82e5..6a4604beae 100644 --- a/crates/sqlbuild-rules/src/rules/main/selected_codes.rs +++ b/crates/sqlbuild-rules/src/rules/main/selected_codes.rs @@ -1,3 +1,3 @@ pub fn selected_codes_json(request_json: &str) -> Result { - crate::rules::_helpers::catalogue::selected_codes_json(request_json) + crate::rules::_helpers::dispatch::catalogue::selected_codes_json(request_json) } diff --git a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs index fe88b7198c..e52ad46cb9 100644 --- a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs +++ b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs @@ -1,7 +1,7 @@ //! `selected_sql_codes`: the built-in SQL Rule codes a `select`/`ignore` pair enables. use crate::models::RulesCodeGrammar; -use crate::rules::_helpers::catalogue::{matches_any_selector, sql_lint_selection}; +use crate::rules::_helpers::dispatch::catalogue::{matches_any_selector, sql_lint_selection}; use crate::rules::main::catalogue::catalogue; /// SQL Rule codes `select` enables (SQLClean's defaults when it is `None`) that no `ignore` selector matches, in catalogue order. diff --git a/crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs b/crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs index 8f79e2c170..90bd8903a2 100644 --- a/crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs +++ b/crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs @@ -1,5 +1,5 @@ use crate::models::EvaluateRequest; pub(crate) fn annotate(request: EvaluateRequest) -> EvaluateRequest { - crate::rules::_helpers::sql_test_rules::annotate_empty_input_only_tests(request) + crate::rules::_helpers::sql_checks::sql_test_rules::annotate_empty_input_only_tests(request) } diff --git a/fensu.toml b/fensu.toml index 9ab61266bc..0868e9b020 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1680,7 +1680,8 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 197891 } +# Raised from 197,891: SQBRPERFORMANCE102 bindings plumbing, +77 with sqlbuild-python-base. +thresholds = { max_crate_code_lines = 197963 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out. Set after #1514 moved the shared runtime into sqlbuild-python-base, plus 4 lines for the janitor preserve check from #1512." [[targets.rust.threshold_overrides]] diff --git a/scripts/cold_compile_performance/_helpers/dense_project.py b/scripts/cold_compile_performance/_helpers/dense_project.py index 608daf5468..1c7a08422d 100644 --- a/scripts/cold_compile_performance/_helpers/dense_project.py +++ b/scripts/cold_compile_performance/_helpers/dense_project.py @@ -67,8 +67,10 @@ def dense_column_count(index: int) -> int: return 256 + (rank - 990) * 24 -def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: - """Write a deterministic query-heavy project without external inputs or services.""" +def write_dense_compile_project( + *, project_dir: Path, model_count: int, rule_select: str = DENSE_RULE_SELECT +) -> None: + """Write a deterministic query-heavy project; the default rules leave delta compiles on.""" project_dir.mkdir(parents=True) domains: str = ", ".join( @@ -77,7 +79,7 @@ def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: (project_dir / "sqlbuild_project.toml").write_text( 'name = "dense_orders"\nadapter = "snowflake"\n' '[defaults]\ndatabase = "warehouse"\nschema = "analytics"\n' - f"[rules]\n{DENSE_RULE_SELECT}\ndomains = [{domains}]\n", + f"[rules]\n{rule_select}\ndomains = [{domains}]\n", encoding="utf-8", ) source_count: int = max(1, model_count * 24 // 100) diff --git a/scripts/cold_compile_performance/_helpers/varied_project.py b/scripts/cold_compile_performance/_helpers/varied_project.py index 7ef058e0e3..90be85a03a 100644 --- a/scripts/cold_compile_performance/_helpers/varied_project.py +++ b/scripts/cold_compile_performance/_helpers/varied_project.py @@ -8,6 +8,7 @@ write_dense_compile_project, ) from scripts.cold_compile_performance.constants import ( + DENSE_ALL_RULE_SELECT, VARIED_ARRAY_VARIANT, VARIED_CASE_SHARE, VARIED_GROUP_SHARE, @@ -32,7 +33,9 @@ def write_varied_compile_project(*, project_dir: Path, model_count: int) -> None: """Write independent neutral SQL with diverse expressions and shared ancestors.""" - write_dense_compile_project(project_dir=project_dir, model_count=model_count) + write_dense_compile_project( + project_dir=project_dir, model_count=model_count, rule_select=DENSE_ALL_RULE_SELECT + ) for index, model in enumerate(sorted((project_dir / "models").rglob("*.sql"))): contents: str = model.read_text(encoding="utf-8") replacements: dict[str, str] = _dependency_replacements( diff --git a/scripts/cold_compile_performance/constants.py b/scripts/cold_compile_performance/constants.py index 63ed73b060..5f4da9c988 100644 --- a/scripts/cold_compile_performance/constants.py +++ b/scripts/cold_compile_performance/constants.py @@ -6,6 +6,9 @@ DENSE_RULE_SELECT: str = ( 'select = ["SQBR", "SQBRPYTHON", "SQBRPYTHON306", "SQBRPYTHON507", "SQLCLEAN", "XSQBR"]' ) +DENSE_ALL_RULE_SELECT: str = DENSE_RULE_SELECT.replace( + '"SQBR", ', '"SQBR", "SQBRPERFORMANCE101", "SQBRPERFORMANCE102", ' +) DENSE_JOIN_THRESHOLDS: tuple[int, ...] = (55, 75, 90, 99) DENSE_JOIN_COUNTS: tuple[int, ...] = (0, 1, 4, 8, 20) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 6073b2a6e5..12ccec2ff4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -50,6 +50,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Findings and exceptions](concepts/rules/findings-and-exceptions.md) (`concepts/rules/findings-and-exceptions`) - Understand Rule diagnostics and record intentional exceptions safely. - [Python Rules](concepts/rules/python-rules.md) (`concepts/rules/python-rules`) - Check the project's Python for annotations, imports, function shape and the layout of python/. - [Orchestration Rules](concepts/rules/orchestration-rules.md) (`concepts/rules/orchestration-rules`) - Keep job and sensor selections from listing nodes one by one. +- [Performance Rules](concepts/rules/performance-rules.md) (`concepts/rules/performance-rules`) - Catch semi-structured payloads that repeat on every row and multiply storage and scan cost. - [Execution and caching](concepts/rules/execution-and-caching.md) (`concepts/rules/execution-and-caching`) - Understand Rule enforcement order, focused runs, and dependency-aware reuse. ## Ship diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index e99ccba08a..c777522ee9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -310,6 +310,7 @@ Compile runs in full instead when: a model-subject Rule must run again for a model that was not edited (for example because it reads an edited upstream file), or reads more for an edited model than that model's SQL, columns, contract and grain, project files and the source list; +- the project selects `SQBRPERFORMANCE102` on the Snowflake dialect; - the previous compile failed its Rules, since that compile writes no SQL tests, functions or audits to `target/compiled/`, or it reported other errors or warnings in files the edit didn't change; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md index 89a8c77fc8..95b637cf5e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md @@ -165,7 +165,9 @@ with `SQBR`. The `SQBRPYTHON` Rules check the project's Python: annotations, imp shape and the layout of `python/`. They run only when `select` names `SQBRPYTHON` or one of its codes; see [Python Rules](rules/python-rules.md). `SQBRORCHESTRATION101` keeps job and sensor selections from listing nodes one by one; see -[Orchestration Rules](rules/orchestration-rules.md). +[Orchestration Rules](rules/orchestration-rules.md). The `SQBRPERFORMANCE` Rules find +Snowflake JSON payloads that repeat on every row; they run only when `select` names their codes. See +[Performance Rules](rules/performance-rules.md). Projects can define custom Python Rules under `rules/**/*.py`. Their codes begin with `XSQBR`, for example `XSQBRARCH001`. Custom Rules use the same selection, findings, exceptions, execution order, @@ -250,5 +252,6 @@ Rules add about 14% to a cold compile and nothing to a warm one. See Understand diagnostics, exact exceptions, path ignores, and stale checks. Check the project's Python for annotations, imports, function shape and the layout of `python/`. Keep each domain's published layer free of consumer-specific variants. + Catch Snowflake JSON payloads copied onto every row of a finer grain. Learn evaluation order, focused execution, and dependency-aware reuse. List, inspect, run, and generate guidance for configured Rules. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md new file mode 100644 index 0000000000..f1f4d02b2d --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/performance-rules.md @@ -0,0 +1,130 @@ + + +# Performance Rules + +> Catch semi-structured payloads that repeat on every row and multiply storage and scan cost. + +Online: https://sqlbuild.com/docs/concepts/rules/performance-rules/ + +## Contents + +- SQBRPERFORMANCE101: a flattened parent copied onto its children +- SQBRPERFORMANCE102: a payload copied onto a finer grain +- Details + +The `SQBRPERFORMANCE` Rules find JSON payloads copied onto rows that are finer than the payload +itself. Each copy stores and scans the same value again, so the cost grows with the row count. +Name their codes in `select` to turn them on: + +```toml +[rules] +select = ["SQBRPERFORMANCE101", "SQBRPERFORMANCE102"] +``` + +Both Rules are off by default: selecting the `SQBRPERFORMANCE` family alone does not run them. They +run on the Snowflake dialect. There, selecting `SQBRPERFORMANCE102` makes every edit compile the +whole project, because a [delta compile](../../cli/compile.md#compiling-after-an-edit) cannot recheck +the models that read or are read by the edited ones. + +| Code | Name | Reports | +|---|---|---| +| `SQBRPERFORMANCE101` | `flatten-parent-copy` | A FLATTEN element copied whole onto the rows of a deeper FLATTEN of its own child array | +| `SQBRPERFORMANCE102` | `json-finer-grain` | A `VARIANT`, `OBJECT` or `ARRAY` column copied from an input with a coarser unique key | + +## SQBRPERFORMANCE101: a flattened parent copied onto its children + +An output column must not copy the `value` of a FLATTEN whose child array a deeper FLATTEN in the +same query reads. Every child row then carries its whole parent, children included: + +```sql +SELECT + r.order_id, + o.value AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l +``` + +```text +error[SQBRPERFORMANCE101]: line_json copies o.value whole onto every row flattened from o.value:options + --> models/line_options.sql:7:3 +``` + +The parent also counts as copied through a cast, a conversion such as `TO_JSON` or `TO_VARIANT`, +or a constructor that embeds it: `OBJECT_CONSTRUCT`, `OBJECT_INSERT` or `ARRAY_CONSTRUCT`. Paths +into it, such as `o.value:sku`, and other functions of it, such as `MD5(TO_VARCHAR(o.value))`, are +not copies. + +Read the fields you need from the parent, such as `o.value:sku::string`, or remove the child array +before copying the rest: + +```sql +SELECT + r.order_id, + OBJECT_DELETE(o.value, 'options') AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l +``` + +## SQBRPERFORMANCE102: a payload copied onto a finer grain + +A `VARIANT`, `OBJECT` or `ARRAY` output column must not come from an input whose unique key is a +strict subset of the model's unique key. Here `orders` has one row per `order_id` and the model has +one row per `(order_id, line_no)`, so each order's payload repeats on every line: + +```sql +SELECT + l.order_id, + l.line_no, + l.sku, + o.payload AS order_payload +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id +``` + +```text +error[SQBRPERFORMANCE102]: order_payload copies VARIANT from orders, one row per (order_id), onto rows keyed by (order_id, line_no), so the same value repeats on every row + --> models/order_lines_enriched.sql:12:3 +``` + +Read the fields the model needs, such as `o.payload:customer::string`, or leave the payload on the +input and join it where it is used. + +The Rule compares the unique keys SQLBuild knows from each relation's declaration: + +| Declaration | Key | +|---|---| +| A column with `nullable false` (or a `not_null` audit) and a `unique` audit | That column | +| An incremental model with `unique_key` and the `merge` strategy | The `unique_key` columns | +| A seed whose file holds at most one data row | No columns | + +A model with another declared key that one of the input's keys covers is not reported: that key already +proves one row per input row. An incremental model with `unique_key [order_id, order_date]` and a +unique, not-null `order_id` column has one row per order. + +Aggregated columns, such as `ARRAY_AGG(l.sku)`, are not copies and are not reported. + +### Published views + +A model whose only readers are published views of its own entity is not reported: the view +publishes the same grain on purpose. The views follow the +[SQBRGRAPH103 allowance](published-layer-rules.md#sqbrgraph103-no-published-reads-within-a-domain): +`shop__mart_v__order_lines` reading `shop__mart__order_lines`. Its `published_layers` option sets +which layers count. + +### Skipped models + +`SQBRPERFORMANCE102` needs column lineage and inferred column types. A model without them is +skipped, not reported. The run counts skipped models in a note: + +```text +note: SQBRPERFORMANCE102 skipped 2 models without column lineage or inferred types; `sqb plan --verbose` lists them +``` + +`sqb plan --verbose` and `sqb build --verbose` list each skipped model with the missing fact. + +## Details + +- `SQBRPERFORMANCE101` reports the select item that copies the parent, inside a CTE or subquery + when the copy is made there. +- `SQBRPERFORMANCE102` reports the column where the root query projects it. +- `SQBRPERFORMANCE102` reports nothing for a model without a declared unique key. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 664656b426..9f0fe7bc80 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -10195,6 +10195,7 @@ class CompilePipelineOptions(NativeRecord): external_sql_reference_resolver: ExternalSqlReferenceResolver | None resolve_python_run_selectors: bool no_cache: bool + verbose: bool max_microbatches: int | None concurrency: int | None selection_diagnostics: bool @@ -10221,6 +10222,7 @@ class CompilePipelineOptions(NativeRecord): external_sql_reference_resolver: ExternalSqlReferenceResolver | None = ..., resolve_python_run_selectors: bool = ..., no_cache: bool = ..., + verbose: bool = ..., max_microbatches: int | None = ..., concurrency: int | None = ..., selection_diagnostics: bool = ..., @@ -10960,6 +10962,7 @@ class RulesRunResult(NativeRecord): cache_misses: int skipped_type_proof_rules: tuple[str, ...] custom_rule_timings: tuple[tuple[str, int], ...] + skipped_models: tuple[tuple[str, str], ...] def __init__( self, findings: tuple[Finding, ...], @@ -10972,6 +10975,7 @@ class RulesRunResult(NativeRecord): cache_misses: int = ..., skipped_type_proof_rules: tuple[str, ...] = ..., custom_rule_timings: tuple[tuple[str, int], ...] = ..., + skipped_models: tuple[tuple[str, str], ...] = ..., ) -> None: ... class CustomRulesOutcome(NativeRecord): @@ -16115,6 +16119,7 @@ class RulesResult(NativeRecord): custom_ms: int custom_cpu_ms: int custom_rule_timings: tuple[tuple[str, int], ...] + skipped_models: tuple[tuple[str, str], ...] def __init__( self, findings: tuple[Finding, ...], @@ -16126,6 +16131,7 @@ class RulesResult(NativeRecord): custom_ms: int = ..., custom_cpu_ms: int = ..., custom_rule_timings: tuple[tuple[str, int], ...] = ..., + skipped_models: tuple[tuple[str, str], ...] = ..., ) -> None: ... class RuleFile(NativeRecord): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 87b79f8819..f560599079 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -210,6 +210,19 @@ def order_folder_siblings(*, model: Model, ctx: RuleContext) -> list[Finding]: 'select = ["XSQBR"]', ) _RULE_READ_MORE: str = "delta compile fell back: a custom rule read more than the changed models" +_PERFORMANCE_RULES_ON: tuple[str, str, str] = ( + "sqlbuild_project.toml", + 'select = ["SQBR", "SQLCLEAN", "XSQBR"]', + 'select = ["SQBR", "SQBRPERFORMANCE101", "SQBRPERFORMANCE102", "SQLCLEAN", "XSQBR"]', +) +_SNOWFLAKE_ADAPTER: tuple[str, str, str] = ( + "sqlbuild_project.toml", + 'adapter = "duckdb"', + 'adapter = "snowflake"', +) +_JSON_GRAINS_READ_ACROSS: str = ( + "delta compile fell back: SQBRPERFORMANCE102 compares keys and readers across models" +) _FAMILY_TYPE_EDIT: tuple[str, str, str] = (_CATEGORY_PIVOT_PATH, "type DOUBLE", "type VARCHAR") _FAMILIES_CHANGED: str = "delta compile fell back: a model's dynamic column families changed" _EXAMPLE_TRACKED_VARIABLE: str = "SQLBUILD_ORDERS_EXAMPLE" @@ -535,6 +548,22 @@ def test_given_project_shape_when_compiling_an_edit_then_it_prints_what_a_full_c env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, expected_stderr_line=_RULE_READ_MORE, ), + DeltaCompileVerifyTestCase( + description="the performance rules off Snowflake keep the delta", + files=(), + stored_edits=(_PERFORMANCE_RULES_ON,), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=DELTA_VERIFIED_LINE, + ), + DeltaCompileVerifyTestCase( + description="SQBRPERFORMANCE102 on Snowflake names why it fell back", + files=(), + stored_edits=(_PERFORMANCE_RULES_ON, _SNOWFLAKE_ADAPTER), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=_JSON_GRAINS_READ_ACROSS, + ), DeltaCompileVerifyTestCase( description="a new tracked variable names what changed", files=(), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index a6f4866ae2..bc10c8923f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -12,6 +12,7 @@ dense_test_path, write_dense_compile_project, ) +from scripts.cold_compile_performance.constants import DENSE_ALL_RULE_SELECT from scripts.cold_compile_performance.main.require_compile_memory_scope import ( require_compile_memory_scope, ) @@ -97,7 +98,11 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ wall_budget: float = test_case.expected_max_wall_seconds * measure_runner_calibration().factor print(f"dense cold models={test_case.model_count} wall_budget={wall_budget:.2f}s", flush=True) project_dir: Path = tmp_path / "dense_orders" - write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) + write_dense_compile_project( + project_dir=project_dir, + model_count=test_case.model_count, + rule_select=DENSE_ALL_RULE_SELECT, + ) config: RulesConfig = load_rules_config(project_dir=project_dir) catalogue: tuple[Rule, ...] = build_rule_catalogue( config=config, project_dir=project_dir, include_custom=True diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 0c995342a5..e2191ebab7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1175,6 +1175,15 @@ class SqlRuleCacheSequenceTestCase: expected_hits_and_misses: tuple[tuple[int, int], ...] +@dataclass(frozen=True) +class PerformanceRulesIntegrationTestCase: + """One compile of the DuckShop performance-rules fixture, optionally with one model added.""" + + description: str + extra_models: dict[str, str] + expected_findings: tuple[tuple[str, str, int, int], ...] + + @dataclass(frozen=True) class NamedSelectorIntegrationTestCase: """One compile of the DuckShop named-selector fixture under a `[rules]` table.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d243810b9e..fce0de2a8f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import shutil import sys import time from collections.abc import Callable @@ -36,6 +37,7 @@ ImplicitAliasRuleIntegrationTestCase, NamedSelectorIntegrationTestCase, NumericRangeDecisionIntegrationTestCase, + PerformanceRulesIntegrationTestCase, ProjectRuleScratchStateTestCase, PythonRuleCacheSequenceTestCase, PythonRulesIntegrationTestCase, @@ -218,6 +220,15 @@ def project_facts(*, project: Project, ctx: RuleContext) -> list[Finding]: ) +_PERFORMANCE_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/performance_rules") +_PERFORMANCE_EXPECTED: dict[str, Any] = json.loads( + (_PERFORMANCE_FIXTURE / "expected_findings.json").read_text(encoding="utf-8") +) +_PERFORMANCE_FINDINGS: tuple[tuple[str, str, int, int], ...] = tuple( + (code, path, line, column) for code, path, line, column in _PERFORMANCE_EXPECTED["findings"] +) + + @pytest.mark.parametrize( "test_case", [ @@ -3692,3 +3703,49 @@ def test_given_orchestration_selections_when_compiling_then_named_selector_limit == test_case.expected_findings ) assert test_case.expected_text in json.dumps(diagnostics) + + +@pytest.mark.parametrize( + "test_case", + [ + PerformanceRulesIntegrationTestCase( + description="copied payloads are reported and the rest of the fixture passes", + extra_models={}, + expected_findings=_PERFORMANCE_FINDINGS, + ), + PerformanceRulesIntegrationTestCase( + description="a second reader ends the published-view exemption", + extra_models={ + "models/order_line_counts.sql": ( + 'MODEL (description "Lines per order.");\n' + "SELECT order_id, COUNT(*) AS line_count\n" + 'FROM __ref("shop__mart__order_lines")\nGROUP BY order_id\n' + ), + }, + expected_findings=( + *_PERFORMANCE_FINDINGS, + ("SQBRPERFORMANCE102", "models/shop__mart__order_lines.sql", 11, 3), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_json_copies_when_compiling_then_performance_rules_report_them( + test_case: PerformanceRulesIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Both performance Rules run end to end; models without lineage are counted, not reported.""" + shutil.copytree(_PERFORMANCE_FIXTURE, tmp_path, dirs_exist_ok=True) + for relative_path, sql in test_case.extra_models.items(): + (tmp_path / relative_path).write_text(sql, encoding="utf-8") + + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + captured: CaptureResult[str] = capsys.readouterr() + diagnostics: list[dict[str, Any]] = json.loads(captured.out)["diagnostics"] + + assert exit_code == 1 + assert tuple( + sorted((item["code"], item["path"], item["line"], item["column"]) for item in diagnostics) + ) == tuple(sorted(test_case.expected_findings)) + assert _PERFORMANCE_EXPECTED["skipped_note"] in captured.err diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 37ad52ab87..27f042fca0 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -294,6 +294,7 @@ Compile runs in full instead when: a model-subject Rule must run again for a model that was not edited (for example because it reads an edited upstream file), or reads more for an edited model than that model's SQL, columns, contract and grain, project files and the source list; +- the project selects `SQBRPERFORMANCE102` on the Snowflake dialect; - the previous compile failed its Rules, since that compile writes no SQL tests, functions or audits to `target/compiled/`, or it reported other errors or warnings in files the edit didn't change; diff --git a/website/src/content/docs/docs/concepts/rules.mdx b/website/src/content/docs/docs/concepts/rules.mdx index c590d7e61e..5509d3733d 100644 --- a/website/src/content/docs/docs/concepts/rules.mdx +++ b/website/src/content/docs/docs/concepts/rules.mdx @@ -152,7 +152,9 @@ with `SQBR`. The `SQBRPYTHON` Rules check the project's Python: annotations, imp shape and the layout of `python/`. They run only when `select` names `SQBRPYTHON` or one of its codes; see [Python Rules](/docs/concepts/rules/python-rules). `SQBRORCHESTRATION101` keeps job and sensor selections from listing nodes one by one; see -[Orchestration Rules](/docs/concepts/rules/orchestration-rules). +[Orchestration Rules](/docs/concepts/rules/orchestration-rules). The `SQBRPERFORMANCE` Rules find +Snowflake JSON payloads that repeat on every row; they run only when `select` names their codes. See +[Performance Rules](/docs/concepts/rules/performance-rules). Projects can define custom Python Rules under `rules/**/*.py`. Their codes begin with `XSQBR`, for example `XSQBRARCH001`. Custom Rules use the same selection, findings, exceptions, execution order, @@ -248,6 +250,9 @@ Rules add about 14% to a cold compile and nothing to a warm one. See Keep each domain's published layer free of consumer-specific variants. + + Catch Snowflake JSON payloads copied onto every row of a finer grain. + Learn evaluation order, focused execution, and dependency-aware reuse. diff --git a/website/src/content/docs/docs/concepts/rules/performance-rules.mdx b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx new file mode 100644 index 0000000000..f005c838de --- /dev/null +++ b/website/src/content/docs/docs/concepts/rules/performance-rules.mdx @@ -0,0 +1,121 @@ +--- +title: "Performance Rules" +description: "Catch semi-structured payloads that repeat on every row and multiply storage and scan cost." +--- + +The `SQBRPERFORMANCE` Rules find JSON payloads copied onto rows that are finer than the payload +itself. Each copy stores and scans the same value again, so the cost grows with the row count. +Name their codes in `select` to turn them on: + +```toml +[rules] +select = ["SQBRPERFORMANCE101", "SQBRPERFORMANCE102"] +``` + +Both Rules are off by default: selecting the `SQBRPERFORMANCE` family alone does not run them. They +run on the Snowflake dialect. There, selecting `SQBRPERFORMANCE102` makes every edit compile the +whole project, because a [delta compile](/docs/cli/compile#compiling-after-an-edit) cannot recheck +the models that read or are read by the edited ones. + +| Code | Name | Reports | +|---|---|---| +| `SQBRPERFORMANCE101` | `flatten-parent-copy` | A FLATTEN element copied whole onto the rows of a deeper FLATTEN of its own child array | +| `SQBRPERFORMANCE102` | `json-finer-grain` | A `VARIANT`, `OBJECT` or `ARRAY` column copied from an input with a coarser unique key | + +## SQBRPERFORMANCE101: a flattened parent copied onto its children + +An output column must not copy the `value` of a FLATTEN whose child array a deeper FLATTEN in the +same query reads. Every child row then carries its whole parent, children included: + +```sql +SELECT + r.order_id, + o.value AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l +``` + +```text +error[SQBRPERFORMANCE101]: line_json copies o.value whole onto every row flattened from o.value:options + --> models/line_options.sql:7:3 +``` + +The parent also counts as copied through a cast, a conversion such as `TO_JSON` or `TO_VARIANT`, +or a constructor that embeds it: `OBJECT_CONSTRUCT`, `OBJECT_INSERT` or `ARRAY_CONSTRUCT`. Paths +into it, such as `o.value:sku`, and other functions of it, such as `MD5(TO_VARCHAR(o.value))`, are +not copies. + +Read the fields you need from the parent, such as `o.value:sku::string`, or remove the child array +before copying the rest: + +```sql +SELECT + r.order_id, + OBJECT_DELETE(o.value, 'options') AS line_json, + l.value:code::string AS option_code +FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l +``` + +## SQBRPERFORMANCE102: a payload copied onto a finer grain + +A `VARIANT`, `OBJECT` or `ARRAY` output column must not come from an input whose unique key is a +strict subset of the model's unique key. Here `orders` has one row per `order_id` and the model has +one row per `(order_id, line_no)`, so each order's payload repeats on every line: + +```sql +SELECT + l.order_id, + l.line_no, + l.sku, + o.payload AS order_payload +FROM __ref("order_lines") l +JOIN __ref("orders") o ON o.order_id = l.order_id +``` + +```text +error[SQBRPERFORMANCE102]: order_payload copies VARIANT from orders, one row per (order_id), onto rows keyed by (order_id, line_no), so the same value repeats on every row + --> models/order_lines_enriched.sql:12:3 +``` + +Read the fields the model needs, such as `o.payload:customer::string`, or leave the payload on the +input and join it where it is used. + +The Rule compares the unique keys SQLBuild knows from each relation's declaration: + +| Declaration | Key | +|---|---| +| A column with `nullable false` (or a `not_null` audit) and a `unique` audit | That column | +| An incremental model with `unique_key` and the `merge` strategy | The `unique_key` columns | +| A seed whose file holds at most one data row | No columns | + +A model with another declared key that one of the input's keys covers is not reported: that key already +proves one row per input row. An incremental model with `unique_key [order_id, order_date]` and a +unique, not-null `order_id` column has one row per order. + +Aggregated columns, such as `ARRAY_AGG(l.sku)`, are not copies and are not reported. + +### Published views + +A model whose only readers are published views of its own entity is not reported: the view +publishes the same grain on purpose. The views follow the +[SQBRGRAPH103 allowance](/docs/concepts/rules/published-layer-rules#sqbrgraph103-no-published-reads-within-a-domain): +`shop__mart_v__order_lines` reading `shop__mart__order_lines`. Its `published_layers` option sets +which layers count. + +### Skipped models + +`SQBRPERFORMANCE102` needs column lineage and inferred column types. A model without them is +skipped, not reported. The run counts skipped models in a note: + +```text +note: SQBRPERFORMANCE102 skipped 2 models without column lineage or inferred types; `sqb plan --verbose` lists them +``` + +`sqb plan --verbose` and `sqb build --verbose` list each skipped model with the missing fact. + +## Details + +- `SQBRPERFORMANCE101` reports the select item that copies the parent, inside a CTE or subquery + when the copy is made there. +- `SQBRPERFORMANCE102` reports the column where the root query projects it. +- `SQBRPERFORMANCE102` reports nothing for a model without a declared unique key. diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 4312a3d731..f9f4f6563e 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -77,6 +77,7 @@ "docs/concepts/rules/findings-and-exceptions", "docs/concepts/rules/python-rules", "docs/concepts/rules/orchestration-rules", + "docs/concepts/rules/performance-rules", "docs/concepts/rules/execution-and-caching" ] }