Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
4a5b46c
wip: catalogue entries for the performance rules, off by default
kvlonge Nov 16, 2026
b8f6c86
wip: SQBRPERFORMANCE101 flags a FLATTEN parent element copied whole o…
kvlonge Nov 16, 2026
e064a5d
wip: SQBRPERFORMANCE102 rule and grain facts on rules models
kvlonge Nov 16, 2026
5317217
wip: build SQBRPERFORMANCE102 grains from lineage, keys and readers
kvlonge Nov 16, 2026
007ffd2
wip: SQBRPERFORMANCE102 skipped-model note, DuckShop fixture, docs an…
kvlonge Nov 16, 2026
d3efc80
wip: narrow SQBRPERFORMANCE101 copies and honour covering keys in SQB…
kvlonge Nov 16, 2026
d6b2d47
wip: count conversions of a FLATTEN parent as whole copies
kvlonge Nov 16, 2026
59a28cf
wip: exempt an input when any of its keys covers a model key; count T…
kvlonge Nov 16, 2026
620b600
Merge remote-tracking branch 'origin/main' into feat/performance-rules
kvlonge Nov 16, 2026
8fa82c0
refactor: group rules helpers and split grain builders for the layout…
kvlonge Nov 16, 2026
8b70a77
Merge remote-tracking branch 'origin/main' into feat/performance-rules
kvlonge Nov 16, 2026
0a8b453
chore: raise the bindings crate cap for the JSON grain rule plumbing
kvlonge Nov 16, 2026
d1ecf84
docs: regenerate the docs skill for performance rules
kvlonge Nov 16, 2026
c1243a0
test: sort the performance rules test case import
kvlonge Nov 16, 2026
cd1f914
fix: report a JSON grain finding at the first positioned output column
kvlonge Nov 16, 2026
9c89ca8
Merge remote-tracking branch 'origin/main' into feat/performance-rules
kvlonge Nov 16, 2026
75da4b7
Merge remote-tracking branch 'origin/main' into feat/performance-rules
kvlonge Nov 16, 2026
ffb1e31
test: select the opt-in performance rules in the dense compile benchmark
kvlonge Nov 16, 2026
fd90a00
test: keep the opt-in performance rules out of the dense edit benchmarks
kvlonge Nov 16, 2026
2b13222
docs: say that SQBRPERFORMANCE102 turns off delta compiles on Snowflake
kvlonge Nov 16, 2026
cd02645
test: describe the dense rule selection without a standalone comment
kvlonge Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"findings": [
["SQBRPERFORMANCE101", "models/line_options.sql", 7, 3],
["SQBRPERFORMANCE102", "models/order_lines_enriched.sql", 12, 3]
],
"skipped_note": "SQBRPERFORMANCE102 skipped 1 model without column lineage or inferred types"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
MODEL (
description "One row per line option with the whole parent line copied onto it.",
);

SELECT
r.order_id,
o.value AS line_json,
l.value:code::string AS option_code
FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
MODEL (
description "One row per line option with the parent line minus its options, and a hash of the whole line.",
);

SELECT
r.order_id,
OBJECT_DELETE(o.value, 'options') AS line_json,
MD5(TO_VARCHAR(o.value)) AS line_hash,
l.value:code::string AS option_code
FROM __ref("orders") r, LATERAL FLATTEN(input => r.payload:lines) o, LATERAL FLATTEN(o.value:options) l
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
MODEL (
description "Orders read from an export table SQLBuild cannot see.",
columns (
order_id (type NUMBER, nullable false, audits [unique]),
),
);

SELECT *
FROM RAW.SHOP.ORDER_EXPORTS
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
MODEL (
description "One row per order, unique by order_id and by order_number.",
columns (
order_id (type NUMBER, nullable false, audits [unique]),
order_number (type VARCHAR, nullable false, audits [unique]),
),
);

SELECT
1 AS order_id,
'SO-1' AS order_number,
PARSE_JSON('{"customer": "c1"}') AS header
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
MODEL (
description "One row per order: keyed by order number and date for pruning, but order_id alone is unique.",
materialized incremental,
incremental_strategy merge,
unique_key [order_number, order_date],
columns (
order_id (type NUMBER, nullable false, audits [unique]),
),
);

SELECT
h.order_id,
h.order_number,
CURRENT_DATE() AS order_date,
h.header AS order_header
FROM __ref("order_headers") h
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
MODEL (
description "One row per order line.",
materialized incremental,
incremental_strategy merge,
unique_key [order_id, line_no],
);

SELECT
o.order_id,
l.index AS line_no,
l.value:sku::string AS sku
FROM __ref("orders") o, LATERAL FLATTEN(input => o.payload:lines) l
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
MODEL (
description "Order lines with one field read from the order payload.",
materialized incremental,
incremental_strategy merge,
unique_key [order_id, line_no],
);

SELECT
l.order_id,
l.line_no,
o.payload:customer::string AS customer_id
FROM __ref("order_lines") l
JOIN __ref("orders") o ON o.order_id = l.order_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
MODEL (
description "Order lines with the whole order payload copied onto every line.",
materialized incremental,
incremental_strategy merge,
unique_key [order_id, line_no],
);

SELECT
l.order_id,
l.line_no,
l.sku,
o.payload AS order_payload
FROM __ref("order_lines") l
JOIN __ref("orders") o ON o.order_id = l.order_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
MODEL (
description "One row per order: the merge key adds a date for pruning, but order_id alone is unique.",
materialized incremental,
incremental_strategy merge,
unique_key [order_id, order_date],
columns (
order_id (type NUMBER, nullable false, audits [unique]),
),
);

SELECT
o.order_id,
CURRENT_DATE() AS order_date,
o.payload AS order_payload
FROM __ref("orders") o
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
MODEL (
description "One row per order with its line SKUs aggregated.",
columns (
order_id (type NUMBER, nullable false, audits [unique]),
),
);

SELECT
l.order_id,
ARRAY_AGG(l.sku) AS skus
FROM __ref("order_lines") l
GROUP BY l.order_id
10 changes: 10 additions & 0 deletions benchmarks/duckshop/edge_cases/performance_rules/models/orders.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
MODEL (
description "One row per order with its raw payload.",
columns (
order_id (type NUMBER, nullable false, audits [unique]),
),
);

SELECT
1 AS order_id,
PARSE_JSON('{"customer": "c1", "lines": [{"sku": "a", "options": [{"code": "gift"}]}]}') AS payload
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
MODEL (
description "Published order lines whose only reader is their own published view.",
materialized incremental,
incremental_strategy merge,
unique_key [order_id, line_no],
);

SELECT
l.order_id,
l.line_no,
o.payload AS order_payload
FROM __ref("order_lines") l
JOIN __ref("orders") o ON o.order_id = l.order_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
MODEL (
description "The published view of order lines.",
materialized view,
);

SELECT order_id, line_no, order_payload
FROM __ref("shop__mart__order_lines")
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
name = "orders"
adapter = "snowflake"

[connection]
database = "ANALYTICS"
schema = "ORDERS"

[rules]
select = ["SQBRPERFORMANCE101", "SQBRPERFORMANCE102"]
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,13 @@ pub struct LineageRow {
pub expression_range: Option<(usize, usize)>,
}

impl LineageRow {
/// Whether the column aggregates its sources rather than carrying one source row's value.
pub fn aggregated(&self) -> bool {
self.transform_code == crate::assembly::analysis_session::constants::TRANSFORM_AGGREGATION
}
}

/// Where a model's lineage facts live.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LineageFacts {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,7 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec {
FieldSpec::new("external_sql_reference_resolver", FieldDefault::None),
FieldSpec::new("resolve_python_run_selectors", FieldDefault::False),
FieldSpec::new("no_cache", FieldDefault::False),
FieldSpec::new("verbose", FieldDefault::False),
FieldSpec::new("max_microbatches", FieldDefault::None),
FieldSpec::new("concurrency", FieldDefault::None),
FieldSpec::new("selection_diagnostics", FieldDefault::False),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ const R_RULES_RUN_RESULT: RecordSpec = RecordSpec {
FieldSpec::new("cache_misses", FieldDefault::Int(0)),
FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple),
FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple),
FieldSpec::new("skipped_models", FieldDefault::EmptyTuple),
],
};
const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec {
Expand All @@ -121,6 +122,7 @@ const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec {
FieldSpec::required("cache_misses"),
FieldSpec::required("built_in_ms"),
FieldSpec::required("reused"),
FieldSpec::new("skipped_models", FieldDefault::EmptyTuple),
],
};
const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec {
Expand Down Expand Up @@ -298,6 +300,7 @@ const R_RULES_RESULT: RecordSpec = RecordSpec {
FieldSpec::new("custom_ms", FieldDefault::Int(0)),
FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)),
FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple),
FieldSpec::new("skipped_models", FieldDefault::EmptyTuple),
],
};
const R_RULE_FILE: RecordSpec = RecordSpec {
Expand Down
1 change: 1 addition & 0 deletions crates/sqlbuild-python-base/src/records/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1080,6 +1080,7 @@ const COMPILE_ANALYSIS: RecordSpec = RecordSpec {
FieldSpec::new("rule_cache_misses", FieldDefault::Int(0)),
FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple),
FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple),
FieldSpec::new("skipped_models", FieldDefault::EmptyTuple),
],
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult<Bound<'py, PyAn
&[
("no_sql_validation", field("no_sql_validation")?),
("no_cache", field("no_cache")?),
("verbose", field("verbose")?),
("selected_target", field("selected_target")?),
("defer_to", field("defer_to")?),
("defer_sources_to", field("defer_sources_to")?),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
use std::collections::HashMap;

use pyo3::PyResult;
use sqlbuild_rules::engine::main::needs_json_grains::needs_json_grains;
use sqlbuild_rules::models::{EvaluateRequest, Model, SqlTestFact};

use sqlbuild_python_base::compile_context::main::delta::delta_unsupported::delta_unsupported;
Expand All @@ -15,6 +16,11 @@ pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult<Evalua
let Some(injection) = injection() else {
return Ok(request);
};
if needs_json_grains(&request).unwrap_or(true) {
return Err(delta_unsupported(
"SQBRPERFORMANCE102 compares keys and readers across models",
));
}
let taken: Option<BaseRules> = injection
.rules
.lock()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,10 @@ fn analyzed<'py>(
"custom_rule_timings",
native_field(&rules_result, "custom_rule_timings")?,
),
(
"skipped_models",
native_field(&rules_result, "skipped_models")?,
),
],
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ use sqlbuild_python_base::records::main::registry::native_attr::native_attr;
use sqlbuild_python_base::records::types::NativeAttr;
use sqlbuild_python_base::records::types::RecordId;
use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice;
use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models;

const BYPASS_OUTCOME: &str = "bypass";

Expand Down Expand Up @@ -424,9 +425,16 @@ fn compiled_with_status<'py>(
}
}
close_status(status)?;
if let Some(skipped_note) = format_skipped_type_proof_rules(
native_field(&analysis, "skipped_type_proof_rules")?.extract()?,
) {
let skipped_notes = [
format_skipped_type_proof_rules(
native_field(&analysis, "skipped_type_proof_rules")?.extract()?,
),
format_skipped_models(
&native_field(&analysis, "skipped_models")?.extract::<Vec<_>>()?,
false,
),
];
for skipped_note in skipped_notes.into_iter().flatten() {
output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?;
}
let dag_artifact_path: Bound<'py, PyAny> =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction};
use sqlbuild_host::entries::models::ExecutorHost;
use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit;
use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice;
use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models;

use crate::bindings::_helpers::compile_driver::phases::monotonic;
use crate::bindings::_helpers::compile_driver::pipeline::{
Expand Down Expand Up @@ -137,7 +138,13 @@ fn run_rules_gate<'py>(
if !on_progress.is_none() {
let elapsed: f64 = monotonic(py)? - started;
let message: String = rules_progress(elapsed, &result, true)?;
on_progress.call1((message,))?;
let note: Option<String> = format_skipped_models(
&native_field(&result, "skipped_models")?.extract::<Vec<_>>()?,
native_field(options, "verbose")?.is_truthy()?,
);
for line in std::iter::once(message).chain(note.map(|note| format!("note: {note}"))) {
on_progress.call1((line,))?;
}
}
let mut failing: Vec<Bound<'py, PyAny>> = Vec::new();
for finding in native_field(&result, "findings")?.try_iter()? {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,10 @@ const DEFAULT_SECTION_ENTRIES: i64 = 50;
const ERROR_SEVERITY: &str = "error";
const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"];
/// Request fields passed to the compile pipeline under the same name.
const PIPELINE_FIELDS: [&str; 11] = [
const PIPELINE_FIELDS: [&str; 12] = [
"no_sql_validation",
"no_cache",
"verbose",
"defer_to",
"defer_sources_to",
"cursor_overrides",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record;
use sqlbuild_python_base::records::main::fields::field::field as native_field;
use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords;
use sqlbuild_python_base::records::types::RecordId;
use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models;

const LIST_ACTION: &str = "list";
const SHOW_ACTION: &str = "show";
Expand Down Expand Up @@ -238,6 +239,12 @@ fn run_rule_selection(
) {
print_line(py, &format!("note: {note}"), true)?;
}
if let Some(note) = format_skipped_models(
&native_field(&result, "skipped_models")?.extract::<Vec<_>>()?,
false,
) {
print_line(py, &format!("note: {note}"), true)?;
}
let fields: Bound<'_, PyDict> = PyDict::new(py);
for field in RESULT_FIELDS {
fields.set_item(field, native_field(&result, field)?)?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ pub(crate) const CUSTOM_HOST_ABANDONED_SUFFIX: &str = ".abandoned";
/// Marker file whose presence tells split custom hosts the run was cancelled.
pub(crate) const CUSTOM_HOST_CANCELLED_MARKER: &str = "cancelled";
/// Rules that need proven column types and are skipped without SQL analysis.
pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 1] = ["SQBRCONTRACT105"];
pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 2] = ["SQBRCONTRACT105", "SQBRPERFORMANCE102"];
/// Runtime identity custom-rule hosts and their cache entries record.
pub(crate) const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v3";

Expand Down
Loading
Loading