Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions coverage/syntax-databricks/reviews/ddl.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2065,3 +2065,20 @@ note = "Re-read on 18 Nov 2026 after the optional-parentheses fix, at parser tre

[review.definitions]
"db.ddl.create-recipient" = "4b313315e6c043fc5164e925ceb412cf511fd3172d08797d36374e16bd47622b"

[[review]]
id = "db-ddl-106"
family = "ddl"
parser_tree = "dafbc1c0aa3e303fd9bbf21b7a8f4d1842d3661e"
reviewed_utc = "2026-11-18T05:46:48Z"
verdict = "changes_requested"
forms = [
"db.ddl.alter-table",
]
note = "Re-read on 18 Nov 2026 at parser tree dafbc1c0 from the existing 2026.45 receipts, because the parser change that keeps tag key spellings changed two of this form's trees: kb4fe2aa33752 ALTER TABLE t1 UNSET TAGS ( 'k' ) and kcba4490330d8 ALTER TABLE t1 SET TAGS ( 'k' = 'v' ) now record that the key was written as a string (string_keys); nothing else in them changed, and no other case's tree or verdict changed. Of db-ddl-073's findings on this form, SET TBLPROPERTIES without = and REFERENCES ... ON DELETE or ON UPDATE NO ACTION now give trees of their own, and a tag key written as a name, backquoted or as a string now gives three trees. One finding still holds: SET TAGS ( 'k' 'v' ) without = gives the same tree as kcba4490330d8 although the grammar requires =, and no receipt shows what the warehouse does. Held until one exists; the text is in the pending recording bundle, and the parser is not changed on a guess."

[review.fingerprints]
"db.ddl.alter-table" = "93fd4219816042c99f96fb384a36b9f46b302cf7437e5cef6d3fe18623dfc7ce"

[review.definitions]
"db.ddl.alter-table" = "45f8d3c275e44a178729e82ee86a7ddf4b83d6b1a93ca41cd0a895c42ad47906"
17 changes: 17 additions & 0 deletions coverage/syntax-databricks/reviews/expressions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -1224,3 +1224,20 @@ note = "Re-binds db-expressions-057, not a re-read: that exclusion was bound to
[[review.exclusion]]
form = "db.expressions.object-type"
reason = "output-only type. The page (doc.data-types.object-type, page_sha256 bb2353b8cf2213122635372bfbe0177c7a19da940f0138ce38766f0368c5546a) says: \"The OBJECT cannot be stored in a table column. It is only exposed when calling schema_of_variant or schema_of_variant_agg.\" The pinned engine (2026.45) and the local parser both refuse OBJECT as input. Reconsider if the page changes or the engine starts accepting OBJECT as input."

[[review]]
id = "db-expressions-062"
family = "expressions"
catalogue_sha256 = "ffc590e2daad0e5c31541285aca1a5a7d1ce7e7bedb0c1ccdc4c1cfcdd32473c"
obligations_sha256 = "697332d6178cfd5465cf1ebae8c6d98f956a76ed507c8807032c76c97f48bc11"
parser_tree = "dafbc1c0aa3e303fd9bbf21b7a8f4d1842d3661e"
reviewed_utc = "2026-11-18T05:39:30Z"
verdict = "approved"
forms = [
"db.expressions.object-type",
]
note = "Re-binds db-expressions-061, not a re-read: that exclusion was bound to parser tree a367f5c7 and went stale when the parser kept backquoted names, tag key spellings, credential value types and every pair of parentheses (parser tree dafbc1c0), so object-type again left the excluded count. The catalogue and obligation files are byte-identical to the ones it bound (catalogue ffc590e2, obligations 697332d6), and the local parser still refuses OBJECT as input in all three cases, with the same error as before. The exclusion and its reason are repeated unchanged."

[[review.exclusion]]
form = "db.expressions.object-type"
reason = "output-only type. The page (doc.data-types.object-type, page_sha256 bb2353b8cf2213122635372bfbe0177c7a19da940f0138ce38766f0368c5546a) says: \"The OBJECT cannot be stored in a table column. It is only exposed when calling schema_of_variant or schema_of_variant_agg.\" The pinned engine (2026.45) and the local parser both refuse OBJECT as input. Reconsider if the page changes or the engine starts accepting OBJECT as input."
23 changes: 15 additions & 8 deletions crates/sqlquack-lint/src/quality/_helpers/cte_columns.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ use crate::quality::models::QualityRequest;
use crate::quality::projection_spans::projection_spans;
use crate::quality::removal_safety::{expression_refusal, reads_whole_rows, select_refusal};
use crate::quality::syntax::{
Projection, columns, cte_select_tokens, lower, projection_removal, projections, qualifier,
sources,
Projection, clauses_around, columns, cte_select_tokens, lower, projection_removal, projections,
qualifier, query_within, sources,
};
use rustc_hash::{FxHashMap, FxHashSet};

Expand Down Expand Up @@ -58,16 +58,22 @@ pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec<LintDiagnostic> {
return Vec::new();
}
let mut found: Vec<LintDiagnostic> = Vec::new();
for statement in request.statements {
for written in request.statements {
let statement: &Expression = query_within(written);
let Expression::Select(root) = statement else {
continue;
};
let Some(with) = root.with.as_ref().filter(|with| !with.recursive) else {
continue;
};
let names: Vec<String> = with.ctes.iter().map(|cte| lower(&cte.alias.name)).collect();
let used: FxHashMap<String, Usage> =
reads(&with.ctes, &names, terminal_nodes(statement, with));
let mut terminal: Vec<&Expression> = terminal_nodes(statement, with);
terminal.extend(
clauses_around(written)
.into_iter()
.flat_map(|clause| clause.dfs()),
);
let used: FxHashMap<String, Usage> = reads(&with.ctes, &names, terminal);
let select_tokens: FxHashMap<String, usize> = cte_select_tokens(request.tokens);
for (index, cte) in with
.ctes
Expand Down Expand Up @@ -152,7 +158,7 @@ fn reads<'a>(

/// The output names of a CTE, or None when a star or unnamed projection hides them.
fn output_names(cte: &Cte) -> Option<FxHashSet<String>> {
let Expression::Select(select) = &cte.this else {
let Expression::Select(select) = query_within(&cte.this) else {
return None;
};
projections(select)
Expand Down Expand Up @@ -333,7 +339,7 @@ fn clause_roots(select: &Select) -> Vec<&Expression> {
/// Lateral alias reuse: a later projection or a clause of the CTE reading an earlier output.
fn self_reads(body: &Expression) -> FxHashSet<String> {
let mut reads: FxHashSet<String> = FxHashSet::default();
let Expression::Select(select) = body else {
let Expression::Select(select) = query_within(body) else {
return reads;
};
let mut earlier: FxHashSet<String> = FxHashSet::default();
Expand All @@ -360,6 +366,7 @@ fn self_reads(body: &Expression) -> FxHashSet<String> {
if let Some(order) = &select.order_by {
clauses.extend(order.expressions.iter().map(|ordered| &ordered.this));
}
clauses.extend(clauses_around(body));
for clause in clauses {
reads.extend(
unqualified_reads(clause)
Expand All @@ -384,7 +391,7 @@ fn unused_outputs(
outputs: CteOutputs<'_>,
) -> Vec<LintDiagnostic> {
let CteOutputs { cte, name, usage } = outputs;
let Expression::Select(select) = &cte.this else {
let Expression::Select(select) = query_within(&cte.this) else {
return Vec::new();
};
let Some(usage) = usage.filter(|usage| !usage.everything && usage.read) else {
Expand Down
8 changes: 5 additions & 3 deletions crates/sqlquack-lint/src/quality/_helpers/keys.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ use sqlquack_syntax::expressions::queries::models::Select;

use crate::quality::constants::FIRST_ROW;
use crate::quality::models::QualityRequest;
use crate::quality::syntax::{Projection, columns, lower, projections, qualifier, sources};
use crate::quality::syntax::{
Projection, columns, lower, projections, qualifier, query_within, sources,
};
use crate::quality::types::{InputKey, Keys};
use crate::tree::main::is_row_number::is_row_number;

Expand Down Expand Up @@ -362,13 +364,13 @@ pub(crate) fn key_environment(request: &QualityRequest<'_>) -> KeyEnvironment {
row_numbers: HashMap::new(),
};
for statement in request.statements {
let Expression::Select(select) = statement else {
let Expression::Select(select) = query_within(statement) else {
continue;
};
for cte in select.with.iter().flat_map(|with| with.ctes.iter()) {
let keys = environment.query_keys(&cte.this);
environment.ctes.insert(lower(&cte.alias.name), keys);
if let Expression::Select(body) = &cte.this {
if let Expression::Select(body) = query_within(&cte.this) {
environment
.row_numbers
.insert(lower(&cte.alias.name), row_number_outputs(body));
Expand Down
72 changes: 64 additions & 8 deletions crates/sqlquack-lint/src/quality/_helpers/syntax.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,62 @@

use sqlquack_syntax::expressions::names::models::Column;
use sqlquack_syntax::expressions::names::models::Star;
use sqlquack_syntax::expressions::queries::models::Select;
use sqlquack_syntax::expressions::queries::models::{Select, Subquery};
use sqlquack_syntax::tokens::models::{Token, TokenType};
use sqlquack_syntax::{Expression, ExpressionWalk};

use crate::quality::constants::MIN_REMOVABLE_OUTPUTS;
use rustc_hash::FxHashMap;

/// The query within `expression`'s parentheses and unaliased subquery nodes.
pub(super) fn query_within(expression: &Expression) -> &Expression {
match expression {
Expression::Paren(paren) => query_within(&paren.this),
Expression::Subquery(subquery) if around(subquery) => query_within(&subquery.this),
other => other,
}
}

/// The clause expressions of the unaliased subquery nodes around `expression`'s query.
pub(super) fn clauses_around(expression: &Expression) -> Vec<&Expression> {
let mut clauses: Vec<&Expression> = Vec::new();
let mut node: &Expression = expression;
loop {
node = match node {
Expression::Paren(paren) => &paren.this,
Expression::Subquery(subquery) if around(subquery) => {
let ordered = subquery
.order_by
.iter()
.flat_map(|order| &order.expressions)
.chain(subquery.sort_by.iter().flat_map(|sort| &sort.expressions))
.chain(
subquery
.cluster_by
.iter()
.flat_map(|cluster| &cluster.expressions),
);
clauses.extend(ordered.map(|ordered| &ordered.this));
clauses.extend(
subquery
.distribute_by
.iter()
.flat_map(|distribute| &distribute.expressions),
);
clauses.extend(subquery.limit.iter().map(|limit| &limit.this));
clauses.extend(subquery.offset.iter().map(|offset| &offset.this));
&subquery.this
}
_ => return clauses,
};
}
}

/// Whether `subquery` only wraps a query: unaliased, without a sample or watermark.
fn around(subquery: &Subquery) -> bool {
subquery.alias.is_none() && subquery.table_sample.is_none() && subquery.watermark.is_none()
}

/// One projection of a select list.
pub(super) enum Projection<'a> {
Named {
Expand Down Expand Up @@ -83,23 +132,30 @@ pub(super) fn sources(select: &Select) -> Vec<(String, Option<String>, &Expressi
found
}

/// Token index of the SELECT opening each `name AS (SELECT` CTE body, by lower-cased name.
/// Token index of the SELECT opening each `name AS ( [( ...] SELECT` CTE body, by lower-cased name.
pub(super) fn cte_select_tokens(tokens: &[Token]) -> FxHashMap<String, usize> {
let significant: Vec<usize> = (0..tokens.len())
.filter(|&index| !is_trivia(&tokens[index]))
.collect();
let mut found: FxHashMap<String, usize> = FxHashMap::default();
for window in significant.windows(4) {
let [name_index, as_index, paren_index, select_index] = window else {
for (position, window) in significant.windows(3).enumerate() {
let [name_index, as_index, paren_index] = window else {
continue;
};
if tokens[*as_index].token_type == TokenType::As
&& tokens[*paren_index].token_type == TokenType::LParen
&& tokens[*select_index].token_type == TokenType::Select
if tokens[*as_index].token_type != TokenType::As
|| tokens[*paren_index].token_type != TokenType::LParen
{
continue;
}
let select_index: Option<&usize> = significant[position + 3..]
.iter()
.find(|&&index| tokens[index].token_type != TokenType::LParen);
if let Some(&select_index) = select_index
&& tokens[select_index].token_type == TokenType::Select
{
found
.entry(lower(&tokens[*name_index].text))
.or_insert(*select_index);
.or_insert(select_index);
}
}
found
Expand Down
116 changes: 116 additions & 0 deletions crates/sqlquack-lint/src/quality/tests/test_quality_rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -470,3 +470,119 @@ fn given_threshold_findings_when_linting_then_remediation_shows_the_exact_settin
}
Ok(())
}

#[test]
fn given_a_query_or_cte_body_in_parentheses_when_linting_then_its_ctes_are_checked_as_without()
-> Result<(), String> {
let test_cases = [
QualityRuleTestCase {
description: "an unused CTE output, the query bare",
sql: "WITH orders AS (SELECT o.order_id, o.status, o.note FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f",
rule: "SQLCLEAN042",
relation_keys: "{}",
expected_anchors: &["o.note"],
expected_fixed_sql: Some(
"WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f",
),
},
QualityRuleTestCase {
description: "an unused CTE output, the whole query in parentheses",
sql: "(WITH orders AS (SELECT o.order_id, o.status, o.note FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f)",
rule: "SQLCLEAN042",
relation_keys: "{}",
expected_anchors: &["o.note"],
expected_fixed_sql: Some(
"(WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f)",
),
},
QualityRuleTestCase {
description: "an unused CTE output, its CTE body in a second pair",
sql: "WITH orders AS ((SELECT o.order_id, o.status, o.note FROM raw_orders AS o)), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f",
rule: "SQLCLEAN042",
relation_keys: "{}",
expected_anchors: &["o.note"],
expected_fixed_sql: Some(
"WITH orders AS ((SELECT o.order_id, o.status FROM raw_orders AS o)), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f",
),
},
QualityRuleTestCase {
description: "an unused CTE output, WITH before a parenthesized query",
sql: "WITH orders AS (SELECT o.order_id, o.status, o.note FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) (SELECT f.order_id, f.status FROM final AS f)",
rule: "SQLCLEAN042",
relation_keys: "{}",
expected_anchors: &["o.note"],
expected_fixed_sql: Some(
"WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) (SELECT f.order_id, f.status FROM final AS f)",
),
},
QualityRuleTestCase {
description: "an unused CTE output, WITH before a parenthesized query with LIMIT inside and ORDER BY after",
sql: "WITH orders AS (SELECT o.order_id, o.status, o.note FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) (SELECT f.order_id, f.status FROM final AS f LIMIT 5) ORDER BY 1",
rule: "SQLCLEAN042",
relation_keys: "{}",
expected_anchors: &["o.note"],
expected_fixed_sql: Some(
"WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) (SELECT f.order_id, f.status FROM final AS f LIMIT 5) ORDER BY 1",
),
},
QualityRuleTestCase {
description: "a CTE column read only by an ORDER BY after the parentheses stays",
sql: "WITH orders AS (SELECT o.order_id, o.status, o.note FROM raw_orders AS o), other AS (SELECT 1 AS one) (SELECT r.order_id FROM orders AS r LIMIT 5) ORDER BY r.note",
rule: "SQLCLEAN042",
relation_keys: "{}",
expected_anchors: &["o.status"],
expected_fixed_sql: Some(
"WITH orders AS (SELECT o.order_id, o.note FROM raw_orders AS o), other AS (SELECT 1 AS one) (SELECT r.order_id FROM orders AS r LIMIT 5) ORDER BY r.note",
),
},
QualityRuleTestCase {
description: "GROUP BY grain is a key, the query bare",
sql: "WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY o.customer_id), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f",
rule: "SQLCLEAN018",
relation_keys: r#"{}"#,
expected_anchors: &[],
expected_fixed_sql: None,
},
QualityRuleTestCase {
description: "GROUP BY grain is a key, the whole query in parentheses",
sql: "(WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY o.customer_id), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f)",
rule: "SQLCLEAN018",
relation_keys: r#"{}"#,
expected_anchors: &[],
expected_fixed_sql: None,
},
QualityRuleTestCase {
description: "a later rn = 1 filter makes the partition a key, the query bare",
sql: "WITH ranked AS (SELECT o.order_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) AS rn FROM orders AS o), latest AS (SELECT r.customer_id, r.order_id FROM ranked AS r WHERE r.rn = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f",
rule: "SQLCLEAN018",
relation_keys: r#"{"orders": [["order_id"]]}"#,
expected_anchors: &[],
expected_fixed_sql: None,
},
QualityRuleTestCase {
description: "a later rn = 1 filter makes the partition a key, the ranking CTE body in a second pair",
sql: "WITH ranked AS ((SELECT o.order_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) AS rn FROM orders AS o)), latest AS (SELECT r.customer_id, r.order_id FROM ranked AS r WHERE r.rn = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f",
rule: "SQLCLEAN018",
relation_keys: r#"{"orders": [["order_id"]]}"#,
expected_anchors: &[],
expected_fixed_sql: None,
},
];
for test_case in &test_cases {
let diagnostics: Vec<Value> =
lint_with_host_fields(test_case, r#"{"dialect": "databricks"}"#)?;
assert_eq!(
anchors(test_case.sql, &diagnostics),
test_case.expected_anchors,
"{}",
test_case.description
);
assert_eq!(
fixed(test_case.sql, &diagnostics).as_deref(),
test_case.expected_fixed_sql,
"{}",
test_case.description
);
}
Ok(())
}
Loading
Loading