Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions coverage/syntax-databricks/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,8 @@ others:
- `ADD COLUMN` against `ADD COLUMNS` and `DROP COLUMN` against `DROP COLUMNS`, because the ALTER
TABLE page writes `{ COLUMN | COLUMNS }` as alternatives of one clause;
- the sign written before an interval value, including `+`;
- a raw string (`r'c'` or `R'c'`) against a plain one, and adjacent strings (`'c' 'c'`) against
the one string they join to, in a `joined_string` literal that keeps each piece;
- the operator written for a construct another documented form shares: `==` against `=`, `!=`
against `<>`, `IS [NOT] DISTINCT FROM` against `<=>`, `REGEXP` and `RLIKE` against the `rlike` call, and `>>>`
against the `shiftrightunsigned` call, in a `written_as` field that analysis ignores;
Expand Down
51 changes: 51 additions & 0 deletions coverage/syntax-databricks/reviews/expressions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -1241,3 +1241,54 @@ note = "Re-binds db-expressions-061, not a re-read: that exclusion was bound to
[[review.exclusion]]
form = "db.expressions.object-type"
reason = "output-only type. The page (doc.data-types.object-type, page_sha256 bb2353b8cf2213122635372bfbe0177c7a19da940f0138ce38766f0368c5546a) says: \"The OBJECT cannot be stored in a table column. It is only exposed when calling schema_of_variant or schema_of_variant_agg.\" The pinned engine (2026.45) and the local parser both refuse OBJECT as input. Reconsider if the page changes or the engine starts accepting OBJECT as input."

[[review]]
id = "db-expressions-063"
family = "expressions"
catalogue_sha256 = "ffc590e2daad0e5c31541285aca1a5a7d1ce7e7bedb0c1ccdc4c1cfcdd32473c"
obligations_sha256 = "697332d6178cfd5465cf1ebae8c6d98f956a76ed507c8807032c76c97f48bc11"
parser_tree = "e41321e692d3aa300122b5affc29d38a151bed13"
reviewed_utc = "2026-11-18T10:11:35Z"
verdict = "approved"
forms = [
"db.expressions.object-type",
]
note = "Re-binds db-expressions-062, not a re-read: that exclusion was bound to parser tree dafbc1c0 and went stale when the parser kept raw and adjacent string spellings, the parentheses of an INSERT query and a WINDOW written after a parenthesized query (parser tree e41321e6), so object-type again left the excluded count. The catalogue and obligation files are byte-identical to the ones it bound (catalogue ffc590e2, obligations 697332d6), and the local parser still refuses OBJECT as input in all three cases, with the same error as before (unsupported data type OBJECT, position 22). The exclusion and its reason are repeated unchanged."

[[review.exclusion]]
form = "db.expressions.object-type"
reason = "output-only type. The page (doc.data-types.object-type, page_sha256 bb2353b8cf2213122635372bfbe0177c7a19da940f0138ce38766f0368c5546a) says: \"The OBJECT cannot be stored in a table column. It is only exposed when calling schema_of_variant or schema_of_variant_agg.\" The pinned engine (2026.45) and the local parser both refuse OBJECT as input. Reconsider if the page changes or the engine starts accepting OBJECT as input."

[[review]]
id = "db-expressions-064"
family = "expressions"
parser_tree = "e41321e692d3aa300122b5affc29d38a151bed13"
reviewed_utc = "2026-11-18T10:11:35Z"
verdict = "approved"
forms = [
"db.expressions.string-type.subscript-dot",
]
note = "Re-read on 18 Nov 2026 after the adjacent-strings fix, at parser tree e41321e6, from the existing 2026.45 receipts; 4 of 4 required obligations are met, and the warehouse accepts all four cases. The gated tree screen (drop, synonym swap, backquote, string, double quote, requote, unquote, insert AS, insert = and paren pair) finds no same-tree variant of any accepted case in 128 variants; the 3 requote variants it allows rewrite one string literal in the other quote style, which the lexer reads as the same token. string-type.subscript-dot: db-expressions-022's (d) finding, held again in db-expressions-040, is answered: SELECT ( 'c' 'c' ) (kcba339110891, o010) is now a joined_string literal that keeps the value cc and both pieces, so it no longer shares the tree of SELECT ( 'cc' ), and a dropped piece gives a plain string. db-expressions-022's (a) finding stays answered by the [[reading]] row. r'c' and R'c' (o008, o009) are raw_string literals apart from 'c' (o007); the two prefixes give one tree, as a keyword's letter case does, because the lexer reads the prefix case-insensitively."

[review.fingerprints]
"db.expressions.string-type.subscript-dot" = "8e1d8018e9410c999219c14a399ccf33865df765423a52e95a27b90d1da18a23"

[review.definitions]
"db.expressions.string-type.subscript-dot" = "6e0e70498e32940a8ecb5d77d0d34cfc3c73896223f0d392bac3582b9b4442b9"

[[review]]
id = "db-expressions-065"
family = "expressions"
catalogue_sha256 = "ffc590e2daad0e5c31541285aca1a5a7d1ce7e7bedb0c1ccdc4c1cfcdd32473c"
obligations_sha256 = "697332d6178cfd5465cf1ebae8c6d98f956a76ed507c8807032c76c97f48bc11"
parser_tree = "243ba4a29f6b3b1da6889bd32bb4efedd9b1c712"
reviewed_utc = "2026-11-18T13:52:45Z"
verdict = "approved"
forms = [
"db.expressions.object-type",
]
note = "Re-binds db-expressions-063, not a re-read: that exclusion was bound to parser tree e41321e6 and went stale when the parser kept a WINDOW written after a parenthesized query on the node after the parentheses, even when the query inside could take it (parser tree 243ba4a2), so object-type again left the excluded count. The catalogue and obligation files are byte-identical to the ones it bound (catalogue ffc590e2, obligations 697332d6), and the local parser still refuses OBJECT as input in all three cases, with the same error as before (unsupported data type OBJECT, position 22). The exclusion and its reason are repeated unchanged."

[[review.exclusion]]
form = "db.expressions.object-type"
reason = "output-only type. The page (doc.data-types.object-type, page_sha256 bb2353b8cf2213122635372bfbe0177c7a19da940f0138ce38766f0368c5546a) says: \"The OBJECT cannot be stored in a table column. It is only exposed when calling schema_of_variant or schema_of_variant_agg.\" The pinned engine (2026.45) and the local parser both refuse OBJECT as input. Reconsider if the page changes or the engine starts accepting OBJECT as input."
5 changes: 5 additions & 0 deletions coverage/syntax-databricks/screen.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@
# covers one token written in another spelling that the lexer reads as the same token, in every
# form; only `requote` ('x' and "x", one string literal) qualifies.
#
# Letter case is allowed without an entry, because the screen makes no case variants: the lexer
# reads a keyword and the r or R prefix of a raw string case-insensitively, so SELECT and select,
# and r'x' and R'x', are one token each and give one tree. A receipt that shows Databricks reading
# two such spellings differently makes it a finding.
#
# [[lexical]]
# mutation = "requote"
# reason = "<why the lexer reads both spellings as one token>"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ const MAX_DECIMAL_PRECISION: usize = 38;
pub(crate) fn databricks_literal_type(literal: &Literal) -> Option<String> {
match literal {
Literal::Number(text) => number_type(text, false),
Literal::String(_) => Some(STRING_TYPE.to_owned()),
Literal::String(_) | Literal::RawString(_) | Literal::JoinedString(..) => {
Some(STRING_TYPE.to_owned())
}
_ => None,
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ fn given_a_databricks_query_when_naming_and_typing_outputs_then_they_are_the_war
DatabricksOutputsTestCase {
description: "literals typed as typeof reports them",
sql: "SELECT 1 AS a, 2147483648 AS b, 9223372036854775808 AS c, 1.50 AS d, \
0.05 AS e, 1e5 AS f, 'x' AS g, -2147483648 AS h",
0.05 AS e, 1e5 AS f, 'x' AS g, -2147483648 AS h, r'x' AS i",
expected_outputs: &[
("a", Some("INT")),
("b", Some("BIGINT")),
Expand All @@ -146,6 +146,7 @@ fn given_a_databricks_query_when_naming_and_typing_outputs_then_they_are_the_war
("f", Some("DOUBLE")),
("g", Some("STRING")),
("h", Some("INT")),
("i", Some("STRING")),
],
},
DatabricksOutputsTestCase {
Expand Down Expand Up @@ -388,6 +389,66 @@ fn given_a_relation_or_query_in_extra_parentheses_when_analysed_then_it_reads_as
sql: "( SELECT id FROM orders ) UNION ( SELECT id FROM customers )",
expected_same_as: "SELECT id FROM orders UNION SELECT id FROM customers",
},
SameReadingTestCase {
description: "an insert's query in parentheses",
sql: "INSERT INTO orders ( SELECT id, nope FROM customers )",
expected_same_as: "INSERT INTO orders SELECT id, nope FROM customers",
},
];
for test_case in test_cases {
assert_eq!(
reading(test_case.sql),
reading(test_case.expected_same_as),
"{}",
test_case.description
);
}
}

#[test]
fn given_a_join_in_parentheses_when_analysed_then_its_conditions_and_items_are_checked_as_without()
{
let test_cases = [
SameReadingTestCase {
description: "an unknown column in a parenthesised join's ON",
sql: "SELECT o.id FROM ( orders o JOIN customers c ON o.nope = c.id )",
expected_same_as: "SELECT o.id FROM orders o JOIN customers c ON o.nope = c.id",
},
SameReadingTestCase {
description: "the same join in a second pair of parentheses",
sql: "SELECT o.id FROM ( ( orders o JOIN customers c ON o.nope = c.id ) )",
expected_same_as: "SELECT o.id FROM orders o JOIN customers c ON o.nope = c.id",
},
SameReadingTestCase {
description: "a USING key one side lacks",
sql: "SELECT o.id FROM ( orders o JOIN customers c USING ( nope ) )",
expected_same_as: "SELECT o.id FROM orders o JOIN customers c USING ( nope )",
},
SameReadingTestCase {
description: "a derived table inside the parenthesised join",
sql: "SELECT o.id FROM ( orders o JOIN ( SELECT nope FROM customers ) d ON o.id = d.nope )",
expected_same_as: "SELECT o.id FROM orders o JOIN ( SELECT nope FROM customers ) d ON o.id = d.nope",
},
SameReadingTestCase {
description: "a parenthesised join joined to an earlier table",
sql: "SELECT o.id FROM customers c2 JOIN ( orders o JOIN customers c ON o.nope = c.id ) ON c2.id = o.id",
expected_same_as: "SELECT o.id FROM customers c2 JOIN orders o ON c2.id = o.id JOIN customers c ON o.nope = c.id",
},
SameReadingTestCase {
description: "a LATERAL item reading the table before it inside the parentheses",
sql: "SELECT l.n FROM ( orders o JOIN LATERAL ( SELECT o.id AS n ) l ON true )",
expected_same_as: "SELECT l.n FROM orders o JOIN LATERAL ( SELECT o.id AS n ) l ON true",
},
SameReadingTestCase {
description: "USING inside the parentheses, its left side only the tables there",
sql: "SELECT 1 FROM orders JOIN ( customers JOIN orders y USING ( quantity ) ) ON true",
expected_same_as: "SELECT 1 FROM customers JOIN orders y USING ( quantity )",
},
SameReadingTestCase {
description: "USING after the parentheses, its right side every table there",
sql: "SELECT 1 FROM orders JOIN ( customers JOIN orders y ON true ) USING ( quantity )",
expected_same_as: "SELECT 1 FROM orders JOIN orders y USING ( quantity )",
},
];
for test_case in test_cases {
assert_eq!(
Expand All @@ -402,6 +463,11 @@ fn given_a_relation_or_query_in_extra_parentheses_when_analysed_then_it_reads_as
#[test]
fn given_a_parenthesized_query_with_clauses_after_it_when_analysed_then_they_read_its_sources() {
let test_cases = [
SameReadingTestCase {
description: "a named window written after the parentheses",
sql: "( SELECT sum ( quantity ) OVER w AS x FROM orders LIMIT 1 ) WINDOW w AS ( ORDER BY quantity )",
expected_same_as: "SELECT sum ( quantity ) OVER w AS x FROM orders WINDOW w AS ( ORDER BY quantity ) LIMIT 1",
},
SameReadingTestCase {
description: "ORDER BY a column the query does not output",
sql: "( SELECT id FROM orders ) ORDER BY quantity",
Expand Down Expand Up @@ -482,3 +548,51 @@ fn given_a_parenthesized_query_with_clauses_after_it_when_analysed_then_they_rea
);
}
}

#[test]
fn given_a_window_named_after_parentheses_when_checked_then_only_an_undefined_name_is_found() {
let test_cases = [
QueryFindingsTestCase {
description: "the window function inside reads the window named after the parentheses",
sql: "( SELECT sum ( quantity ) OVER w AS x FROM orders LIMIT 1 ) WINDOW w AS ( ORDER BY quantity )",
expected_findings: &[],
},
QueryFindingsTestCase {
description: "a window no WINDOW clause names is still unknown",
sql: "( SELECT sum ( quantity ) OVER w2 AS x FROM orders LIMIT 1 ) WINDOW w AS ( ORDER BY quantity )",
expected_findings: &[("E232", "Unknown named window 'w2'", 15)],
},
];
for test_case in test_cases {
assert_eq!(
databricks_findings(test_case.sql),
owned_findings(test_case.expected_findings),
"{}",
test_case.description
);
}
}

#[test]
fn given_adjacent_strings_when_analysed_then_they_read_as_the_string_they_join_to() {
let test_cases = [
SameReadingTestCase {
description: "an output's type",
sql: "SELECT 'a' 'b' AS x",
expected_same_as: "SELECT 'ab' AS x",
},
SameReadingTestCase {
description: "a number column compared with a string that is not a number",
sql: "SELECT id FROM orders WHERE quantity = 'x' 'y'",
expected_same_as: "SELECT id FROM orders WHERE quantity = 'xy'",
},
];
for test_case in test_cases {
assert_eq!(
reading(test_case.sql),
reading(test_case.expected_same_as),
"{}",
test_case.description
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ pub(crate) fn predicate(engine: Engine, family: Family) -> bool {
fn string_literal_text(expression: &Expression) -> Option<&str> {
match expression {
Expression::Literal(literal) => match literal.as_ref() {
Literal::String(value) => Some(value),
Literal::String(value) | Literal::JoinedString(value, _) => Some(value),
_ => None,
},
Expression::Alias(alias) => string_literal_text(&alias.this),
Expand Down Expand Up @@ -372,10 +372,12 @@ pub(crate) fn invalid_literal_for(engine: Engine, expression: &Expression, targe
Expression::Alias(alias) => invalid_literal_for(engine, &alias.this, target),
Expression::Paren(paren) => invalid_literal_for(engine, &paren.this, target),
Expression::Literal(literal) => match literal.as_ref() {
Literal::String(value) if engine == Engine::Postgres => {
Literal::String(value) | Literal::JoinedString(value, _)
if engine == Engine::Postgres =>
{
invalid_postgres_literal(value, target)
}
Literal::String(value) => match target {
Literal::String(value) | Literal::JoinedString(value, _) => match target {
Family::Integer | Family::Numeric => value.trim().parse::<f64>().is_err(),
Family::Date | Family::Timestamp => invalid_temporal_literal(value),
Family::Boolean => !matches!(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ use crate::lineage::_helpers::checks::signatures::{
argument_finding, check_call, dispatch_name, has_signature, located, unify,
};
use crate::lineage::_helpers::tracing::expressions::expression_lineage;
use crate::lineage::_helpers::tracing::sources::every_join;
use crate::lineage::constants::{
DUCKDB_KEYWORD_ONLY_FUNCTIONS, NAME_PHASE, RUNTIME_WARNING, TYPE_PHASE,
};
Expand Down Expand Up @@ -214,7 +215,7 @@ fn clause_predicates(
.iter()
.map(|clause| ("QUALIFY", &clause.this)),
);
for join in &select.joins {
for join in every_join(select) {
clauses.extend(join.on.iter().map(|on| ("JOIN ON", on)));
clauses.extend(
join.match_condition
Expand Down Expand Up @@ -338,7 +339,7 @@ fn expression_checks(node: &Expression, level: NodeLevel<'_, '_>) -> Vec<NameFin
}
Expression::Interval(interval) => {
if let Some(Expression::Literal(literal)) = &interval.this
&& let Literal::String(value) = literal.as_ref()
&& let Literal::String(value) | Literal::JoinedString(value, _) = literal.as_ref()
&& !value.chars().any(|character| character.is_ascii_digit())
{
let code: Option<&'static str> = match engine {
Expand Down Expand Up @@ -653,7 +654,11 @@ fn scalar_subqueries(node: &Expression, level: NodeLevel<'_, '_>) -> Vec<NameFin
children.extend(select.where_clause.iter().map(|clause| &clause.this));
children.extend(select.having.iter().map(|clause| &clause.this));
children.extend(select.qualify.iter().map(|clause| &clause.this));
children.extend(select.joins.iter().filter_map(|join| join.on.as_ref()));
children.extend(
every_join(select)
.into_iter()
.filter_map(|join| join.on.as_ref()),
);
children
}
Expression::Eq(_)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use crate::lineage::_helpers::checks::coercion::{
use crate::lineage::_helpers::checks::families::expression_family;
use crate::lineage::_helpers::checks::issues::issue;
use crate::lineage::_helpers::checks::sets::{Branches, families, width};
use crate::lineage::_helpers::tracing::sources::from_parts;
use crate::lineage::constants::TYPE_PHASE;
use crate::lineage::main::type_family::type_family;
use crate::lineage::models::Engine;
Expand Down Expand Up @@ -126,17 +127,13 @@ pub(crate) fn using_findings(
select: &Select,
(scope, context): (&LineageScope<'_>, LineageContext<'_>),
) -> Vec<NameFinding> {
let single_from: bool = select
.from
.as_ref()
.is_some_and(|from| from.expressions.len() == 1);
let [join] = select.joins.as_slice() else {
let [(_, None), (_, Some(join))] = from_parts(select)[..] else {
return Vec::new();
};
let [left_source, right_source] = scope.sources.as_slice() else {
return Vec::new();
};
if !single_from || join.using.is_empty() {
if join.using.is_empty() {
return Vec::new();
}
let engine: Engine = Engine::of(context);
Expand Down
Loading
Loading