diff --git a/benchmarks/duckshop/project/rules/quoted_aliases.py b/benchmarks/duckshop/project/rules/quoted_aliases.py new file mode 100644 index 0000000000..bfb855b1ce --- /dev/null +++ b/benchmarks/duckshop/project/rules/quoted_aliases.py @@ -0,0 +1,35 @@ +"""DuckShop house rule that keeps case-sensitive output names inside partner export models.""" + +from sqlbuild.rules import Finding, Model, RuleContext, rule + +EXPORT_SUFFIX = "_export" + + +@rule( + code="XSQBRDUCK003", + message="Only partner export models may give columns quoted mixed-case names", + remediation=( + "Name the column in unquoted snake_case, or move the projection into a model whose name " + "ends in _export, which writes a partner's own feed format." + ), +) +def mixed_case_names_stay_in_exports(*, model: Model, ctx: RuleContext) -> list[Finding]: + if model.path.stem.endswith(EXPORT_SUFFIX): + return [] + sql = ctx.sql.for_model(model).expanded + if '"' not in sql.source: + return [] + findings: list[Finding] = [] + for node in sql.ast().walk(): + if node.kind != "alias" or not node.quoted or node.name is None: + continue + if node.name == node.name.lower(): + continue + text = sql.source[node.span[0] : node.span[1]] if node.span else node.name + findings.append( + ctx.finding( + subject=model, + message=f"Quoted mixed-case name {node.name!r} in `{text}`", + ) + ) + return findings diff --git a/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py new file mode 100644 index 0000000000..9a6079175b --- /dev/null +++ b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py @@ -0,0 +1,50 @@ +from rules.quoted_aliases import mixed_case_names_stay_in_exports + +from sqlbuild.rules import RuleCase, RuleFile, evaluate_rule + +_HEADER = 'MODEL (description "Listings");\n' +_MIXED_CASE = ( + _HEADER + + "SELECT\n" + + ' l.listing_id AS "listingId",\n' + + ' l.note AS "note",\n' + + " 'isLive' AS status_label\n" + + 'FROM __ref("dim_listings") AS l\n' +) +_LISTINGS = RuleFile( + path="models/marts/catalogue/dim_listings.sql", + source=_HEADER + "SELECT 1 AS listing_id, 'new' AS note\n", +) + + +def test_given_quoted_mixed_case_alias_outside_an_export_when_evaluating_then_reports_it() -> None: + result = evaluate_rule( + rule=mixed_case_names_stay_in_exports, + test_case=RuleCase( + description="a mart names a column in a partner's case", + source=_MIXED_CASE, + path="models/marts/catalogue/rpt_listings.sql", + files=(_LISTINGS,), + expected_finding_count=1, + ), + ) + + assert result.finding_count == 1 + assert result.findings[0].message == ( + "Quoted mixed-case name 'listingId' in `l.listing_id AS \"listingId\"`" + ) + + +def test_given_quoted_mixed_case_alias_in_an_export_when_evaluating_then_passes() -> None: + result = evaluate_rule( + rule=mixed_case_names_stay_in_exports, + test_case=RuleCase( + description="a partner export writes the partner's own names", + source=_MIXED_CASE, + path="models/marts/edge_cases/rpt_edge_partner_listing_export.sql", + files=(_LISTINGS,), + expected_finding_count=0, + ), + ) + + assert result.finding_count == 0 diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/located_markers_as_names.rs b/crates/sqlbuild-analysis/src/relation_markers/main/located_markers_as_names.rs new file mode 100644 index 0000000000..c96f88e66d --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/main/located_markers_as_names.rs @@ -0,0 +1,63 @@ +//! `located_markers_as_names`: relation marker calls in code as plain names that keep every position, with where each call and name stand. + +use regex::Regex; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::relation_markers::_helpers::pattern::relation_marker_pattern; +use crate::relation_markers::constants::{DBT_REF_FUNCTION, MARKER_NAME_PREFIX}; +use crate::relation_markers::types::{MarkerCall, MarkerRanges}; + +/// `sql` with each marker call in code as `__sqb_`, padded so later text keeps its line and column, the calls by index, and each call's and padded name's byte ranges. +pub fn located_markers_as_names( + sql: &str, + syntax: &LexicalSyntax, +) -> Result<(String, Vec, Vec), String> { + let pattern: &Regex = relation_marker_pattern()?; + let mut protected: Option> = None; + let mut named: String = String::with_capacity(sql.len()); + let mut calls: Vec = Vec::new(); + let mut located: Vec = Vec::new(); + let mut cursor: usize = 0; + for captures in pattern.captures_iter(sql) { + let (Some(full), Some(function), Some(first)) = + (captures.get(0), captures.get(1), captures.get(2)) + else { + continue; + }; + let ranges = protected.get_or_insert_with(|| dialect_non_code_ranges(sql, syntax)); + let following: usize = ranges.partition_point(|(start, _)| *start <= full.start()); + if following > 0 && full.start() < ranges[following - 1].1 { + continue; + } + let function_name: String = function.as_str().to_ascii_lowercase(); + let referenced_name: String = match (captures.get(3), function_name == DBT_REF_FUNCTION) { + (None, _) => first.as_str().to_owned(), + (Some(second), true) => format!("{}__{}", first.as_str(), second.as_str()), + (Some(_), false) => continue, + }; + named.push_str(&sql[cursor..full.start()]); + let name_start: usize = named.len(); + named.push_str(&padded_name( + &format!("{MARKER_NAME_PREFIX}{}", calls.len()), + full.as_str(), + )); + cursor = full.end(); + calls.push((function_name, referenced_name)); + located.push((full.range(), name_start..named.len())); + } + named.push_str(&sql[cursor..]); + Ok((named, calls, located)) +} + +/// `name`, then the call's line breaks, then spaces as wide as the call's last line. +fn padded_name(name: &str, call: &str) -> String { + let breaks: usize = call.matches('\n').count(); + let last_line: &str = call.rsplit('\n').next().unwrap_or(call); + let width: usize = if breaks == 0 { + call.chars().count().saturating_sub(name.chars().count()) + } else { + last_line.chars().count() + }; + format!("{name}{}{}", "\n".repeat(breaks), " ".repeat(width)) +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs b/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs index 9330c94939..c3df988592 100644 --- a/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs +++ b/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs @@ -1,11 +1,8 @@ //! `markers_as_names`: relation marker calls in code as plain names that keep every position. -use regex::Regex; -use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::relation_markers::_helpers::pattern::relation_marker_pattern; -use crate::relation_markers::constants::{DBT_REF_FUNCTION, MARKER_NAME_PREFIX}; +use crate::relation_markers::main::located_markers_as_names::located_markers_as_names; use crate::relation_markers::types::MarkerCall; /// `sql` with each marker call in code as `__sqb_`, padded so later text keeps its line and column, and the calls by index. @@ -13,48 +10,5 @@ pub fn markers_as_names( sql: &str, syntax: &LexicalSyntax, ) -> Result<(String, Vec), String> { - let pattern: &Regex = relation_marker_pattern()?; - let mut protected: Option> = None; - let mut named: String = String::with_capacity(sql.len()); - let mut calls: Vec = Vec::new(); - let mut cursor: usize = 0; - for captures in pattern.captures_iter(sql) { - let (Some(full), Some(function), Some(first)) = - (captures.get(0), captures.get(1), captures.get(2)) - else { - continue; - }; - let ranges = protected.get_or_insert_with(|| dialect_non_code_ranges(sql, syntax)); - let following: usize = ranges.partition_point(|(start, _)| *start <= full.start()); - if following > 0 && full.start() < ranges[following - 1].1 { - continue; - } - let function_name: String = function.as_str().to_ascii_lowercase(); - let referenced_name: String = match (captures.get(3), function_name == DBT_REF_FUNCTION) { - (None, _) => first.as_str().to_owned(), - (Some(second), true) => format!("{}__{}", first.as_str(), second.as_str()), - (Some(_), false) => continue, - }; - named.push_str(&sql[cursor..full.start()]); - named.push_str(&padded_name( - &format!("{MARKER_NAME_PREFIX}{}", calls.len()), - full.as_str(), - )); - cursor = full.end(); - calls.push((function_name, referenced_name)); - } - named.push_str(&sql[cursor..]); - Ok((named, calls)) -} - -/// `name`, then the call's line breaks, then spaces as wide as the call's last line. -fn padded_name(name: &str, call: &str) -> String { - let breaks: usize = call.matches('\n').count(); - let last_line: &str = call.rsplit('\n').next().unwrap_or(call); - let width: usize = if breaks == 0 { - call.chars().count().saturating_sub(name.chars().count()) - } else { - last_line.chars().count() - }; - format!("{name}{}{}", "\n".repeat(breaks), " ".repeat(width)) + located_markers_as_names(sql, syntax).map(|(named, calls, _)| (named, calls)) } diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs b/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs index abc0f2dc58..35bbba4636 100644 --- a/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs +++ b/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs @@ -1,2 +1,3 @@ pub mod lateral_start; +pub mod located_markers_as_names; pub mod markers_as_names; diff --git a/crates/sqlbuild-analysis/src/relation_markers/types.rs b/crates/sqlbuild-analysis/src/relation_markers/types.rs index 9b90a22d2a..13dab5ded5 100644 --- a/crates/sqlbuild-analysis/src/relation_markers/types.rs +++ b/crates/sqlbuild-analysis/src/relation_markers/types.rs @@ -1,4 +1,9 @@ //! Shapes of relation marker calls. +use std::ops::Range; + /// A marker call's lower-case function, such as `__ref`, and the name it references. pub type MarkerCall = (String, String); + +/// A marker call's byte range in the source, and the byte range of the padded name that stands for it. +pub type MarkerRanges = (Range, Range); diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/clauses.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/clauses.rs index 9fa047167c..e8a0712125 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/clauses.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/clauses.rs @@ -1,15 +1,17 @@ //! Join and CTE clauses, which SQLQuack keeps as statement fields rather than expressions. use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::names::models::Identifier; use sqlquack_syntax::expressions::queries::models::{Cte, Join, JoinKind, With}; use sqlquack_syntax::tokens::models::Span; use crate::sql_tree::constants::{CTE_KIND, JOIN_KIND}; -/// One clause of a statement: its kind, its span, and the expressions it owns. +/// One clause of a statement: its kind, its span, its name, and the expressions it owns. pub(crate) struct Clause<'ast> { pub(crate) kind: &'static str, pub(crate) span: Option, + pub(crate) name: Option<&'ast Identifier>, pub(crate) children: Vec<&'ast Expression>, } @@ -45,6 +47,7 @@ fn cte_clause(cte: &Cte) -> Clause<'_> { Clause { kind: CTE_KIND, span: cte.span.0, + name: Some(&cte.alias), children: vec![&cte.this], } } @@ -57,6 +60,7 @@ fn join_clause(join: &Join) -> Clause<'_> { Clause { kind: JOIN_KIND, span: join.span.0, + name: None, children, } } diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs index 942bd41b25..66e6f6f7d7 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs @@ -1,8 +1,10 @@ -//! Parser selection, parsing, the tree walk, clause nodes, source positions and table wrapping. +//! Parser selection, parsing, the tree walk, clause nodes, node facts, source positions and table wrapping. pub(crate) mod clauses; +pub(crate) mod node_facts; pub(crate) mod parse; pub(crate) mod positions; pub(crate) mod rules_tree_parser; +pub(crate) mod source_map; pub(crate) mod table_wrapped; pub(crate) mod walk; diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs new file mode 100644 index 0000000000..43d629debb --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs @@ -0,0 +1,100 @@ +//! What a node names or holds: identifier, column, alias, table and CTE names, and literal values. + +use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::names::models::{Identifier, Literal, TableRef}; +use sqlquack_syntax::tokens::models::Span; + +use crate::relation_markers::constants::MARKER_NAME_PREFIX; +use crate::sql_tree::_helpers::source_map::SourceMap; +use crate::sql_tree::models::{NodeLiteral, NodeName}; + +/// The name `expression` carries, its qualifier outermost first; a relation marker's name is its call. +pub(crate) fn node_name(expression: &Expression, map: &SourceMap<'_>) -> Option { + let (name, qualifier): (&Identifier, Vec<&Identifier>) = match expression { + Expression::Identifier(identifier) => (identifier, Vec::new()), + Expression::Column(column) => (&column.name, column.table.iter().collect()), + Expression::Alias(alias) => (&alias.alias, Vec::new()), + Expression::Table(table) => ( + &table.name, + table.catalog.iter().chain(table.schema.iter()).collect(), + ), + _ => return None, + }; + Some(named(name, &qualifier, map)) +} + +/// `name` and its qualifier as written, marker names read back as their calls. +pub(crate) fn named(name: &Identifier, qualifier: &[&Identifier], map: &SourceMap<'_>) -> NodeName { + let (text, quoted) = written(name, map); + NodeName { + name: text, + quoted, + qualifier: qualifier.iter().map(|part| written(part, map).0).collect(), + } +} + +/// The value and snake_case type of a literal node. +pub(crate) fn node_literal(expression: &Expression) -> Option { + let Expression::Literal(literal) = expression else { + return None; + }; + Some(NodeLiteral { + value: literal.value_str().to_owned(), + literal_type: literal_type(literal), + }) +} + +/// A table's source range: from its outermost qualifier to the end of its name, without its alias. +pub(crate) fn table_span(table: &TableRef) -> Option { + let end: Span = table.name.span?; + let first: &Identifier = table + .catalog + .as_ref() + .or(table.schema.as_ref()) + .unwrap_or(&table.name); + let qualified = [table.catalog.as_ref(), table.schema.as_ref()]; + if qualified.iter().flatten().any(|part| part.span.is_none()) { + return None; + } + first.span.map(|start| start.through(end)) +} + +/// An identifier's text and quoting; a marker name reads as the call it stands for, unquoted. +fn written(identifier: &Identifier, map: &SourceMap<'_>) -> (String, bool) { + match marker_call(identifier, map) { + Some(call) => (call.to_owned(), false), + None => (identifier.name.clone(), identifier.quoted), + } +} + +/// The call an unquoted marker name stands for. +fn marker_call<'source>(identifier: &Identifier, map: &SourceMap<'source>) -> Option<&'source str> { + if identifier.quoted { + return None; + } + let index: &str = identifier.name.strip_prefix(MARKER_NAME_PREFIX)?; + match index.parse::() { + Ok(index) => map.calls.get(index).copied(), + Err(_not_a_marker) => None, + } +} + +fn literal_type(literal: &Literal) -> &'static str { + match literal { + Literal::String(_) => "string", + Literal::Number(_) => "number", + Literal::HexString(_) => "hex_string", + Literal::HexNumber(_) => "hex_number", + Literal::BitString(_) => "bit_string", + Literal::ByteString(_) => "byte_string", + Literal::NationalString(_) => "national_string", + Literal::Date(_) => "date", + Literal::Time(_) => "time", + Literal::Timestamp(_) => "timestamp", + Literal::Datetime(_) => "datetime", + Literal::TripleQuotedString(_, _) => "triple_quoted_string", + Literal::EscapeString(_) => "escape_string", + Literal::DollarString(_) => "dollar_string", + Literal::RawString(_) => "raw_string", + } +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs new file mode 100644 index 0000000000..171852bbcd --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs @@ -0,0 +1,119 @@ +//! Spans in the parsed text read back as positions in the rule's source, across marker names and `TABLE( )` wrappers. + +use std::ops::Range; + +use sqlquack_syntax::tokens::models::Span; + +use crate::relation_markers::types::MarkerRanges; +use crate::sql_tree::_helpers::positions::{line_column, line_starts}; +use crate::sql_tree::models::NodeExtent; + +/// The parsed text's `parsed` characters stand where the earlier text's `source` characters were. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Rewrite { + pub(crate) parsed: Range, + pub(crate) source: Range, +} + +/// The rule's source as the parser's text maps back to it. +pub(crate) struct SourceMap<'source> { + /// Each marker call's source text, by marker index. + pub(crate) calls: Vec<&'source str>, + line_starts: Vec, + /// Marker names over their calls, in order: named text to source. + markers: Vec, + /// `TABLE( )` wrappers, in order: parsed text to named text. + wrappers: Vec, +} + +impl<'source> SourceMap<'source> { + /// The map of `source`, whose marker calls `named` holds as names at `ranges`, parsed after `wrappers`. + pub(crate) fn new( + source: &'source str, + named: &str, + ranges: &[MarkerRanges], + wrappers: Vec, + ) -> Self { + let calls: Vec> = char_ranges(source, ranges.iter().map(|(call, _)| call)); + let names: Vec> = char_ranges(named, ranges.iter().map(|(_, name)| name)); + Self { + calls: ranges + .iter() + .map(|(call, _)| &source[call.clone()]) + .collect(), + line_starts: line_starts(source), + markers: names + .into_iter() + .zip(calls) + .map(|(parsed, source)| Rewrite { parsed, source }) + .collect(), + wrappers, + } + } + + /// The 1-based line and column of a parsed character offset where a node starts. + pub(crate) fn start(&self, offset: usize) -> (usize, usize) { + line_column( + &self.line_starts, + source_start(&self.markers, source_start(&self.wrappers, offset)), + ) + } + + /// The source extent of a parsed span: a node starting or ending inside a marker name covers its whole call. + pub(crate) fn extent(&self, span: Span) -> NodeExtent { + let start: usize = source_start(&self.markers, source_start(&self.wrappers, span.start)); + let end: usize = source_end(&self.markers, source_end(&self.wrappers, span.end)); + let (end_line, end_column) = line_column(&self.line_starts, end); + NodeExtent { + end_line, + end_column, + span: (start, end), + } + } +} + +/// The earlier text's offset for a start offset: inside a rewrite, the start of what it replaced. +fn source_start(rewrites: &[Rewrite], offset: usize) -> usize { + let after: usize = rewrites.partition_point(|rewrite| rewrite.parsed.end <= offset); + match rewrites.get(after) { + Some(rewrite) if rewrite.parsed.start <= offset => rewrite.source.start, + _ => shifted(rewrites[..after].last(), offset), + } +} + +/// The earlier text's offset for an exclusive end offset: inside or closing a rewrite, the end of what it replaced, and at a removal, where the removed text began. +fn source_end(rewrites: &[Rewrite], offset: usize) -> usize { + let after: usize = rewrites.partition_point(|rewrite| rewrite.parsed.end < offset); + match rewrites.get(after) { + Some(rewrite) if rewrite.parsed.start < offset => rewrite.source.end, + Some(rewrite) if rewrite.parsed.is_empty() && rewrite.parsed.start == offset => { + rewrite.source.start + } + _ => shifted(rewrites[..after].last(), offset), + } +} + +/// `offset` past the last rewrite before it, moved as that rewrite moved the text after it. +fn shifted(previous: Option<&Rewrite>, offset: usize) -> usize { + previous.map_or(offset, |rewrite| { + rewrite.source.end + (offset - rewrite.parsed.end) + }) +} + +/// Ascending byte `ranges` of `text` as character ranges, counted in one pass. +fn char_ranges<'a>( + text: &str, + ranges: impl Iterator>, +) -> Vec> { + let mut byte: usize = 0; + let mut chars: usize = 0; + ranges + .map(|range| { + chars += text[byte..range.start].chars().count(); + let start: usize = chars; + chars += text[range.clone()].chars().count(); + byte = range.end; + start..chars + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/table_wrapped.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/table_wrapped.rs index 6bb74987db..c8ca4b19e1 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/table_wrapped.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/table_wrapped.rs @@ -1,7 +1,5 @@ //! Snowflake table function markers wrapped in `TABLE( )`, as SQLBuild renders them. -use std::cmp::Reverse; - use sqlbuild_sqltext::sql_scan::constants::TABLE_WRAPPER; use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_paren; use sqlbuild_sqltext::sql_scan::main::follows_table_wrapper::follows_table_wrapper; @@ -9,11 +7,16 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::relation_markers::constants::{MARKER_NAME_PREFIX, TABLE_FUNCTION_MARKER}; use crate::relation_markers::types::MarkerCall; +use crate::sql_tree::_helpers::source_map::Rewrite; -/// `named` with each table function marker call in `TABLE( )`, written over padding so later text keeps its column. -pub(crate) fn table_wrapped(named: &str, calls: &[MarkerCall], syntax: &LexicalSyntax) -> String { +/// `named` with each table function marker call in `TABLE( )`, written over padding so later text keeps its column, and the rewrites that did it. +pub(crate) fn table_wrapped( + named: &str, + calls: &[MarkerCall], + syntax: &LexicalSyntax, +) -> (String, Vec) { let bytes = named.as_bytes(); - let mut edits: Vec<(usize, usize, String)> = Vec::new(); + let mut edits: Vec<(usize, usize, &str)> = Vec::new(); for (index, (function, _)) in calls.iter().enumerate() { if function != TABLE_FUNCTION_MARKER { continue; @@ -42,15 +45,41 @@ pub(crate) fn table_wrapped(named: &str, calls: &[MarkerCall], syntax: &LexicalS let replaced = usize::from(matches!(bytes.get(after), Some(b' ' | b'\t'))); let text_follows = !matches!(bytes.get(after), None | Some(b'\n' | b'\r')); let borrowed = spaces.min(TABLE_WRAPPER.len() + usize::from(text_follows) - replaced); - edits.push((start, name_end + borrowed, format!("{TABLE_WRAPPER}{name}"))); - edits.push((after, after + replaced, ")".to_owned())); + edits.push((start, start, TABLE_WRAPPER)); + edits.push((name_end, name_end + borrowed, "")); + edits.push((after, after, ")")); + edits.push((after, after + replaced, "")); } - edits.sort_by_key(|(start, _, _)| Reverse(*start)); - let mut wrapped = named.to_owned(); - for (start, end, replacement) in edits { - wrapped.replace_range(start..end, &replacement); + edits.sort_by_key(|(start, _, _)| *start); + applied(named, &edits) +} + +/// `named` with each ascending edit's byte range replaced by its text, and each edit as a character rewrite. +fn applied(named: &str, edits: &[(usize, usize, &str)]) -> (String, Vec) { + let mut wrapped: String = String::with_capacity(named.len() + edits.len()); + let mut rewrites: Vec = Vec::with_capacity(edits.len()); + let (mut cursor, mut named_chars, mut wrapped_chars) = (0, 0, 0); + for (start, end, text) in edits + .iter() + .filter(|(start, end, text)| start < end || !text.is_empty()) + { + let kept: &str = &named[cursor..*start]; + wrapped.push_str(kept); + named_chars += kept.chars().count(); + wrapped_chars += kept.chars().count(); + let removed: usize = named[*start..*end].chars().count(); + let added: usize = text.chars().count(); + rewrites.push(Rewrite { + parsed: wrapped_chars..wrapped_chars + added, + source: named_chars..named_chars + removed, + }); + wrapped.push_str(text); + named_chars += removed; + wrapped_chars += added; + cursor = *end; } - wrapped + wrapped.push_str(&named[cursor..]); + (wrapped, rewrites) } /// Where `name` first stands as a whole word in `bytes`. diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/walk.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/walk.rs index 2d59d234d8..bde0b75c17 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/walk.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/walk.rs @@ -6,7 +6,8 @@ use sqlquack_syntax::tokens::models::Span; use sqlquack_syntax::{Expression, ExpressionWalk}; use crate::sql_tree::_helpers::clauses::clauses; -use crate::sql_tree::_helpers::positions::line_column; +use crate::sql_tree::_helpers::node_facts::{named, node_literal, node_name, table_span}; +use crate::sql_tree::_helpers::source_map::SourceMap; use crate::sql_tree::constants::OPAQUE_KINDS; use crate::sql_tree::errors::SqlTreeError; use crate::sql_tree::models::SqlTreeNode; @@ -17,7 +18,7 @@ type Owners = HashMap<*const Expression, usize>; /// The nodes of `statements` in pre-order, walked with an explicit stack so depth cannot overflow. pub(crate) fn walk( statements: &[Expression], - starts: &[usize], + map: &SourceMap<'_>, ) -> Result, SqlTreeError> { let mut nodes: Vec = Vec::new(); let mut stack: Vec<(&Expression, Option)> = statements @@ -27,8 +28,8 @@ pub(crate) fn walk( .collect(); while let Some((expression, parent)) = stack.pop() { let index: usize = nodes.len(); - let node: SqlTreeNode = expression_node(&nodes, expression, parent, starts)?; - let (clause_nodes, owners) = clause_nodes(expression, &node, index, starts); + let node: SqlTreeNode = expression_node(&nodes, expression, parent, map)?; + let (clause_nodes, owners) = clause_nodes(expression, &node, index, map); nodes.push(node); nodes.extend(clause_nodes); for child in expression.children().into_iter().rev() { @@ -42,12 +43,12 @@ pub(crate) fn walk( Ok(nodes) } -/// `expression`'s node, at its own start or else at its parent's. +/// `expression`'s node, at its own start or else at its parent's, with its end only where the parser recorded one. fn expression_node( nodes: &[SqlTreeNode], expression: &Expression, parent: Option, - starts: &[usize], + map: &SourceMap<'_>, ) -> Result { let kind: &'static str = expression.variant_name(); if OPAQUE_KINDS.contains(&kind) { @@ -56,13 +57,18 @@ fn expression_node( )); } let inherited = parent.map_or((1, 1), |parent| (nodes[parent].line, nodes[parent].column)); - let (line, column) = - own_span(expression).map_or(inherited, |span| line_column(starts, span.start)); + let span: Option = recorded_span(expression); + let (line, column) = span + .or_else(|| qualified_star_start(expression)) + .map_or(inherited, |span| map.start(span.start)); Ok(SqlTreeNode { kind, line, column, parent, + extent: span.map(|span| map.extent(span)), + name: node_name(expression, map), + literal: node_literal(expression), }) } @@ -71,20 +77,23 @@ fn clause_nodes( expression: &Expression, node: &SqlTreeNode, index: usize, - starts: &[usize], + map: &SourceMap<'_>, ) -> (Vec, Owners) { let mut built: Vec = Vec::new(); let mut owners: Owners = HashMap::new(); for clause in clauses(expression) { let clause_index: usize = index + 1 + built.len(); - let (line, column) = clause.span.map_or((node.line, node.column), |span| { - line_column(starts, span.start) - }); + let (line, column) = clause + .span + .map_or((node.line, node.column), |span| map.start(span.start)); built.push(SqlTreeNode { kind: clause.kind, line, column, parent: Some(index), + extent: clause.span.map(|span| map.extent(span)), + name: clause.name.map(|name| named(name, &[], map)), + literal: None, }); for child in clause.children { let _ = owners.insert(std::ptr::from_ref(child), clause_index); @@ -93,10 +102,18 @@ fn clause_nodes( (built, owners) } -/// The node's span; a qualified star without one starts at its qualifier. -fn own_span(expression: &Expression) -> Option { +/// The node's recorded span; a table's runs from its first qualifier to its name. +fn recorded_span(expression: &Expression) -> Option { expression.source_span().or_else(|| match expression { - Expression::Star(star) => star.table.as_ref().and_then(|table| table.span), + Expression::Table(table) => table_span(table), _ => None, }) } + +/// Where a qualified star without a span starts: at its qualifier, which is not where it ends. +fn qualified_star_start(expression: &Expression) -> Option { + match expression { + Expression::Star(star) => star.table.as_ref().and_then(|table| table.span), + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/main/sql_tree_nodes.rs b/crates/sqlbuild-analysis/src/sql_tree/main/sql_tree_nodes.rs index c1a036e032..fed642126f 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/main/sql_tree_nodes.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/main/sql_tree_nodes.rs @@ -3,10 +3,10 @@ use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; use sqlquack_syntax::Expression; -use crate::relation_markers::main::markers_as_names::markers_as_names; +use crate::relation_markers::main::located_markers_as_names::located_markers_as_names; use crate::sql_tree::_helpers::parse::parse; -use crate::sql_tree::_helpers::positions::line_starts; use crate::sql_tree::_helpers::rules_tree_parser::rules_tree_parser; +use crate::sql_tree::_helpers::source_map::SourceMap; use crate::sql_tree::_helpers::table_wrapped::table_wrapped; use crate::sql_tree::_helpers::walk::walk; use crate::sql_tree::constants::{SNOWFLAKE_DIALECT, TREE_WORKER_STACK_BYTES}; @@ -14,7 +14,7 @@ use crate::sql_tree::errors::SqlTreeError; use crate::sql_tree::models::SqlTreeNode; use crate::worker_stack::main::run_on_worker_stack::run_on_worker_stack; -/// Every node of `sql` parsed in `dialect`, in pre-order, joins and CTEs owning their expressions; relation markers read as tables. +/// Every node of `sql` parsed in `dialect`, in pre-order, joins and CTEs owning their expressions; relation markers read as tables named by their calls, and every position is in `sql`. pub fn sql_tree_nodes(sql: &str, dialect: &str) -> Result, SqlTreeError> { run_on_worker_stack(TREE_WORKER_STACK_BYTES, "the parser panicked", || { tree_nodes(sql, dialect) @@ -26,12 +26,15 @@ fn tree_nodes(sql: &str, dialect: &str) -> Result, SqlTreeError let parser = rules_tree_parser(dialect) .ok_or_else(|| SqlTreeError::UnavailableDialect(dialect.to_owned()))?; let syntax = dialect_lexical_syntax(dialect); - let (named, calls) = markers_as_names(sql, &syntax).map_err(SqlTreeError::Unanalysable)?; - let named = if dialect == SNOWFLAKE_DIALECT { - table_wrapped(&named, &calls, &syntax) + let (named, calls, ranges) = + located_markers_as_names(sql, &syntax).map_err(SqlTreeError::Unanalysable)?; + let (wrapped, wrappers) = if dialect == SNOWFLAKE_DIALECT { + let (wrapped, wrappers) = table_wrapped(&named, &calls, &syntax); + (Some(wrapped), wrappers) } else { - named + (None, Vec::new()) }; - let statements: Vec = parse(&named, parser)?; - walk(&statements, &line_starts(&named)) + let map = SourceMap::new(sql, &named, &ranges, wrappers); + let statements: Vec = parse(wrapped.as_deref().unwrap_or(&named), parser)?; + walk(&statements, &map) } diff --git a/crates/sqlbuild-analysis/src/sql_tree/models.rs b/crates/sqlbuild-analysis/src/sql_tree/models.rs index 22964a4c2d..1e4b2f8811 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/models.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/models.rs @@ -1,6 +1,6 @@ //! Nodes of a rule's SQL syntax tree. -/// One node: its SQLQuack kind, where it starts, and its parent's index in pre-order. +/// One node: its SQLQuack kind, where it starts and ends, what it names or holds, and its parent's index in pre-order. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SqlTreeNode { /// SQLQuack's snake_case node kind, such as `select`, `subquery`, `star`, `join` or `cte`. @@ -11,6 +11,43 @@ pub struct SqlTreeNode { pub column: usize, /// The parent's index; `None` for a statement. pub parent: Option, + /// The source text the node covers; `None` when the parser recorded no span for it. + pub extent: Option, + /// The name an identifier, column, alias or table node carries. + pub name: Option, + /// The value a literal node holds. + pub literal: Option, +} + +/// Where a node's source text ends, and the character range it covers in the source. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeExtent { + /// 1-based line of the position just after the node's last character. + pub end_line: usize, + /// 1-based column, in characters, of the position just after the node's last character. + pub end_column: usize, + /// The node's start and exclusive end as character offsets into the source. + pub span: (usize, usize), +} + +/// A name as written; a relation marker's name is its call, such as `__ref("orders")`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeName { + /// The name without quote characters. + pub name: String, + /// Whether the source quoted the name. + pub quoted: bool, + /// The names before it, outermost first, such as `["db", "sales"]` in `db.sales.orders`. + pub qualifier: Vec, +} + +/// A literal's value as written, without quotes, and its SQLQuack type, such as `string` or `number`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeLiteral { + /// The value's text. + pub value: String, + /// The snake_case literal type. + pub literal_type: &'static str, } /// The SQLQuack parser that reads a dialect's SQL for rules. diff --git a/crates/sqlbuild-analysis/src/sql_tree/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_tree/tests/helpers.rs index 8338f0309f..37326bf842 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/tests/helpers.rs @@ -1,6 +1,8 @@ -use crate::sql_tree::models::SqlTreeNode; +use crate::sql_tree::models::{NodeExtent, NodeName, SqlTreeNode}; const SHAPE_KINDS: [&str; 5] = ["select", "cte", "join", "subquery", "star"]; +/// How a name's fact reads unquoted, then quoted. +const QUOTING: [&str; 2] = ["", " quoted"]; /// The listed kinds' nodes with their parent's kind, sorted by position, since child order is SQLQuack's. pub(crate) fn shape( @@ -17,3 +19,43 @@ pub(crate) fn shape( shape.sort_by_key(|(kind, line, column, _)| (*line, *column, *kind)); shape } + +/// Each node as `kind line:column`, then what it names or holds, then its end and the source text it covers. +pub(crate) fn facts(sql: &str, nodes: &[SqlTreeNode]) -> Vec { + nodes + .iter() + .map(|node| { + format!( + "{} {}:{}{}{}{}", + node.kind, + node.line, + node.column, + node.name.as_ref().map(name_fact).unwrap_or_default(), + node.literal + .as_ref() + .map(|literal| format!(" {}={}", literal.literal_type, literal.value)) + .unwrap_or_default(), + node.extent + .as_ref() + .map(|extent| extent_fact(sql, extent)) + .unwrap_or_default(), + ) + }) + .collect() +} + +fn name_fact(name: &NodeName) -> String { + let quoted: &str = QUOTING[usize::from(name.quoted)]; + let qualifier: String = name + .qualifier + .first() + .map(|_| format!(" in={}", name.qualifier.join("."))) + .unwrap_or_default(); + format!(" name={}{quoted}{qualifier}", name.name) +} + +fn extent_fact(sql: &str, extent: &NodeExtent) -> String { + let (start, end) = extent.span; + let text: String = sql.chars().skip(start).take(end - start).collect(); + format!(" to {}:{} {text:?}", extent.end_line, extent.end_column) +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/tests/test_sql_tree.rs b/crates/sqlbuild-analysis/src/sql_tree/tests/test_sql_tree.rs index 14eaf3a826..b16b116e67 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/tests/test_sql_tree.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/tests/test_sql_tree.rs @@ -1,13 +1,18 @@ use crate::sql_tree::errors::SqlTreeError; use crate::sql_tree::main::sql_tree_nodes::sql_tree_nodes; -use crate::sql_tree::tests::helpers::shape; -use crate::sql_tree::tests::test_types::{TreeFailureTestCase, TreeShapeTestCase}; +use crate::sql_tree::tests::helpers::{facts, shape}; +use crate::sql_tree::tests::test_types::{ + TreeFactsTestCase, TreeFailureTestCase, TreeShapeTestCase, +}; const CTE_AND_JOIN_SQL: &str = "WITH x AS (SELECT 1 AS id)\nSELECT *\n FROM x JOIN (SELECT 2 AS id) s ON x.id = s.id"; const SNOWFLAKE_MARKER_SQL: &str = "WITH o AS (SELECT * FROM __ref(\"orders\"))\nSELECT o.*, c.name\nFROM o JOIN __ref(\"customers\") c ON o.id = c.id JOIN (SELECT 2 AS id) s ON s.id = o.id\nWHERE o.id IN (SELECT id FROM __source(\"raw_returns\"))"; +const NAMES_AND_LITERALS_SQL: &str = + "SELECT o.\"Id\" AS order_id, 'x''y' AS s, 42 + 1\nFROM db.sch.orders AS o WHERE o.v = N'z'"; + #[test] fn given_sql_when_building_a_rule_tree_then_clauses_own_their_queries_at_their_source_positions() { let test_cases = [ @@ -140,6 +145,108 @@ fn given_sql_when_building_a_rule_tree_then_clauses_own_their_queries_at_their_s } } +#[test] +fn given_sql_when_building_a_rule_tree_then_nodes_carry_names_literals_and_source_extents() { + let test_cases = [ + TreeFactsTestCase { + description: "names keep quoting and qualifiers; a literal without its own span has no end", + dialect: "duckdb", + sql: NAMES_AND_LITERALS_SQL, + expected_facts: &[ + "select 1:1 to 2:41 \"SELECT o.\\\"Id\\\" AS order_id, 'x''y' AS s, 42 + 1\\nFROM db.sch.orders AS o WHERE o.v = N'z'\"", + "alias 1:8 name=order_id to 1:26 \"o.\\\"Id\\\" AS order_id\"", + "column 1:8 name=Id quoted in=o to 1:14 \"o.\\\"Id\\\"\"", + "alias 1:28 name=s to 1:39 \"'x''y' AS s\"", + "literal 1:28 string=x'y", + "add 1:41 to 1:47 \"42 + 1\"", + "literal 1:41 number=42", + "literal 1:41 number=1", + "table 2:6 name=orders in=db.sch to 2:19 \"db.sch.orders\"", + "eq 2:31 to 2:41 \"o.v = N'z'\"", + "column 2:31 name=v in=o to 2:34 \"o.v\"", + "literal 2:31 national_string=z", + ], + }, + TreeFactsTestCase { + description: "native PostgreSQL records the same names and spans, a typed literal as a cast", + dialect: "postgres", + sql: "SELECT o.\"Id\" AS order_id\nFROM db.sch.orders AS o WHERE o.v = DATE '2024-01-02'", + expected_facts: &[ + "select 1:1 to 2:54 \"SELECT o.\\\"Id\\\" AS order_id\\nFROM db.sch.orders AS o WHERE o.v = DATE '2024-01-02'\"", + "alias 1:8 name=order_id to 1:26 \"o.\\\"Id\\\" AS order_id\"", + "column 1:8 name=Id quoted in=o to 1:14 \"o.\\\"Id\\\"\"", + "table 2:6 name=orders in=db.sch to 2:19 \"db.sch.orders\"", + "eq 2:31 to 2:54 \"o.v = DATE '2024-01-02'\"", + "column 2:31 name=v in=o to 2:34 \"o.v\"", + "cast 2:37 to 2:54 \"DATE '2024-01-02'\"", + "literal 2:37 string=2024-01-02", + ], + }, + TreeFactsTestCase { + description: "a marker over three lines is named by its call and ends where the call ends", + dialect: "duckdb", + sql: "SELECT c.id\nFROM __ref(\n \"customers\"\n) AS c\nJOIN orders o ON o.cid = c.id", + expected_facts: &[ + "select 1:1 to 5:30 \"SELECT c.id\\nFROM __ref(\\n \\\"customers\\\"\\n) AS c\\nJOIN orders o ON o.cid = c.id\"", + "join 5:1 to 5:30 \"JOIN orders o ON o.cid = c.id\"", + "column 1:8 name=id in=c to 1:12 \"c.id\"", + "table 2:6 name=__ref(\n \"customers\"\n) to 4:2 \"__ref(\\n \\\"customers\\\"\\n)\"", + "table 5:6 name=orders to 5:12 \"orders\"", + "eq 5:18 to 5:30 \"o.cid = c.id\"", + "column 5:18 name=cid in=o to 5:23 \"o.cid\"", + "column 5:26 name=id in=c to 5:30 \"c.id\"", + ], + }, + TreeFactsTestCase { + description: "a Snowflake table function marker read inside TABLE( ) covers only its source call", + dialect: "snowflake", + sql: "SELECT l.k\nFROM __table_fn(\"lines_for\")('B-1', __ref(\"orders\").k) AS l JOIN t ON t.k = l.k", + expected_facts: &[ + "select 1:1 to 2:80 \"SELECT l.k\\nFROM __table_fn(\\\"lines_for\\\")('B-1', __ref(\\\"orders\\\").k) AS l JOIN t ON t.k = l.k\"", + "join 2:61 to 2:80 \"JOIN t ON t.k = l.k\"", + "column 1:8 name=k in=l to 1:11 \"l.k\"", + "alias 2:6 name=l to 2:60 \"__table_fn(\\\"lines_for\\\")('B-1', __ref(\\\"orders\\\").k) AS l\"", + "function 2:6", + "function 2:6 to 2:55 \"__table_fn(\\\"lines_for\\\")('B-1', __ref(\\\"orders\\\").k)\"", + "literal 2:6 string=B-1", + "column 2:37 name=k in=__ref(\"orders\") to 2:54 \"__ref(\\\"orders\\\").k\"", + "table 2:66 name=t to 2:67 \"t\"", + "eq 2:71 to 2:80 \"t.k = l.k\"", + "column 2:71 name=k in=t to 2:74 \"t.k\"", + "column 2:77 name=k in=l to 2:80 \"l.k\"", + ], + }, + TreeFactsTestCase { + description: "a CTE is named; non-ASCII text keeps character offsets; a two-name marker is its call", + dialect: "snowflake", + sql: "WITH \"Top\" AS (SELECT 'café' AS n FROM __dbt_ref(\"shop\", \"raw\"))\nSELECT x.n FROM \"Top\" x", + expected_facts: &[ + "select 1:1 to 2:24 \"WITH \\\"Top\\\" AS (SELECT 'café' AS n FROM __dbt_ref(\\\"shop\\\", \\\"raw\\\"))\\nSELECT x.n FROM \\\"Top\\\" x\"", + "cte 1:6 name=Top quoted to 1:65 \"\\\"Top\\\" AS (SELECT 'café' AS n FROM __dbt_ref(\\\"shop\\\", \\\"raw\\\"))\"", + "column 2:8 name=n in=x to 2:11 \"x.n\"", + "table 2:17 name=Top quoted to 2:22 \"\\\"Top\\\"\"", + "select 1:16 to 1:64 \"SELECT 'café' AS n FROM __dbt_ref(\\\"shop\\\", \\\"raw\\\")\"", + "alias 1:23 name=n to 1:34 \"'café' AS n\"", + "literal 1:23 string=café", + "table 1:40 name=__dbt_ref(\"shop\", \"raw\") to 1:64 \"__dbt_ref(\\\"shop\\\", \\\"raw\\\")\"", + ], + }, + ]; + for test_case in test_cases { + let nodes = sql_tree_nodes(test_case.sql, test_case.dialect); + assert_eq!( + nodes.as_deref().map(|nodes| facts(test_case.sql, nodes)), + Ok(test_case + .expected_facts + .iter() + .map(|fact| (*fact).to_owned()) + .collect()), + "{}", + test_case.description + ); + } +} + #[test] fn given_sql_without_a_full_tree_when_building_a_rule_tree_then_the_reason_is_reported() { let test_cases = [ diff --git a/crates/sqlbuild-analysis/src/sql_tree/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_tree/tests/test_types.rs index ae152e0f0e..c328f404aa 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/tests/test_types.rs @@ -17,3 +17,11 @@ pub(crate) struct TreeFailureTestCase { pub(crate) sql: String, pub(crate) expected_error: fn(&SqlTreeError) -> bool, } + +/// SQL, its dialect, and each node's facts as `helpers::facts` writes them. +pub(crate) struct TreeFactsTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_facts: &'static [&'static str], +} diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs index 2d6cef8b39..9350081971 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs @@ -8,7 +8,9 @@ use pyo3::types::PyTuple; use pyo3::{IntoPyObject, pyclass, pymethods}; use sqlbuild_analysis::sql_tree::errors::SqlTreeError; use sqlbuild_analysis::sql_tree::main::sql_tree_nodes::sql_tree_nodes; -use sqlbuild_analysis::sql_tree::models::SqlTreeNode as TreeNode; +use sqlbuild_analysis::sql_tree::models::{ + NodeExtent, NodeLiteral, NodeName, SqlTreeNode as TreeNode, +}; use sqlbuild_python_base::records::main::construct::new_annotated_error::new_annotated_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -57,6 +59,56 @@ impl SqlTreeNode { .map_or(1, |node| node.column) } + /// The name an identifier, column, alias, table or CTE carries, unquoted; a relation marker's is its call text. + #[getter] + fn name(&self) -> Option<&str> { + self.node_name().map(|name| name.name.as_str()) + } + + /// Whether the source quoted the name; `None` without one. + #[getter] + fn quoted(&self) -> Option { + self.node_name().map(|name| name.quoted) + } + + /// The names qualifying the name, outermost first, such as `("db", "sales")`; `None` without a name. + #[getter] + fn qualifier<'py>(&self, py: Python<'py>) -> PyResult>> { + self.node_name() + .map(|name| PyTuple::new(py, &name.qualifier)) + .transpose() + } + + /// A literal's value as written, without quotes. + #[getter] + fn value(&self) -> Option<&str> { + self.node_literal().map(|literal| literal.value.as_str()) + } + + /// A literal's snake_case type, such as `string`, `number` or `date`. + #[getter] + fn literal_type(&self) -> Option<&'static str> { + self.node_literal().map(|literal| literal.literal_type) + } + + /// The 1-based line just after the node's last character; `None` when the parser recorded no span. + #[getter] + fn end_line(&self) -> Option { + self.extent().map(|extent| extent.end_line) + } + + /// The 1-based column just after the node's last character; `None` when the parser recorded no span. + #[getter] + fn end_column(&self) -> Option { + self.extent().map(|extent| extent.end_column) + } + + /// The node's `(start, end)` character offsets into the document's source, end exclusive. + #[getter] + fn span(&self) -> Option<(usize, usize)> { + self.extent().map(|extent| extent.span) + } + /// The node's direct children, in SQLQuack's traversal order. #[getter] fn children<'py>(&self, py: Python<'py>) -> PyResult> { @@ -88,6 +140,25 @@ impl SqlTreeNode { } } +impl SqlTreeNode { + /// The node's tree record; `None` for the document root. + fn record(&self) -> Option<&TreeNode> { + self.tree.nodes.get(self.index) + } + + fn node_name(&self) -> Option<&NodeName> { + self.record().and_then(|node| node.name.as_ref()) + } + + fn node_literal(&self) -> Option<&NodeLiteral> { + self.record().and_then(|node| node.literal.as_ref()) + } + + fn extent(&self) -> Option<&NodeExtent> { + self.record().and_then(|node| node.extent.as_ref()) + } +} + /// One SQL text a custom rule reads, parsed on first use by its dialect's SQLQuack parser. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct SqlDocument { diff --git a/fensu.toml b/fensu.toml index 5b730b140a..55e33573b9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -3852,3 +3852,7 @@ reason = "Each builds its own dbt manifest record type with that type's fields; [[dupes.allowlist]] paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs"] reason = "Each is its own Python entry point with its own keyword contract and dbt command shape; both end in the shared argv builder." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs"] +reason = "source_start and source_end apply opposite boundary rules: a start inside a rewrite maps to the start of the replaced text, while an exclusive end inside or closing one maps to its end, and an end at a removal maps to where the removed text began. Only the partition-and-shift shape is alike." diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md index c81e21ff98..b855a669d6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md @@ -78,18 +78,36 @@ if enum_types.get("fulfillment_method") != "fulfillment_method": ## Syntax tree -Use the syntax tree only when the common structures are insufficient: +Use the syntax tree only when the common structures are insufficient. Its nodes carry the names, +literal values and source spans a rule reads: ```python -for node in sql.expanded.ast().walk(): - if node.kind == "window_function": - ... +sql = ctx.sql.for_model(model).expanded +for node in sql.ast().walk(): + if node.kind == "alias" and node.quoted and node.name != node.name.lower(): + start, end = node.span + return [ctx.finding(subject=model, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a `kind`, a `line` and `column` where it starts, `children`, and `walk()`, which yields the node and its descendants in pre-order. Kinds are SQLQuack's snake_case names, such as `select`, `subquery`, -`star`, `join`, `cte`, `column`, `function` and `window_function`. +`star`, `join`, `cte`, `alias`, `column`, `table`, `literal`, `function` and `window_function`. + +Nodes also have these attributes, which are `None` on nodes they don't apply to: + +| Attribute | Nodes | Value | +| --- | --- | --- | +| `name` | `identifier`, `column`, `alias`, `table`, `cte` | The name without quotes, such as `order_id` | +| `quoted` | The same | Whether the source quotes the name | +| `qualifier` | The same | The names before it, outermost first: `("db", "sales")` for `db.sales.orders` | +| `value` | `literal` | The value as written, without quotes: `it's` for `'it''s'` | +| `literal_type` | `literal` | `string`, `number`, `national_string`, `date`, `timestamp` and the other literal types | +| `end_line`, `end_column` | Nodes with a source span | Where the node ends, just after its last character | +| `span` | Nodes with a source span | `(start, end)` character offsets into `source`, end exclusive | + +A relation marker, such as `__ref("orders")`, reads as a `table` node whose `name` and `span` are +the marker call as written. Parsing is lazy. A path or contract Rule does not pay for SQL parsing merely because syntax tree access exists elsewhere. @@ -102,8 +120,9 @@ The tree comes from the SQLQuack parser SQLBuild uses for the project's dialect: | DuckDB, MotherDuck, Snowflake, SQLite | SQLQuack's shared parser | | Other dialects | Unavailable | -For the shared parser, a node the parser records no position for, such as a bare `*` or the query -inside a subquery, reports the position of its parent. +A node without a source span has no end: `end_line`, `end_column` and `span` are `None`, and it +reports its parent's start. Literals have no span. Under the shared parser, neither does the query +inside a subquery. Reading the tree, or any common structure, raises `RuleUsageError` when: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index da5760b90e..798647548d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4335,6 +4335,14 @@ class SqlTreeNode: kind: str line: int column: int + name: str | None + quoted: bool | None + qualifier: tuple[str, ...] | None + value: str | None + literal_type: str | None + end_line: int | None + end_column: int | None + span: tuple[int, int] | None children: tuple[SqlTreeNode, ...] def walk(self) -> list[SqlTreeNode]: ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py index 3dd38677c8..309675d9c4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py @@ -47,6 +47,21 @@ " ]\n" ), ) +_FACTS_RULE: str = custom_rule_source( + code="XSQBRTREE002", + body=( + " sql = ctx.sql.for_model(model).expanded\n" + " return [\n" + " ctx.finding(\n" + " subject=model,\n" + ' message=f"{node.kind}:{node.qualifier or ()}:{node.name or node.value}:"\n' + ' + (sql.source[node.span[0] : node.span[1]] if node.span else "-"),\n' + " )\n" + " for node in sql.ast().walk()\n" + ' if node.kind in ("alias", "column", "literal", "table")\n' + " ]\n" + ), +) @pytest.mark.parametrize( @@ -180,3 +195,59 @@ def test_given_custom_rule_over_the_syntax_tree_when_compiling_then_nodes_point_ assert tuple(sorted(item["message"] for item in custom_rule_diagnostics(result))) == ( test_case.expected_messages ), result.stdout + result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + SyntaxTreeRuleE2ETestCase( + description="a rule reads names, literal values and source spans, a marker as its call", + adapter="snowflake", + model_sql=( + 'MODEL (description "Orders", database analytics, schema analytics);\n' + "SELECT o.\"Status\" AS status_label, 'it''s' AS note\n" + 'FROM __ref("raw_orders") AS o\n' + "WHERE o.region = 'EU'\n" + ), + other_models={ + "raw_orders": ( + 'MODEL (description "raw_orders", database analytics, schema analytics);\n' + "SELECT 'open' AS \"Status\", 'EU' AS region\n" + ) + }, + expected_messages=( + "alias:():Status:'open' AS \"Status\"", + "alias:():note:'it''s' AS note", + "alias:():region:'EU' AS region", + 'alias:():status_label:o."Status" AS status_label', + "column:('o',):Status:o.\"Status\"", + "column:('o',):region:o.region", + "literal:():EU:-", + "literal:():EU:-", + "literal:():it's:-", + "literal:():open:-", + 'table:():__ref("raw_orders"):__ref("raw_orders")', + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_custom_rule_reading_node_facts_when_compiling_then_it_sees_the_authored_text( + test_case: SyntaxTreeRuleE2ETestCase, tmp_path: Path +) -> None: + write_custom_rule_project( + project_dir=tmp_path, + selected_rules=("XSQBRTREE002",), + files=( + ("rules/facts.py", _FACTS_RULE), + *((f"models/{name}.sql", sql) for name, sql in test_case.other_models.items()), + ), + adapter=test_case.adapter, + ) + (tmp_path / "models" / "orders.sql").write_text(test_case.model_sql, encoding="utf-8") + + result: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + assert tuple(sorted(item["message"] for item in custom_rule_diagnostics(result))) == ( + test_case.expected_messages + ), result.stdout + result.stderr diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx index 72215349ce..e715257b94 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx @@ -66,18 +66,36 @@ if enum_types.get("fulfillment_method") != "fulfillment_method": ## Syntax tree -Use the syntax tree only when the common structures are insufficient: +Use the syntax tree only when the common structures are insufficient. Its nodes carry the names, +literal values and source spans a rule reads: ```python -for node in sql.expanded.ast().walk(): - if node.kind == "window_function": - ... +sql = ctx.sql.for_model(model).expanded +for node in sql.ast().walk(): + if node.kind == "alias" and node.quoted and node.name != node.name.lower(): + start, end = node.span + return [ctx.finding(subject=model, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a `kind`, a `line` and `column` where it starts, `children`, and `walk()`, which yields the node and its descendants in pre-order. Kinds are SQLQuack's snake_case names, such as `select`, `subquery`, -`star`, `join`, `cte`, `column`, `function` and `window_function`. +`star`, `join`, `cte`, `alias`, `column`, `table`, `literal`, `function` and `window_function`. + +Nodes also have these attributes, which are `None` on nodes they don't apply to: + +| Attribute | Nodes | Value | +| --- | --- | --- | +| `name` | `identifier`, `column`, `alias`, `table`, `cte` | The name without quotes, such as `order_id` | +| `quoted` | The same | Whether the source quotes the name | +| `qualifier` | The same | The names before it, outermost first: `("db", "sales")` for `db.sales.orders` | +| `value` | `literal` | The value as written, without quotes: `it's` for `'it''s'` | +| `literal_type` | `literal` | `string`, `number`, `national_string`, `date`, `timestamp` and the other literal types | +| `end_line`, `end_column` | Nodes with a source span | Where the node ends, just after its last character | +| `span` | Nodes with a source span | `(start, end)` character offsets into `source`, end exclusive | + +A relation marker, such as `__ref("orders")`, reads as a `table` node whose `name` and `span` are +the marker call as written. Parsing is lazy. A path or contract Rule does not pay for SQL parsing merely because syntax tree access exists elsewhere. @@ -90,8 +108,9 @@ The tree comes from the SQLQuack parser SQLBuild uses for the project's dialect: | DuckDB, MotherDuck, Snowflake, SQLite | SQLQuack's shared parser | | Other dialects | Unavailable | -For the shared parser, a node the parser records no position for, such as a bare `*` or the query -inside a subquery, reports the position of its parent. +A node without a source span has no end: `end_line`, `end_column` and `span` are `None`, and it +reports its parent's start. Literals have no span. Under the shared parser, neither does the query +inside a subquery. Reading the tree, or any common structure, raises `RuleUsageError` when: