From 8b95dc5bc6188b13f6aeec887f7741617e9da15d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:41:53 +0000 Subject: [PATCH 1/7] feat(rules): give syntax tree nodes names, literal values and source extents Rules can read a node's name, quoting and qualifier, a literal's value and type, and where a node ends (end_line, end_column and a character span into the document source). A relation marker reads as the call it stands for, and offsets map back through marker names and Snowflake TABLE( ) wrappers to the text as written. Nodes the parser records no span for have no end. --- .../duckshop/project/rules/quoted_aliases.py | 36 ++++++ .../project/tests/test_rule_quoted_aliases.py | 50 ++++++++ .../main/located_markers_as_names.rs | 63 ++++++++++ .../relation_markers/main/markers_as_names.rs | 50 +------- .../src/relation_markers/main/mod.rs | 1 + .../src/relation_markers/types.rs | 5 + .../src/sql_tree/_helpers/clauses.rs | 6 +- .../src/sql_tree/_helpers/mod.rs | 4 +- .../src/sql_tree/_helpers/node_facts.rs | 100 +++++++++++++++ .../src/sql_tree/_helpers/source_map.rs | 119 ++++++++++++++++++ .../src/sql_tree/_helpers/table_wrapped.rs | 53 ++++++-- .../src/sql_tree/_helpers/walk.rs | 47 ++++--- .../src/sql_tree/main/sql_tree_nodes.rs | 21 ++-- .../sqlbuild-analysis/src/sql_tree/models.rs | 39 +++++- .../src/sql_tree/tests/helpers.rs | 44 ++++++- .../src/sql_tree/tests/test_sql_tree.rs | 111 +++++++++++++++- .../src/sql_tree/tests/test_types.rs | 8 ++ .../src/rules/_helpers/sql/sql_document.rs | 73 ++++++++++- .../rules/custom-rules/rule-context.md | 33 +++-- src/sqlbuild/_native.pyi | 8 ++ .../main/rules/test_rule_positions.py | 71 +++++++++++ .../rules/custom-rules/rule-context.mdx | 33 +++-- 22 files changed, 870 insertions(+), 105 deletions(-) create mode 100644 benchmarks/duckshop/project/rules/quoted_aliases.py create mode 100644 benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py create mode 100644 crates/sqlbuild-analysis/src/relation_markers/main/located_markers_as_names.rs create mode 100644 crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs create mode 100644 crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs diff --git a/benchmarks/duckshop/project/rules/quoted_aliases.py b/benchmarks/duckshop/project/rules/quoted_aliases.py new file mode 100644 index 0000000000..292cf6ee24 --- /dev/null +++ b/benchmarks/duckshop/project/rules/quoted_aliases.py @@ -0,0 +1,36 @@ +"""DuckShop house rule that keeps case-sensitive output names inside partner export models.""" + +from sqlbuild.rules import Finding, Model, RuleContext, rule + +EXPORT_SUFFIX = "_export" + + +@rule( + code="XSQBRDUCK003", + message="Only partner export models may give columns quoted mixed-case names", + remediation=( + "Name the column in unquoted snake_case, or move the projection into a model whose name " + "ends in _export, which writes a partner's own feed format." + ), +) +def mixed_case_names_stay_in_exports(*, model: Model, ctx: RuleContext) -> list[Finding]: + if model.path.stem.endswith(EXPORT_SUFFIX): + return [] + sql = ctx.sql.for_model(model).expanded + if '"' not in sql.source: + return [] + findings: list[Finding] = [] + for node in sql.ast().walk(): + if node.kind != "alias" or not node.quoted or node.name is None: + continue + if node.name == node.name.lower(): + continue + text = sql.source[node.span[0] : node.span[1]] if node.span else node.name + findings.append( + ctx.finding( + subject=model, + node=node, + message=f"Quoted mixed-case name {node.name!r} in `{text}`", + ) + ) + return findings diff --git a/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py new file mode 100644 index 0000000000..722af47d38 --- /dev/null +++ b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py @@ -0,0 +1,50 @@ +from rules.quoted_aliases import mixed_case_names_stay_in_exports + +from sqlbuild.rules import RuleCase, RuleFile, evaluate_rule + +_HEADER = 'MODEL (description "Listings");\n' +_MIXED_CASE = ( + _HEADER + + "SELECT\n" + + " l.listing_id AS \"listingId\",\n" + + " l.note AS \"note\",\n" + + " 'isLive' AS status_label\n" + + 'FROM __ref("dim_listings") AS l\n' +) +_LISTINGS = RuleFile( + path="models/marts/catalogue/dim_listings.sql", + source=_HEADER + "SELECT 1 AS listing_id, 'new' AS note\n", +) + + +def test_given_quoted_mixed_case_alias_outside_an_export_when_evaluating_then_reports_it() -> None: + result = evaluate_rule( + rule=mixed_case_names_stay_in_exports, + test_case=RuleCase( + description="a mart names a column in a partner's case", + source=_MIXED_CASE, + path="models/marts/catalogue/rpt_listings.sql", + files=(_LISTINGS,), + expected_finding_count=1, + ), + ) + + assert result.finding_count == 1 + assert result.findings[0].message == ( + "Quoted mixed-case name 'listingId' in `l.listing_id AS \"listingId\"`" + ) + + +def test_given_quoted_mixed_case_alias_in_an_export_when_evaluating_then_passes() -> None: + result = evaluate_rule( + rule=mixed_case_names_stay_in_exports, + test_case=RuleCase( + description="a partner export writes the partner's own names", + source=_MIXED_CASE, + path="models/marts/edge_cases/rpt_edge_partner_listing_export.sql", + files=(_LISTINGS,), + expected_finding_count=0, + ), + ) + + assert result.finding_count == 0 diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/located_markers_as_names.rs b/crates/sqlbuild-analysis/src/relation_markers/main/located_markers_as_names.rs new file mode 100644 index 0000000000..c96f88e66d --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/main/located_markers_as_names.rs @@ -0,0 +1,63 @@ +//! `located_markers_as_names`: relation marker calls in code as plain names that keep every position, with where each call and name stand. + +use regex::Regex; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::relation_markers::_helpers::pattern::relation_marker_pattern; +use crate::relation_markers::constants::{DBT_REF_FUNCTION, MARKER_NAME_PREFIX}; +use crate::relation_markers::types::{MarkerCall, MarkerRanges}; + +/// `sql` with each marker call in code as `__sqb_`, padded so later text keeps its line and column, the calls by index, and each call's and padded name's byte ranges. +pub fn located_markers_as_names( + sql: &str, + syntax: &LexicalSyntax, +) -> Result<(String, Vec, Vec), String> { + let pattern: &Regex = relation_marker_pattern()?; + let mut protected: Option> = None; + let mut named: String = String::with_capacity(sql.len()); + let mut calls: Vec = Vec::new(); + let mut located: Vec = Vec::new(); + let mut cursor: usize = 0; + for captures in pattern.captures_iter(sql) { + let (Some(full), Some(function), Some(first)) = + (captures.get(0), captures.get(1), captures.get(2)) + else { + continue; + }; + let ranges = protected.get_or_insert_with(|| dialect_non_code_ranges(sql, syntax)); + let following: usize = ranges.partition_point(|(start, _)| *start <= full.start()); + if following > 0 && full.start() < ranges[following - 1].1 { + continue; + } + let function_name: String = function.as_str().to_ascii_lowercase(); + let referenced_name: String = match (captures.get(3), function_name == DBT_REF_FUNCTION) { + (None, _) => first.as_str().to_owned(), + (Some(second), true) => format!("{}__{}", first.as_str(), second.as_str()), + (Some(_), false) => continue, + }; + named.push_str(&sql[cursor..full.start()]); + let name_start: usize = named.len(); + named.push_str(&padded_name( + &format!("{MARKER_NAME_PREFIX}{}", calls.len()), + full.as_str(), + )); + cursor = full.end(); + calls.push((function_name, referenced_name)); + located.push((full.range(), name_start..named.len())); + } + named.push_str(&sql[cursor..]); + Ok((named, calls, located)) +} + +/// `name`, then the call's line breaks, then spaces as wide as the call's last line. +fn padded_name(name: &str, call: &str) -> String { + let breaks: usize = call.matches('\n').count(); + let last_line: &str = call.rsplit('\n').next().unwrap_or(call); + let width: usize = if breaks == 0 { + call.chars().count().saturating_sub(name.chars().count()) + } else { + last_line.chars().count() + }; + format!("{name}{}{}", "\n".repeat(breaks), " ".repeat(width)) +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs b/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs index 9330c94939..c3df988592 100644 --- a/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs +++ b/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs @@ -1,11 +1,8 @@ //! `markers_as_names`: relation marker calls in code as plain names that keep every position. -use regex::Regex; -use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::relation_markers::_helpers::pattern::relation_marker_pattern; -use crate::relation_markers::constants::{DBT_REF_FUNCTION, MARKER_NAME_PREFIX}; +use crate::relation_markers::main::located_markers_as_names::located_markers_as_names; use crate::relation_markers::types::MarkerCall; /// `sql` with each marker call in code as `__sqb_`, padded so later text keeps its line and column, and the calls by index. @@ -13,48 +10,5 @@ pub fn markers_as_names( sql: &str, syntax: &LexicalSyntax, ) -> Result<(String, Vec), String> { - let pattern: &Regex = relation_marker_pattern()?; - let mut protected: Option> = None; - let mut named: String = String::with_capacity(sql.len()); - let mut calls: Vec = Vec::new(); - let mut cursor: usize = 0; - for captures in pattern.captures_iter(sql) { - let (Some(full), Some(function), Some(first)) = - (captures.get(0), captures.get(1), captures.get(2)) - else { - continue; - }; - let ranges = protected.get_or_insert_with(|| dialect_non_code_ranges(sql, syntax)); - let following: usize = ranges.partition_point(|(start, _)| *start <= full.start()); - if following > 0 && full.start() < ranges[following - 1].1 { - continue; - } - let function_name: String = function.as_str().to_ascii_lowercase(); - let referenced_name: String = match (captures.get(3), function_name == DBT_REF_FUNCTION) { - (None, _) => first.as_str().to_owned(), - (Some(second), true) => format!("{}__{}", first.as_str(), second.as_str()), - (Some(_), false) => continue, - }; - named.push_str(&sql[cursor..full.start()]); - named.push_str(&padded_name( - &format!("{MARKER_NAME_PREFIX}{}", calls.len()), - full.as_str(), - )); - cursor = full.end(); - calls.push((function_name, referenced_name)); - } - named.push_str(&sql[cursor..]); - Ok((named, calls)) -} - -/// `name`, then the call's line breaks, then spaces as wide as the call's last line. -fn padded_name(name: &str, call: &str) -> String { - let breaks: usize = call.matches('\n').count(); - let last_line: &str = call.rsplit('\n').next().unwrap_or(call); - let width: usize = if breaks == 0 { - call.chars().count().saturating_sub(name.chars().count()) - } else { - last_line.chars().count() - }; - format!("{name}{}{}", "\n".repeat(breaks), " ".repeat(width)) + located_markers_as_names(sql, syntax).map(|(named, calls, _)| (named, calls)) } diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs b/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs index abc0f2dc58..35bbba4636 100644 --- a/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs +++ b/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs @@ -1,2 +1,3 @@ pub mod lateral_start; +pub mod located_markers_as_names; pub mod markers_as_names; diff --git a/crates/sqlbuild-analysis/src/relation_markers/types.rs b/crates/sqlbuild-analysis/src/relation_markers/types.rs index 9b90a22d2a..13dab5ded5 100644 --- a/crates/sqlbuild-analysis/src/relation_markers/types.rs +++ b/crates/sqlbuild-analysis/src/relation_markers/types.rs @@ -1,4 +1,9 @@ //! Shapes of relation marker calls. +use std::ops::Range; + /// A marker call's lower-case function, such as `__ref`, and the name it references. pub type MarkerCall = (String, String); + +/// A marker call's byte range in the source, and the byte range of the padded name that stands for it. +pub type MarkerRanges = (Range, Range); diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/clauses.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/clauses.rs index 9fa047167c..e8a0712125 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/clauses.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/clauses.rs @@ -1,15 +1,17 @@ //! Join and CTE clauses, which SQLQuack keeps as statement fields rather than expressions. use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::names::models::Identifier; use sqlquack_syntax::expressions::queries::models::{Cte, Join, JoinKind, With}; use sqlquack_syntax::tokens::models::Span; use crate::sql_tree::constants::{CTE_KIND, JOIN_KIND}; -/// One clause of a statement: its kind, its span, and the expressions it owns. +/// One clause of a statement: its kind, its span, its name, and the expressions it owns. pub(crate) struct Clause<'ast> { pub(crate) kind: &'static str, pub(crate) span: Option, + pub(crate) name: Option<&'ast Identifier>, pub(crate) children: Vec<&'ast Expression>, } @@ -45,6 +47,7 @@ fn cte_clause(cte: &Cte) -> Clause<'_> { Clause { kind: CTE_KIND, span: cte.span.0, + name: Some(&cte.alias), children: vec![&cte.this], } } @@ -57,6 +60,7 @@ fn join_clause(join: &Join) -> Clause<'_> { Clause { kind: JOIN_KIND, span: join.span.0, + name: None, children, } } diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs index 942bd41b25..66e6f6f7d7 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs @@ -1,8 +1,10 @@ -//! Parser selection, parsing, the tree walk, clause nodes, source positions and table wrapping. +//! Parser selection, parsing, the tree walk, clause nodes, node facts, source positions and table wrapping. pub(crate) mod clauses; +pub(crate) mod node_facts; pub(crate) mod parse; pub(crate) mod positions; pub(crate) mod rules_tree_parser; +pub(crate) mod source_map; pub(crate) mod table_wrapped; pub(crate) mod walk; diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs new file mode 100644 index 0000000000..43d629debb --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs @@ -0,0 +1,100 @@ +//! What a node names or holds: identifier, column, alias, table and CTE names, and literal values. + +use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::names::models::{Identifier, Literal, TableRef}; +use sqlquack_syntax::tokens::models::Span; + +use crate::relation_markers::constants::MARKER_NAME_PREFIX; +use crate::sql_tree::_helpers::source_map::SourceMap; +use crate::sql_tree::models::{NodeLiteral, NodeName}; + +/// The name `expression` carries, its qualifier outermost first; a relation marker's name is its call. +pub(crate) fn node_name(expression: &Expression, map: &SourceMap<'_>) -> Option { + let (name, qualifier): (&Identifier, Vec<&Identifier>) = match expression { + Expression::Identifier(identifier) => (identifier, Vec::new()), + Expression::Column(column) => (&column.name, column.table.iter().collect()), + Expression::Alias(alias) => (&alias.alias, Vec::new()), + Expression::Table(table) => ( + &table.name, + table.catalog.iter().chain(table.schema.iter()).collect(), + ), + _ => return None, + }; + Some(named(name, &qualifier, map)) +} + +/// `name` and its qualifier as written, marker names read back as their calls. +pub(crate) fn named(name: &Identifier, qualifier: &[&Identifier], map: &SourceMap<'_>) -> NodeName { + let (text, quoted) = written(name, map); + NodeName { + name: text, + quoted, + qualifier: qualifier.iter().map(|part| written(part, map).0).collect(), + } +} + +/// The value and snake_case type of a literal node. +pub(crate) fn node_literal(expression: &Expression) -> Option { + let Expression::Literal(literal) = expression else { + return None; + }; + Some(NodeLiteral { + value: literal.value_str().to_owned(), + literal_type: literal_type(literal), + }) +} + +/// A table's source range: from its outermost qualifier to the end of its name, without its alias. +pub(crate) fn table_span(table: &TableRef) -> Option { + let end: Span = table.name.span?; + let first: &Identifier = table + .catalog + .as_ref() + .or(table.schema.as_ref()) + .unwrap_or(&table.name); + let qualified = [table.catalog.as_ref(), table.schema.as_ref()]; + if qualified.iter().flatten().any(|part| part.span.is_none()) { + return None; + } + first.span.map(|start| start.through(end)) +} + +/// An identifier's text and quoting; a marker name reads as the call it stands for, unquoted. +fn written(identifier: &Identifier, map: &SourceMap<'_>) -> (String, bool) { + match marker_call(identifier, map) { + Some(call) => (call.to_owned(), false), + None => (identifier.name.clone(), identifier.quoted), + } +} + +/// The call an unquoted marker name stands for. +fn marker_call<'source>(identifier: &Identifier, map: &SourceMap<'source>) -> Option<&'source str> { + if identifier.quoted { + return None; + } + let index: &str = identifier.name.strip_prefix(MARKER_NAME_PREFIX)?; + match index.parse::() { + Ok(index) => map.calls.get(index).copied(), + Err(_not_a_marker) => None, + } +} + +fn literal_type(literal: &Literal) -> &'static str { + match literal { + Literal::String(_) => "string", + Literal::Number(_) => "number", + Literal::HexString(_) => "hex_string", + Literal::HexNumber(_) => "hex_number", + Literal::BitString(_) => "bit_string", + Literal::ByteString(_) => "byte_string", + Literal::NationalString(_) => "national_string", + Literal::Date(_) => "date", + Literal::Time(_) => "time", + Literal::Timestamp(_) => "timestamp", + Literal::Datetime(_) => "datetime", + Literal::TripleQuotedString(_, _) => "triple_quoted_string", + Literal::EscapeString(_) => "escape_string", + Literal::DollarString(_) => "dollar_string", + Literal::RawString(_) => "raw_string", + } +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs new file mode 100644 index 0000000000..171852bbcd --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs @@ -0,0 +1,119 @@ +//! Spans in the parsed text read back as positions in the rule's source, across marker names and `TABLE( )` wrappers. + +use std::ops::Range; + +use sqlquack_syntax::tokens::models::Span; + +use crate::relation_markers::types::MarkerRanges; +use crate::sql_tree::_helpers::positions::{line_column, line_starts}; +use crate::sql_tree::models::NodeExtent; + +/// The parsed text's `parsed` characters stand where the earlier text's `source` characters were. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Rewrite { + pub(crate) parsed: Range, + pub(crate) source: Range, +} + +/// The rule's source as the parser's text maps back to it. +pub(crate) struct SourceMap<'source> { + /// Each marker call's source text, by marker index. + pub(crate) calls: Vec<&'source str>, + line_starts: Vec, + /// Marker names over their calls, in order: named text to source. + markers: Vec, + /// `TABLE( )` wrappers, in order: parsed text to named text. + wrappers: Vec, +} + +impl<'source> SourceMap<'source> { + /// The map of `source`, whose marker calls `named` holds as names at `ranges`, parsed after `wrappers`. + pub(crate) fn new( + source: &'source str, + named: &str, + ranges: &[MarkerRanges], + wrappers: Vec, + ) -> Self { + let calls: Vec> = char_ranges(source, ranges.iter().map(|(call, _)| call)); + let names: Vec> = char_ranges(named, ranges.iter().map(|(_, name)| name)); + Self { + calls: ranges + .iter() + .map(|(call, _)| &source[call.clone()]) + .collect(), + line_starts: line_starts(source), + markers: names + .into_iter() + .zip(calls) + .map(|(parsed, source)| Rewrite { parsed, source }) + .collect(), + wrappers, + } + } + + /// The 1-based line and column of a parsed character offset where a node starts. + pub(crate) fn start(&self, offset: usize) -> (usize, usize) { + line_column( + &self.line_starts, + source_start(&self.markers, source_start(&self.wrappers, offset)), + ) + } + + /// The source extent of a parsed span: a node starting or ending inside a marker name covers its whole call. + pub(crate) fn extent(&self, span: Span) -> NodeExtent { + let start: usize = source_start(&self.markers, source_start(&self.wrappers, span.start)); + let end: usize = source_end(&self.markers, source_end(&self.wrappers, span.end)); + let (end_line, end_column) = line_column(&self.line_starts, end); + NodeExtent { + end_line, + end_column, + span: (start, end), + } + } +} + +/// The earlier text's offset for a start offset: inside a rewrite, the start of what it replaced. +fn source_start(rewrites: &[Rewrite], offset: usize) -> usize { + let after: usize = rewrites.partition_point(|rewrite| rewrite.parsed.end <= offset); + match rewrites.get(after) { + Some(rewrite) if rewrite.parsed.start <= offset => rewrite.source.start, + _ => shifted(rewrites[..after].last(), offset), + } +} + +/// The earlier text's offset for an exclusive end offset: inside or closing a rewrite, the end of what it replaced, and at a removal, where the removed text began. +fn source_end(rewrites: &[Rewrite], offset: usize) -> usize { + let after: usize = rewrites.partition_point(|rewrite| rewrite.parsed.end < offset); + match rewrites.get(after) { + Some(rewrite) if rewrite.parsed.start < offset => rewrite.source.end, + Some(rewrite) if rewrite.parsed.is_empty() && rewrite.parsed.start == offset => { + rewrite.source.start + } + _ => shifted(rewrites[..after].last(), offset), + } +} + +/// `offset` past the last rewrite before it, moved as that rewrite moved the text after it. +fn shifted(previous: Option<&Rewrite>, offset: usize) -> usize { + previous.map_or(offset, |rewrite| { + rewrite.source.end + (offset - rewrite.parsed.end) + }) +} + +/// Ascending byte `ranges` of `text` as character ranges, counted in one pass. +fn char_ranges<'a>( + text: &str, + ranges: impl Iterator>, +) -> Vec> { + let mut byte: usize = 0; + let mut chars: usize = 0; + ranges + .map(|range| { + chars += text[byte..range.start].chars().count(); + let start: usize = chars; + chars += text[range.clone()].chars().count(); + byte = range.end; + start..chars + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/table_wrapped.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/table_wrapped.rs index 6bb74987db..c8ca4b19e1 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/table_wrapped.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/table_wrapped.rs @@ -1,7 +1,5 @@ //! Snowflake table function markers wrapped in `TABLE( )`, as SQLBuild renders them. -use std::cmp::Reverse; - use sqlbuild_sqltext::sql_scan::constants::TABLE_WRAPPER; use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_paren; use sqlbuild_sqltext::sql_scan::main::follows_table_wrapper::follows_table_wrapper; @@ -9,11 +7,16 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::relation_markers::constants::{MARKER_NAME_PREFIX, TABLE_FUNCTION_MARKER}; use crate::relation_markers::types::MarkerCall; +use crate::sql_tree::_helpers::source_map::Rewrite; -/// `named` with each table function marker call in `TABLE( )`, written over padding so later text keeps its column. -pub(crate) fn table_wrapped(named: &str, calls: &[MarkerCall], syntax: &LexicalSyntax) -> String { +/// `named` with each table function marker call in `TABLE( )`, written over padding so later text keeps its column, and the rewrites that did it. +pub(crate) fn table_wrapped( + named: &str, + calls: &[MarkerCall], + syntax: &LexicalSyntax, +) -> (String, Vec) { let bytes = named.as_bytes(); - let mut edits: Vec<(usize, usize, String)> = Vec::new(); + let mut edits: Vec<(usize, usize, &str)> = Vec::new(); for (index, (function, _)) in calls.iter().enumerate() { if function != TABLE_FUNCTION_MARKER { continue; @@ -42,15 +45,41 @@ pub(crate) fn table_wrapped(named: &str, calls: &[MarkerCall], syntax: &LexicalS let replaced = usize::from(matches!(bytes.get(after), Some(b' ' | b'\t'))); let text_follows = !matches!(bytes.get(after), None | Some(b'\n' | b'\r')); let borrowed = spaces.min(TABLE_WRAPPER.len() + usize::from(text_follows) - replaced); - edits.push((start, name_end + borrowed, format!("{TABLE_WRAPPER}{name}"))); - edits.push((after, after + replaced, ")".to_owned())); + edits.push((start, start, TABLE_WRAPPER)); + edits.push((name_end, name_end + borrowed, "")); + edits.push((after, after, ")")); + edits.push((after, after + replaced, "")); } - edits.sort_by_key(|(start, _, _)| Reverse(*start)); - let mut wrapped = named.to_owned(); - for (start, end, replacement) in edits { - wrapped.replace_range(start..end, &replacement); + edits.sort_by_key(|(start, _, _)| *start); + applied(named, &edits) +} + +/// `named` with each ascending edit's byte range replaced by its text, and each edit as a character rewrite. +fn applied(named: &str, edits: &[(usize, usize, &str)]) -> (String, Vec) { + let mut wrapped: String = String::with_capacity(named.len() + edits.len()); + let mut rewrites: Vec = Vec::with_capacity(edits.len()); + let (mut cursor, mut named_chars, mut wrapped_chars) = (0, 0, 0); + for (start, end, text) in edits + .iter() + .filter(|(start, end, text)| start < end || !text.is_empty()) + { + let kept: &str = &named[cursor..*start]; + wrapped.push_str(kept); + named_chars += kept.chars().count(); + wrapped_chars += kept.chars().count(); + let removed: usize = named[*start..*end].chars().count(); + let added: usize = text.chars().count(); + rewrites.push(Rewrite { + parsed: wrapped_chars..wrapped_chars + added, + source: named_chars..named_chars + removed, + }); + wrapped.push_str(text); + named_chars += removed; + wrapped_chars += added; + cursor = *end; } - wrapped + wrapped.push_str(&named[cursor..]); + (wrapped, rewrites) } /// Where `name` first stands as a whole word in `bytes`. diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/walk.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/walk.rs index 2d59d234d8..bde0b75c17 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/walk.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/walk.rs @@ -6,7 +6,8 @@ use sqlquack_syntax::tokens::models::Span; use sqlquack_syntax::{Expression, ExpressionWalk}; use crate::sql_tree::_helpers::clauses::clauses; -use crate::sql_tree::_helpers::positions::line_column; +use crate::sql_tree::_helpers::node_facts::{named, node_literal, node_name, table_span}; +use crate::sql_tree::_helpers::source_map::SourceMap; use crate::sql_tree::constants::OPAQUE_KINDS; use crate::sql_tree::errors::SqlTreeError; use crate::sql_tree::models::SqlTreeNode; @@ -17,7 +18,7 @@ type Owners = HashMap<*const Expression, usize>; /// The nodes of `statements` in pre-order, walked with an explicit stack so depth cannot overflow. pub(crate) fn walk( statements: &[Expression], - starts: &[usize], + map: &SourceMap<'_>, ) -> Result, SqlTreeError> { let mut nodes: Vec = Vec::new(); let mut stack: Vec<(&Expression, Option)> = statements @@ -27,8 +28,8 @@ pub(crate) fn walk( .collect(); while let Some((expression, parent)) = stack.pop() { let index: usize = nodes.len(); - let node: SqlTreeNode = expression_node(&nodes, expression, parent, starts)?; - let (clause_nodes, owners) = clause_nodes(expression, &node, index, starts); + let node: SqlTreeNode = expression_node(&nodes, expression, parent, map)?; + let (clause_nodes, owners) = clause_nodes(expression, &node, index, map); nodes.push(node); nodes.extend(clause_nodes); for child in expression.children().into_iter().rev() { @@ -42,12 +43,12 @@ pub(crate) fn walk( Ok(nodes) } -/// `expression`'s node, at its own start or else at its parent's. +/// `expression`'s node, at its own start or else at its parent's, with its end only where the parser recorded one. fn expression_node( nodes: &[SqlTreeNode], expression: &Expression, parent: Option, - starts: &[usize], + map: &SourceMap<'_>, ) -> Result { let kind: &'static str = expression.variant_name(); if OPAQUE_KINDS.contains(&kind) { @@ -56,13 +57,18 @@ fn expression_node( )); } let inherited = parent.map_or((1, 1), |parent| (nodes[parent].line, nodes[parent].column)); - let (line, column) = - own_span(expression).map_or(inherited, |span| line_column(starts, span.start)); + let span: Option = recorded_span(expression); + let (line, column) = span + .or_else(|| qualified_star_start(expression)) + .map_or(inherited, |span| map.start(span.start)); Ok(SqlTreeNode { kind, line, column, parent, + extent: span.map(|span| map.extent(span)), + name: node_name(expression, map), + literal: node_literal(expression), }) } @@ -71,20 +77,23 @@ fn clause_nodes( expression: &Expression, node: &SqlTreeNode, index: usize, - starts: &[usize], + map: &SourceMap<'_>, ) -> (Vec, Owners) { let mut built: Vec = Vec::new(); let mut owners: Owners = HashMap::new(); for clause in clauses(expression) { let clause_index: usize = index + 1 + built.len(); - let (line, column) = clause.span.map_or((node.line, node.column), |span| { - line_column(starts, span.start) - }); + let (line, column) = clause + .span + .map_or((node.line, node.column), |span| map.start(span.start)); built.push(SqlTreeNode { kind: clause.kind, line, column, parent: Some(index), + extent: clause.span.map(|span| map.extent(span)), + name: clause.name.map(|name| named(name, &[], map)), + literal: None, }); for child in clause.children { let _ = owners.insert(std::ptr::from_ref(child), clause_index); @@ -93,10 +102,18 @@ fn clause_nodes( (built, owners) } -/// The node's span; a qualified star without one starts at its qualifier. -fn own_span(expression: &Expression) -> Option { +/// The node's recorded span; a table's runs from its first qualifier to its name. +fn recorded_span(expression: &Expression) -> Option { expression.source_span().or_else(|| match expression { - Expression::Star(star) => star.table.as_ref().and_then(|table| table.span), + Expression::Table(table) => table_span(table), _ => None, }) } + +/// Where a qualified star without a span starts: at its qualifier, which is not where it ends. +fn qualified_star_start(expression: &Expression) -> Option { + match expression { + Expression::Star(star) => star.table.as_ref().and_then(|table| table.span), + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/main/sql_tree_nodes.rs b/crates/sqlbuild-analysis/src/sql_tree/main/sql_tree_nodes.rs index c1a036e032..fed642126f 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/main/sql_tree_nodes.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/main/sql_tree_nodes.rs @@ -3,10 +3,10 @@ use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; use sqlquack_syntax::Expression; -use crate::relation_markers::main::markers_as_names::markers_as_names; +use crate::relation_markers::main::located_markers_as_names::located_markers_as_names; use crate::sql_tree::_helpers::parse::parse; -use crate::sql_tree::_helpers::positions::line_starts; use crate::sql_tree::_helpers::rules_tree_parser::rules_tree_parser; +use crate::sql_tree::_helpers::source_map::SourceMap; use crate::sql_tree::_helpers::table_wrapped::table_wrapped; use crate::sql_tree::_helpers::walk::walk; use crate::sql_tree::constants::{SNOWFLAKE_DIALECT, TREE_WORKER_STACK_BYTES}; @@ -14,7 +14,7 @@ use crate::sql_tree::errors::SqlTreeError; use crate::sql_tree::models::SqlTreeNode; use crate::worker_stack::main::run_on_worker_stack::run_on_worker_stack; -/// Every node of `sql` parsed in `dialect`, in pre-order, joins and CTEs owning their expressions; relation markers read as tables. +/// Every node of `sql` parsed in `dialect`, in pre-order, joins and CTEs owning their expressions; relation markers read as tables named by their calls, and every position is in `sql`. pub fn sql_tree_nodes(sql: &str, dialect: &str) -> Result, SqlTreeError> { run_on_worker_stack(TREE_WORKER_STACK_BYTES, "the parser panicked", || { tree_nodes(sql, dialect) @@ -26,12 +26,15 @@ fn tree_nodes(sql: &str, dialect: &str) -> Result, SqlTreeError let parser = rules_tree_parser(dialect) .ok_or_else(|| SqlTreeError::UnavailableDialect(dialect.to_owned()))?; let syntax = dialect_lexical_syntax(dialect); - let (named, calls) = markers_as_names(sql, &syntax).map_err(SqlTreeError::Unanalysable)?; - let named = if dialect == SNOWFLAKE_DIALECT { - table_wrapped(&named, &calls, &syntax) + let (named, calls, ranges) = + located_markers_as_names(sql, &syntax).map_err(SqlTreeError::Unanalysable)?; + let (wrapped, wrappers) = if dialect == SNOWFLAKE_DIALECT { + let (wrapped, wrappers) = table_wrapped(&named, &calls, &syntax); + (Some(wrapped), wrappers) } else { - named + (None, Vec::new()) }; - let statements: Vec = parse(&named, parser)?; - walk(&statements, &line_starts(&named)) + let map = SourceMap::new(sql, &named, &ranges, wrappers); + let statements: Vec = parse(wrapped.as_deref().unwrap_or(&named), parser)?; + walk(&statements, &map) } diff --git a/crates/sqlbuild-analysis/src/sql_tree/models.rs b/crates/sqlbuild-analysis/src/sql_tree/models.rs index 22964a4c2d..1e4b2f8811 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/models.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/models.rs @@ -1,6 +1,6 @@ //! Nodes of a rule's SQL syntax tree. -/// One node: its SQLQuack kind, where it starts, and its parent's index in pre-order. +/// One node: its SQLQuack kind, where it starts and ends, what it names or holds, and its parent's index in pre-order. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SqlTreeNode { /// SQLQuack's snake_case node kind, such as `select`, `subquery`, `star`, `join` or `cte`. @@ -11,6 +11,43 @@ pub struct SqlTreeNode { pub column: usize, /// The parent's index; `None` for a statement. pub parent: Option, + /// The source text the node covers; `None` when the parser recorded no span for it. + pub extent: Option, + /// The name an identifier, column, alias or table node carries. + pub name: Option, + /// The value a literal node holds. + pub literal: Option, +} + +/// Where a node's source text ends, and the character range it covers in the source. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeExtent { + /// 1-based line of the position just after the node's last character. + pub end_line: usize, + /// 1-based column, in characters, of the position just after the node's last character. + pub end_column: usize, + /// The node's start and exclusive end as character offsets into the source. + pub span: (usize, usize), +} + +/// A name as written; a relation marker's name is its call, such as `__ref("orders")`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeName { + /// The name without quote characters. + pub name: String, + /// Whether the source quoted the name. + pub quoted: bool, + /// The names before it, outermost first, such as `["db", "sales"]` in `db.sales.orders`. + pub qualifier: Vec, +} + +/// A literal's value as written, without quotes, and its SQLQuack type, such as `string` or `number`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeLiteral { + /// The value's text. + pub value: String, + /// The snake_case literal type. + pub literal_type: &'static str, } /// The SQLQuack parser that reads a dialect's SQL for rules. diff --git a/crates/sqlbuild-analysis/src/sql_tree/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_tree/tests/helpers.rs index 8338f0309f..37326bf842 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/tests/helpers.rs @@ -1,6 +1,8 @@ -use crate::sql_tree::models::SqlTreeNode; +use crate::sql_tree::models::{NodeExtent, NodeName, SqlTreeNode}; const SHAPE_KINDS: [&str; 5] = ["select", "cte", "join", "subquery", "star"]; +/// How a name's fact reads unquoted, then quoted. +const QUOTING: [&str; 2] = ["", " quoted"]; /// The listed kinds' nodes with their parent's kind, sorted by position, since child order is SQLQuack's. pub(crate) fn shape( @@ -17,3 +19,43 @@ pub(crate) fn shape( shape.sort_by_key(|(kind, line, column, _)| (*line, *column, *kind)); shape } + +/// Each node as `kind line:column`, then what it names or holds, then its end and the source text it covers. +pub(crate) fn facts(sql: &str, nodes: &[SqlTreeNode]) -> Vec { + nodes + .iter() + .map(|node| { + format!( + "{} {}:{}{}{}{}", + node.kind, + node.line, + node.column, + node.name.as_ref().map(name_fact).unwrap_or_default(), + node.literal + .as_ref() + .map(|literal| format!(" {}={}", literal.literal_type, literal.value)) + .unwrap_or_default(), + node.extent + .as_ref() + .map(|extent| extent_fact(sql, extent)) + .unwrap_or_default(), + ) + }) + .collect() +} + +fn name_fact(name: &NodeName) -> String { + let quoted: &str = QUOTING[usize::from(name.quoted)]; + let qualifier: String = name + .qualifier + .first() + .map(|_| format!(" in={}", name.qualifier.join("."))) + .unwrap_or_default(); + format!(" name={}{quoted}{qualifier}", name.name) +} + +fn extent_fact(sql: &str, extent: &NodeExtent) -> String { + let (start, end) = extent.span; + let text: String = sql.chars().skip(start).take(end - start).collect(); + format!(" to {}:{} {text:?}", extent.end_line, extent.end_column) +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/tests/test_sql_tree.rs b/crates/sqlbuild-analysis/src/sql_tree/tests/test_sql_tree.rs index 14eaf3a826..b16b116e67 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/tests/test_sql_tree.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/tests/test_sql_tree.rs @@ -1,13 +1,18 @@ use crate::sql_tree::errors::SqlTreeError; use crate::sql_tree::main::sql_tree_nodes::sql_tree_nodes; -use crate::sql_tree::tests::helpers::shape; -use crate::sql_tree::tests::test_types::{TreeFailureTestCase, TreeShapeTestCase}; +use crate::sql_tree::tests::helpers::{facts, shape}; +use crate::sql_tree::tests::test_types::{ + TreeFactsTestCase, TreeFailureTestCase, TreeShapeTestCase, +}; const CTE_AND_JOIN_SQL: &str = "WITH x AS (SELECT 1 AS id)\nSELECT *\n FROM x JOIN (SELECT 2 AS id) s ON x.id = s.id"; const SNOWFLAKE_MARKER_SQL: &str = "WITH o AS (SELECT * FROM __ref(\"orders\"))\nSELECT o.*, c.name\nFROM o JOIN __ref(\"customers\") c ON o.id = c.id JOIN (SELECT 2 AS id) s ON s.id = o.id\nWHERE o.id IN (SELECT id FROM __source(\"raw_returns\"))"; +const NAMES_AND_LITERALS_SQL: &str = + "SELECT o.\"Id\" AS order_id, 'x''y' AS s, 42 + 1\nFROM db.sch.orders AS o WHERE o.v = N'z'"; + #[test] fn given_sql_when_building_a_rule_tree_then_clauses_own_their_queries_at_their_source_positions() { let test_cases = [ @@ -140,6 +145,108 @@ fn given_sql_when_building_a_rule_tree_then_clauses_own_their_queries_at_their_s } } +#[test] +fn given_sql_when_building_a_rule_tree_then_nodes_carry_names_literals_and_source_extents() { + let test_cases = [ + TreeFactsTestCase { + description: "names keep quoting and qualifiers; a literal without its own span has no end", + dialect: "duckdb", + sql: NAMES_AND_LITERALS_SQL, + expected_facts: &[ + "select 1:1 to 2:41 \"SELECT o.\\\"Id\\\" AS order_id, 'x''y' AS s, 42 + 1\\nFROM db.sch.orders AS o WHERE o.v = N'z'\"", + "alias 1:8 name=order_id to 1:26 \"o.\\\"Id\\\" AS order_id\"", + "column 1:8 name=Id quoted in=o to 1:14 \"o.\\\"Id\\\"\"", + "alias 1:28 name=s to 1:39 \"'x''y' AS s\"", + "literal 1:28 string=x'y", + "add 1:41 to 1:47 \"42 + 1\"", + "literal 1:41 number=42", + "literal 1:41 number=1", + "table 2:6 name=orders in=db.sch to 2:19 \"db.sch.orders\"", + "eq 2:31 to 2:41 \"o.v = N'z'\"", + "column 2:31 name=v in=o to 2:34 \"o.v\"", + "literal 2:31 national_string=z", + ], + }, + TreeFactsTestCase { + description: "native PostgreSQL records the same names and spans, a typed literal as a cast", + dialect: "postgres", + sql: "SELECT o.\"Id\" AS order_id\nFROM db.sch.orders AS o WHERE o.v = DATE '2024-01-02'", + expected_facts: &[ + "select 1:1 to 2:54 \"SELECT o.\\\"Id\\\" AS order_id\\nFROM db.sch.orders AS o WHERE o.v = DATE '2024-01-02'\"", + "alias 1:8 name=order_id to 1:26 \"o.\\\"Id\\\" AS order_id\"", + "column 1:8 name=Id quoted in=o to 1:14 \"o.\\\"Id\\\"\"", + "table 2:6 name=orders in=db.sch to 2:19 \"db.sch.orders\"", + "eq 2:31 to 2:54 \"o.v = DATE '2024-01-02'\"", + "column 2:31 name=v in=o to 2:34 \"o.v\"", + "cast 2:37 to 2:54 \"DATE '2024-01-02'\"", + "literal 2:37 string=2024-01-02", + ], + }, + TreeFactsTestCase { + description: "a marker over three lines is named by its call and ends where the call ends", + dialect: "duckdb", + sql: "SELECT c.id\nFROM __ref(\n \"customers\"\n) AS c\nJOIN orders o ON o.cid = c.id", + expected_facts: &[ + "select 1:1 to 5:30 \"SELECT c.id\\nFROM __ref(\\n \\\"customers\\\"\\n) AS c\\nJOIN orders o ON o.cid = c.id\"", + "join 5:1 to 5:30 \"JOIN orders o ON o.cid = c.id\"", + "column 1:8 name=id in=c to 1:12 \"c.id\"", + "table 2:6 name=__ref(\n \"customers\"\n) to 4:2 \"__ref(\\n \\\"customers\\\"\\n)\"", + "table 5:6 name=orders to 5:12 \"orders\"", + "eq 5:18 to 5:30 \"o.cid = c.id\"", + "column 5:18 name=cid in=o to 5:23 \"o.cid\"", + "column 5:26 name=id in=c to 5:30 \"c.id\"", + ], + }, + TreeFactsTestCase { + description: "a Snowflake table function marker read inside TABLE( ) covers only its source call", + dialect: "snowflake", + sql: "SELECT l.k\nFROM __table_fn(\"lines_for\")('B-1', __ref(\"orders\").k) AS l JOIN t ON t.k = l.k", + expected_facts: &[ + "select 1:1 to 2:80 \"SELECT l.k\\nFROM __table_fn(\\\"lines_for\\\")('B-1', __ref(\\\"orders\\\").k) AS l JOIN t ON t.k = l.k\"", + "join 2:61 to 2:80 \"JOIN t ON t.k = l.k\"", + "column 1:8 name=k in=l to 1:11 \"l.k\"", + "alias 2:6 name=l to 2:60 \"__table_fn(\\\"lines_for\\\")('B-1', __ref(\\\"orders\\\").k) AS l\"", + "function 2:6", + "function 2:6 to 2:55 \"__table_fn(\\\"lines_for\\\")('B-1', __ref(\\\"orders\\\").k)\"", + "literal 2:6 string=B-1", + "column 2:37 name=k in=__ref(\"orders\") to 2:54 \"__ref(\\\"orders\\\").k\"", + "table 2:66 name=t to 2:67 \"t\"", + "eq 2:71 to 2:80 \"t.k = l.k\"", + "column 2:71 name=k in=t to 2:74 \"t.k\"", + "column 2:77 name=k in=l to 2:80 \"l.k\"", + ], + }, + TreeFactsTestCase { + description: "a CTE is named; non-ASCII text keeps character offsets; a two-name marker is its call", + dialect: "snowflake", + sql: "WITH \"Top\" AS (SELECT 'café' AS n FROM __dbt_ref(\"shop\", \"raw\"))\nSELECT x.n FROM \"Top\" x", + expected_facts: &[ + "select 1:1 to 2:24 \"WITH \\\"Top\\\" AS (SELECT 'café' AS n FROM __dbt_ref(\\\"shop\\\", \\\"raw\\\"))\\nSELECT x.n FROM \\\"Top\\\" x\"", + "cte 1:6 name=Top quoted to 1:65 \"\\\"Top\\\" AS (SELECT 'café' AS n FROM __dbt_ref(\\\"shop\\\", \\\"raw\\\"))\"", + "column 2:8 name=n in=x to 2:11 \"x.n\"", + "table 2:17 name=Top quoted to 2:22 \"\\\"Top\\\"\"", + "select 1:16 to 1:64 \"SELECT 'café' AS n FROM __dbt_ref(\\\"shop\\\", \\\"raw\\\")\"", + "alias 1:23 name=n to 1:34 \"'café' AS n\"", + "literal 1:23 string=café", + "table 1:40 name=__dbt_ref(\"shop\", \"raw\") to 1:64 \"__dbt_ref(\\\"shop\\\", \\\"raw\\\")\"", + ], + }, + ]; + for test_case in test_cases { + let nodes = sql_tree_nodes(test_case.sql, test_case.dialect); + assert_eq!( + nodes.as_deref().map(|nodes| facts(test_case.sql, nodes)), + Ok(test_case + .expected_facts + .iter() + .map(|fact| (*fact).to_owned()) + .collect()), + "{}", + test_case.description + ); + } +} + #[test] fn given_sql_without_a_full_tree_when_building_a_rule_tree_then_the_reason_is_reported() { let test_cases = [ diff --git a/crates/sqlbuild-analysis/src/sql_tree/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_tree/tests/test_types.rs index ae152e0f0e..c328f404aa 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/tests/test_types.rs @@ -17,3 +17,11 @@ pub(crate) struct TreeFailureTestCase { pub(crate) sql: String, pub(crate) expected_error: fn(&SqlTreeError) -> bool, } + +/// SQL, its dialect, and each node's facts as `helpers::facts` writes them. +pub(crate) struct TreeFactsTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_facts: &'static [&'static str], +} diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs index 2d6cef8b39..9350081971 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs @@ -8,7 +8,9 @@ use pyo3::types::PyTuple; use pyo3::{IntoPyObject, pyclass, pymethods}; use sqlbuild_analysis::sql_tree::errors::SqlTreeError; use sqlbuild_analysis::sql_tree::main::sql_tree_nodes::sql_tree_nodes; -use sqlbuild_analysis::sql_tree::models::SqlTreeNode as TreeNode; +use sqlbuild_analysis::sql_tree::models::{ + NodeExtent, NodeLiteral, NodeName, SqlTreeNode as TreeNode, +}; use sqlbuild_python_base::records::main::construct::new_annotated_error::new_annotated_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -57,6 +59,56 @@ impl SqlTreeNode { .map_or(1, |node| node.column) } + /// The name an identifier, column, alias, table or CTE carries, unquoted; a relation marker's is its call text. + #[getter] + fn name(&self) -> Option<&str> { + self.node_name().map(|name| name.name.as_str()) + } + + /// Whether the source quoted the name; `None` without one. + #[getter] + fn quoted(&self) -> Option { + self.node_name().map(|name| name.quoted) + } + + /// The names qualifying the name, outermost first, such as `("db", "sales")`; `None` without a name. + #[getter] + fn qualifier<'py>(&self, py: Python<'py>) -> PyResult>> { + self.node_name() + .map(|name| PyTuple::new(py, &name.qualifier)) + .transpose() + } + + /// A literal's value as written, without quotes. + #[getter] + fn value(&self) -> Option<&str> { + self.node_literal().map(|literal| literal.value.as_str()) + } + + /// A literal's snake_case type, such as `string`, `number` or `date`. + #[getter] + fn literal_type(&self) -> Option<&'static str> { + self.node_literal().map(|literal| literal.literal_type) + } + + /// The 1-based line just after the node's last character; `None` when the parser recorded no span. + #[getter] + fn end_line(&self) -> Option { + self.extent().map(|extent| extent.end_line) + } + + /// The 1-based column just after the node's last character; `None` when the parser recorded no span. + #[getter] + fn end_column(&self) -> Option { + self.extent().map(|extent| extent.end_column) + } + + /// The node's `(start, end)` character offsets into the document's source, end exclusive. + #[getter] + fn span(&self) -> Option<(usize, usize)> { + self.extent().map(|extent| extent.span) + } + /// The node's direct children, in SQLQuack's traversal order. #[getter] fn children<'py>(&self, py: Python<'py>) -> PyResult> { @@ -88,6 +140,25 @@ impl SqlTreeNode { } } +impl SqlTreeNode { + /// The node's tree record; `None` for the document root. + fn record(&self) -> Option<&TreeNode> { + self.tree.nodes.get(self.index) + } + + fn node_name(&self) -> Option<&NodeName> { + self.record().and_then(|node| node.name.as_ref()) + } + + fn node_literal(&self) -> Option<&NodeLiteral> { + self.record().and_then(|node| node.literal.as_ref()) + } + + fn extent(&self) -> Option<&NodeExtent> { + self.record().and_then(|node| node.extent.as_ref()) + } +} + /// One SQL text a custom rule reads, parsed on first use by its dialect's SQLQuack parser. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct SqlDocument { diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md index c81e21ff98..2e5fb645d5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md @@ -78,18 +78,36 @@ if enum_types.get("fulfillment_method") != "fulfillment_method": ## Syntax tree -Use the syntax tree only when the common structures are insufficient: +Use the syntax tree only when the common structures are insufficient. Its nodes carry the names, +literal values and source spans a rule reads: ```python -for node in sql.expanded.ast().walk(): - if node.kind == "window_function": - ... +sql = ctx.sql.for_model(model).expanded +for node in sql.ast().walk(): + if node.kind == "alias" and node.quoted and node.name != node.name.lower(): + start, end = node.span + return [ctx.finding(subject=model, node=node, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a `kind`, a `line` and `column` where it starts, `children`, and `walk()`, which yields the node and its descendants in pre-order. Kinds are SQLQuack's snake_case names, such as `select`, `subquery`, -`star`, `join`, `cte`, `column`, `function` and `window_function`. +`star`, `join`, `cte`, `alias`, `column`, `table`, `literal`, `function` and `window_function`. + +Nodes also have these attributes, which are `None` on nodes they don't apply to: + +| Attribute | Nodes | Value | +| --- | --- | --- | +| `name` | `identifier`, `column`, `alias`, `table`, `cte` | The name without quotes, such as `order_id` | +| `quoted` | The same | Whether the source quotes the name | +| `qualifier` | The same | The names before it, outermost first: `("db", "sales")` for `db.sales.orders` | +| `value` | `literal` | The value as written, without quotes: `it's` for `'it''s'` | +| `literal_type` | `literal` | `string`, `number`, `national_string`, `date`, `timestamp` and the other literal types | +| `end_line`, `end_column` | Nodes with a source span | Where the node ends, just after its last character | +| `span` | Nodes with a source span | `(start, end)` character offsets into `source`, end exclusive | + +A relation marker, such as `__ref("orders")`, reads as a `table` node whose `name` and `span` are +the marker call as written. Parsing is lazy. A path or contract Rule does not pay for SQL parsing merely because syntax tree access exists elsewhere. @@ -102,8 +120,9 @@ The tree comes from the SQLQuack parser SQLBuild uses for the project's dialect: | DuckDB, MotherDuck, Snowflake, SQLite | SQLQuack's shared parser | | Other dialects | Unavailable | -For the shared parser, a node the parser records no position for, such as a bare `*` or the query -inside a subquery, reports the position of its parent. +A node without a source span has no end: `end_line`, `end_column` and `span` are `None`, and it +reports its parent's start. Literals have no span. Under the shared parser, neither does the query +inside a subquery. Reading the tree, or any common structure, raises `RuleUsageError` when: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index da5760b90e..798647548d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4335,6 +4335,14 @@ class SqlTreeNode: kind: str line: int column: int + name: str | None + quoted: bool | None + qualifier: tuple[str, ...] | None + value: str | None + literal_type: str | None + end_line: int | None + end_column: int | None + span: tuple[int, int] | None children: tuple[SqlTreeNode, ...] def walk(self) -> list[SqlTreeNode]: ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py index 3dd38677c8..309675d9c4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py @@ -47,6 +47,21 @@ " ]\n" ), ) +_FACTS_RULE: str = custom_rule_source( + code="XSQBRTREE002", + body=( + " sql = ctx.sql.for_model(model).expanded\n" + " return [\n" + " ctx.finding(\n" + " subject=model,\n" + ' message=f"{node.kind}:{node.qualifier or ()}:{node.name or node.value}:"\n' + ' + (sql.source[node.span[0] : node.span[1]] if node.span else "-"),\n' + " )\n" + " for node in sql.ast().walk()\n" + ' if node.kind in ("alias", "column", "literal", "table")\n' + " ]\n" + ), +) @pytest.mark.parametrize( @@ -180,3 +195,59 @@ def test_given_custom_rule_over_the_syntax_tree_when_compiling_then_nodes_point_ assert tuple(sorted(item["message"] for item in custom_rule_diagnostics(result))) == ( test_case.expected_messages ), result.stdout + result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + SyntaxTreeRuleE2ETestCase( + description="a rule reads names, literal values and source spans, a marker as its call", + adapter="snowflake", + model_sql=( + 'MODEL (description "Orders", database analytics, schema analytics);\n' + "SELECT o.\"Status\" AS status_label, 'it''s' AS note\n" + 'FROM __ref("raw_orders") AS o\n' + "WHERE o.region = 'EU'\n" + ), + other_models={ + "raw_orders": ( + 'MODEL (description "raw_orders", database analytics, schema analytics);\n' + "SELECT 'open' AS \"Status\", 'EU' AS region\n" + ) + }, + expected_messages=( + "alias:():Status:'open' AS \"Status\"", + "alias:():note:'it''s' AS note", + "alias:():region:'EU' AS region", + 'alias:():status_label:o."Status" AS status_label', + "column:('o',):Status:o.\"Status\"", + "column:('o',):region:o.region", + "literal:():EU:-", + "literal:():EU:-", + "literal:():it's:-", + "literal:():open:-", + 'table:():__ref("raw_orders"):__ref("raw_orders")', + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_custom_rule_reading_node_facts_when_compiling_then_it_sees_the_authored_text( + test_case: SyntaxTreeRuleE2ETestCase, tmp_path: Path +) -> None: + write_custom_rule_project( + project_dir=tmp_path, + selected_rules=("XSQBRTREE002",), + files=( + ("rules/facts.py", _FACTS_RULE), + *((f"models/{name}.sql", sql) for name, sql in test_case.other_models.items()), + ), + adapter=test_case.adapter, + ) + (tmp_path / "models" / "orders.sql").write_text(test_case.model_sql, encoding="utf-8") + + result: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + assert tuple(sorted(item["message"] for item in custom_rule_diagnostics(result))) == ( + test_case.expected_messages + ), result.stdout + result.stderr diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx index 72215349ce..8798fb9afd 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx @@ -66,18 +66,36 @@ if enum_types.get("fulfillment_method") != "fulfillment_method": ## Syntax tree -Use the syntax tree only when the common structures are insufficient: +Use the syntax tree only when the common structures are insufficient. Its nodes carry the names, +literal values and source spans a rule reads: ```python -for node in sql.expanded.ast().walk(): - if node.kind == "window_function": - ... +sql = ctx.sql.for_model(model).expanded +for node in sql.ast().walk(): + if node.kind == "alias" and node.quoted and node.name != node.name.lower(): + start, end = node.span + return [ctx.finding(subject=model, node=node, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a `kind`, a `line` and `column` where it starts, `children`, and `walk()`, which yields the node and its descendants in pre-order. Kinds are SQLQuack's snake_case names, such as `select`, `subquery`, -`star`, `join`, `cte`, `column`, `function` and `window_function`. +`star`, `join`, `cte`, `alias`, `column`, `table`, `literal`, `function` and `window_function`. + +Nodes also have these attributes, which are `None` on nodes they don't apply to: + +| Attribute | Nodes | Value | +| --- | --- | --- | +| `name` | `identifier`, `column`, `alias`, `table`, `cte` | The name without quotes, such as `order_id` | +| `quoted` | The same | Whether the source quotes the name | +| `qualifier` | The same | The names before it, outermost first: `("db", "sales")` for `db.sales.orders` | +| `value` | `literal` | The value as written, without quotes: `it's` for `'it''s'` | +| `literal_type` | `literal` | `string`, `number`, `national_string`, `date`, `timestamp` and the other literal types | +| `end_line`, `end_column` | Nodes with a source span | Where the node ends, just after its last character | +| `span` | Nodes with a source span | `(start, end)` character offsets into `source`, end exclusive | + +A relation marker, such as `__ref("orders")`, reads as a `table` node whose `name` and `span` are +the marker call as written. Parsing is lazy. A path or contract Rule does not pay for SQL parsing merely because syntax tree access exists elsewhere. @@ -90,8 +108,9 @@ The tree comes from the SQLQuack parser SQLBuild uses for the project's dialect: | DuckDB, MotherDuck, Snowflake, SQLite | SQLQuack's shared parser | | Other dialects | Unavailable | -For the shared parser, a node the parser records no position for, such as a bare `*` or the query -inside a subquery, reports the position of its parent. +A node without a source span has no end: `end_line`, `end_column` and `span` are `None`, and it +reports its parent's start. Literals have no span. Under the shared parser, neither does the query +inside a subquery. Reading the tree, or any common structure, raises `RuleUsageError` when: From bc6317e4481d71af543b562357a15a032d80d6b7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:03:06 +0000 Subject: [PATCH 2/7] docs(rules): report tree-reading findings at the model, not the node ctx.finding(node=...) takes a node's query-relative position, so the docs example and the DuckShop rule report at the model and quote the node's authored text in the message instead. --- benchmarks/duckshop/project/rules/quoted_aliases.py | 1 - .../references/docs/concepts/rules/custom-rules/rule-context.md | 2 +- .../docs/docs/concepts/rules/custom-rules/rule-context.mdx | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/benchmarks/duckshop/project/rules/quoted_aliases.py b/benchmarks/duckshop/project/rules/quoted_aliases.py index 292cf6ee24..bfb855b1ce 100644 --- a/benchmarks/duckshop/project/rules/quoted_aliases.py +++ b/benchmarks/duckshop/project/rules/quoted_aliases.py @@ -29,7 +29,6 @@ def mixed_case_names_stay_in_exports(*, model: Model, ctx: RuleContext) -> list[ findings.append( ctx.finding( subject=model, - node=node, message=f"Quoted mixed-case name {node.name!r} in `{text}`", ) ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md index 2e5fb645d5..b855a669d6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md @@ -86,7 +86,7 @@ sql = ctx.sql.for_model(model).expanded for node in sql.ast().walk(): if node.kind == "alias" and node.quoted and node.name != node.name.lower(): start, end = node.span - return [ctx.finding(subject=model, node=node, message=sql.source[start:end])] + return [ctx.finding(subject=model, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx index 8798fb9afd..e715257b94 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx @@ -74,7 +74,7 @@ sql = ctx.sql.for_model(model).expanded for node in sql.ast().walk(): if node.kind == "alias" and node.quoted and node.name != node.name.lower(): start, end = node.span - return [ctx.finding(subject=model, node=node, message=sql.source[start:end])] + return [ctx.finding(subject=model, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a From a95944888c3c25fc900b68c24970c7dacf52b942 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:11:19 +0000 Subject: [PATCH 3/7] chore(fensu): record the source map's start and end mapping as an intentional mirror --- fensu.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fensu.toml b/fensu.toml index 5b730b140a..55e33573b9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -3852,3 +3852,7 @@ reason = "Each builds its own dbt manifest record type with that type's fields; [[dupes.allowlist]] paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs"] reason = "Each is its own Python entry point with its own keyword contract and dbt command shape; both end in the shared argv builder." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs"] +reason = "source_start and source_end apply opposite boundary rules: a start inside a rewrite maps to the start of the replaced text, while an exclusive end inside or closing one maps to its end, and an end at a removal maps to where the removed text began. Only the partition-and-shift shape is alike." From 2fee400eafa47ed462fd0d2f2db31ccde00bf169 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:16:22 +0000 Subject: [PATCH 4/7] style: format the DuckShop quoted alias rule tests --- benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py index 722af47d38..9a6079175b 100644 --- a/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py +++ b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py @@ -6,8 +6,8 @@ _MIXED_CASE = ( _HEADER + "SELECT\n" - + " l.listing_id AS \"listingId\",\n" - + " l.note AS \"note\",\n" + + ' l.listing_id AS "listingId",\n' + + ' l.note AS "note",\n' + " 'isLive' AS status_label\n" + 'FROM __ref("dim_listings") AS l\n' ) From 765f44bff31bdb9d8fc82aa02bf3bc476d0e16a3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:57:05 +0000 Subject: [PATCH 5/7] fix(rules): report custom rule findings at a node's place in the model file ctx.finding(subject=model, node=...) took the node's line and column in the model's expanded query and reported them against the model file, so a multi-line MODEL header or a multi-line macro expansion moved the finding. The expanded SQL document now carries where the query sits in the file, read the way built-in model rules map positions, and the finding maps the node through it; a node inside macro output points at the macro call. The query place travels in the custom host payload and in the replayed sql.for_model fact, so cached findings follow it. --- .../duckshop/project/rules/quoted_aliases.py | 1 + .../project/tests/test_rule_quoted_aliases.py | 1 + .../_helpers/custom/custom_host_payload.rs | 3 + .../src/rules/_helpers/fact/fact_replay.rs | 1 + .../src/rules/_helpers/sql/sql_document.rs | 57 +++++++++++++++-- .../src/compiled_views/models.rs | 24 ++++++- .../src/engine/_helpers/project_rows.rs | 6 +- crates/sqlbuild-rules/src/engine/main/mod.rs | 1 + .../src/engine/main/model_query_place.rs | 26 ++++++++ crates/sqlbuild-rules/src/models.rs | 13 ++++ .../src/query_facts/main/mod.rs | 1 + .../query_facts/main/query_file_position.rs | 13 ++++ .../rules/custom-rules/rule-context.md | 5 +- src/sqlbuild/_native.pyi | 5 +- .../rule_engine/_helpers/host/host_payload.py | 4 ++ .../rule_engine/classes/rule_context.py | 5 ++ src/sqlbuild/rule_engine/classes/sql_facts.py | 6 +- .../main/rules/test_rule_positions.py | 62 +++++++++++++++++++ .../rules/custom-rules/rule-context.mdx | 5 +- 19 files changed, 226 insertions(+), 13 deletions(-) create mode 100644 crates/sqlbuild-rules/src/engine/main/model_query_place.rs create mode 100644 crates/sqlbuild-rules/src/query_facts/main/query_file_position.rs diff --git a/benchmarks/duckshop/project/rules/quoted_aliases.py b/benchmarks/duckshop/project/rules/quoted_aliases.py index bfb855b1ce..292cf6ee24 100644 --- a/benchmarks/duckshop/project/rules/quoted_aliases.py +++ b/benchmarks/duckshop/project/rules/quoted_aliases.py @@ -29,6 +29,7 @@ def mixed_case_names_stay_in_exports(*, model: Model, ctx: RuleContext) -> list[ findings.append( ctx.finding( subject=model, + node=node, message=f"Quoted mixed-case name {node.name!r} in `{text}`", ) ) diff --git a/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py index 9a6079175b..1303b6da7f 100644 --- a/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py +++ b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py @@ -30,6 +30,7 @@ def test_given_quoted_mixed_case_alias_outside_an_export_when_evaluating_then_re ) assert result.finding_count == 1 + assert (result.findings[0].line, result.findings[0].column) == (3, 3) assert result.findings[0].message == ( "Quoted mixed-case name 'listingId' in `l.listing_id AS \"listingId\"`" ) diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs index 0a6cc63b53..5e4ab4b24c 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs @@ -212,6 +212,8 @@ impl PayloadWriter { self.part(TEXT_TAG, authored.as_bytes())?, ); model.insert("query".to_owned(), self.part(TEXT_TAG, query.as_bytes())?); + let place: String = view.query_place(py)?; + model.insert("place".to_owned(), self.part(TEXT_TAG, place.as_bytes())?); let inferred: Value = match view.inferred_rows(py)? { Some(rows) => self.part(INFERRED_TAG, &json_bytes(&rows)?)?, None => Value::Null, @@ -242,6 +244,7 @@ impl PayloadWriter { model.insert("authored".to_owned(), authored); let query: Value = self.text(&native_field(record, "query_sql")?)?; model.insert("query".to_owned(), query); + model.insert("place".to_owned(), Value::Null); let inferred: Bound<'_, PyAny> = native_field(record, "inferred_columns")?; let inferred: Value = if inferred.is_none() { Value::Null diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs index a12a8f9472..c49145ecd6 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs @@ -431,6 +431,7 @@ impl<'py> FactReplay<'py> { JsonValue::Array(vec![ JsonValue::String(authored), JsonValue::String(expanded), + JsonValue::String(view.query_place(py)?), ]) } None => { diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs index 9350081971..3ae82b5c4e 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs @@ -2,7 +2,8 @@ use std::sync::Arc; -use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyResult, Python}; +use pyo3::exceptions::{PyAttributeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; use pyo3::{IntoPyObject, pyclass, pymethods}; @@ -11,9 +12,12 @@ use sqlbuild_analysis::sql_tree::main::sql_tree_nodes::sql_tree_nodes; use sqlbuild_analysis::sql_tree::models::{ NodeExtent, NodeLiteral, NodeName, SqlTreeNode as TreeNode, }; +use sqlbuild_rules::models::ModelQueryPlace; +use sqlbuild_rules::query_facts::main::query_file_position::query_file_position; use sqlbuild_python_base::records::main::construct::new_annotated_error::new_annotated_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; @@ -165,19 +169,26 @@ pub(crate) struct SqlDocument { #[pyo3(get)] source: String, dialect: String, + /// Where `source`, a model's expanded query, sits in the model file; `None` for other SQL. + place: Option, tree: PyOnceLock>, } #[pymethods] impl SqlDocument { #[new] - #[pyo3(signature = (*, source, dialect))] - fn new(source: String, dialect: String) -> Self { - Self { + #[pyo3(signature = (*, source, dialect, place=None))] + fn new(source: String, dialect: String, place: Option<&str>) -> PyResult { + let place: Option = place + .map(serde_json::from_str) + .transpose() + .map_err(|error| PyValueError::new_err(format!("invalid query place: {error}")))?; + Ok(Self { source, dialect, + place, tree: PyOnceLock::new(), - } + }) } /// The syntax tree: a `document` node whose children are the statements. @@ -207,6 +218,29 @@ impl SqlDocument { self.nodes(py, "Subquery") } + /// Where `node`, read from this model query, is in the model file; `None` for other SQL. + #[pyo3(signature = (*, node))] + fn file_position(&self, node: &Bound<'_, PyAny>) -> PyResult> { + let Some(place) = self.place.as_ref() else { + return Ok(None); + }; + let (Some(line), Some(column)) = (node_index(node, "line")?, node_index(node, "column")?) + else { + return Ok(None); + }; + let above: usize = self + .source + .split_inclusive('\n') + .take(line - 1) + .map(|text| text.chars().count()) + .sum(); + Ok(Some(query_file_position( + &self.source, + place, + above + column - 1, + ))) + } + /// Every projected star, bare or qualified; a star a function call takes, such as `COUNT(*)`, is no projection. fn star_projections<'py>(&self, py: Python<'py>) -> PyResult> { self.source_nodes(py, "Star", |nodes, node| { @@ -253,6 +287,19 @@ impl SqlDocument { } } +/// A node's 1-based `line` or `column` attribute; `None` when it has none or it is not positive. +fn node_index(node: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let value: i64 = match native_field(node, name) { + Ok(value) => value.extract()?, + Err(error) if error.is_instance_of::(node.py()) => return Ok(None), + Err(error) => return Err(error), + }; + Ok(match usize::try_from(value) { + Ok(index) if index > 0 => Some(index), + _ => None, + }) +} + /// `nodes` with each node's children listed, in ascending index order, under its parent. fn tree(nodes: Vec) -> Arc { let mut children: Vec> = vec![Vec::new(); nodes.len() + 1]; diff --git a/crates/sqlbuild-python-project/src/compiled_views/models.rs b/crates/sqlbuild-python-project/src/compiled_views/models.rs index 28438e56d5..dc513bdecb 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/models.rs @@ -1,5 +1,8 @@ //! Public records of compiled views. +use sqlbuild_rules::engine::main::model_query_place::model_query_place; +use sqlbuild_rules::models::ModelQueryPlace; + use crate::compiled_project::main::reference_row_tuple::reference_row_tuple; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_project::models::ReferenceTuple; @@ -420,12 +423,23 @@ impl NativeCompiledModel { /// The model's authored file text and expanded query, from its recorded facts. pub fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> { + self.with_facts(py, |facts| { + (facts.authored_sql.to_string(), facts.query_sql.to_string()) + }) + } + + /// `read` applied to the model's recorded facts. + pub fn with_facts( + &self, + py: Python<'_>, + read: impl FnOnce(&CompiledModelFacts) -> T, + ) -> PyResult { let name: String = self.name(py)?; let owner = self.owner.bind(py).borrow(); let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") })?; - Ok((facts.authored_sql.to_string(), facts.query_sql.to_string())) + Ok(read(facts)) } pub(crate) fn model_file( @@ -685,6 +699,14 @@ impl NativeCompiledModel { self.model_file(py, |file| Ok(file.contents(py))) } + /// Where the expanded query sits in the model file, as JSON a rule's SQL document maps positions through. + #[getter] + pub fn query_place(&self, py: Python<'_>) -> PyResult { + let place: ModelQueryPlace = self.with_facts(py, model_query_place)?; + serde_json::to_string(&place) + .map_err(|error| pyo3::exceptions::PyRuntimeError::new_err(error.to_string())) + } + #[getter] pub fn authored_query_sql<'py>(&self, py: Python<'py>) -> PyResult> { self.model_file(py, |file| Ok(file.query_sql(py))) diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index cffac3626c..c4e5ed2fc0 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -118,7 +118,7 @@ pub(crate) fn declaration(fact: &DeclarationFact) -> Declaration { } /// The expansion passes back to the authored query and where that query starts in the file. -fn authored_query_map(facts: &CompiledModelFacts) -> Option { +pub(crate) fn authored_query_map(facts: &CompiledModelFacts) -> Option { let span = |span: &CharSpan| ExpansionSpan { source_start: character_offset(span.source_start), source_end: character_offset(span.source_end), @@ -151,7 +151,7 @@ fn character_offset(offset: usize) -> i64 { } /// The file-line characters before the query's first content that the query itself lacks. -fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 { +pub(crate) fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 { let line_prefix = |text: &str, at: usize| { text[..at] .rsplit('\n') @@ -172,7 +172,7 @@ fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 { } /// The lines of `file` above `query`, the rendered form of its `authored_query`; 0 if unknown. -fn query_line_offset(file: &str, authored_query: &str, query: &str) -> u64 { +pub(crate) fn query_line_offset(file: &str, authored_query: &str, query: &str) -> u64 { let leading_lines = |text: &str| { let body: &str = text.trim_start(); text[..text.len() - body.len()].matches('\n').count() diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index 1acb27f0d2..6f601523a8 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -6,6 +6,7 @@ pub mod evaluate_rows; pub mod finalize_rows; pub mod json_grain_skipped_models; pub mod json_grains; +pub mod model_query_place; pub mod needs_json_grains; pub mod project_model_rows; pub mod retained_request; diff --git a/crates/sqlbuild-rules/src/engine/main/model_query_place.rs b/crates/sqlbuild-rules/src/engine/main/model_query_place.rs new file mode 100644 index 0000000000..aab857992e --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/model_query_place.rs @@ -0,0 +1,26 @@ +//! Where a compiled model's expanded query sits in its file. + +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; + +use crate::engine::_helpers::project_rows::{ + authored_query_map, query_column_offset, query_line_offset, +}; +use crate::models::ModelQueryPlace; + +/// The file place of `facts`' expanded query, read the way built-in model rules read it. +#[must_use] +pub fn model_query_place(facts: &CompiledModelFacts) -> ModelQueryPlace { + let file: &str = &facts.authored_sql; + let authored_query: &str = &facts.authored_query_sql; + ModelQueryPlace { + authored_sql: if authored_query.is_empty() { + file + } else { + authored_query + } + .to_owned(), + line_offset: query_line_offset(file, authored_query, &facts.query_sql), + column_offset: query_column_offset(file, authored_query, &facts.query_sql), + authored_query: authored_query_map(facts), + } +} diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index bd71cf5cbe..96ea05491c 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -952,6 +952,19 @@ impl fensu_policy::policy::types::PolicyRule for RuleMetadata { } } +/// Where a model's expanded query sits in its file, as built-in rules map query positions. +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct ModelQueryPlace { + /// The authored query, or the whole file when the model has none. + pub authored_sql: String, + /// The lines of the model file above the query. + pub line_offset: u64, + /// The file-line characters before the query's first content that the query itself lacks. + pub column_offset: u64, + /// The expansion passes back to the authored query, when expansion changed it. + pub authored_query: Option, +} + /// The expansion passes from a model's authored query to `query_sql`, and the query's file place. #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] pub struct AuthoredQueryMap { diff --git a/crates/sqlbuild-rules/src/query_facts/main/mod.rs b/crates/sqlbuild-rules/src/query_facts/main/mod.rs index a4b8b0f0c7..fbd4506534 100644 --- a/crates/sqlbuild-rules/src/query_facts/main/mod.rs +++ b/crates/sqlbuild-rules/src/query_facts/main/mod.rs @@ -3,4 +3,5 @@ pub mod facts_available; pub mod model_shape; pub mod parse_fixture; pub mod query_facts; +pub mod query_file_position; pub mod stub_call; diff --git a/crates/sqlbuild-rules/src/query_facts/main/query_file_position.rs b/crates/sqlbuild-rules/src/query_facts/main/query_file_position.rs new file mode 100644 index 0000000000..c1dc00056a --- /dev/null +++ b/crates/sqlbuild-rules/src/query_facts/main/query_file_position.rs @@ -0,0 +1,13 @@ +//! The model-file position of a character of a model's expanded query. + +use crate::models::ModelQueryPlace; +use crate::query_facts::models::ModelSource; + +/// The 1-based file line and column of `sql`'s character `character`; expansion output maps to its call. +#[must_use] +pub fn query_file_position(sql: &str, place: &ModelQueryPlace, character: usize) -> (u64, u64) { + let offsets: Vec = (0..=sql.chars().count()).collect(); + ModelSource::new(sql, &offsets, (place.line_offset, place.column_offset)) + .with_authored(&place.authored_sql, place.authored_query.as_ref()) + .line_column(character) +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md index b855a669d6..3293adb007 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md @@ -64,6 +64,9 @@ for star in sql.expanded.star_projections(): text, not the model file. Authored SQL is the whole model file, including its `MODEL` header, so it does not parse as SQL; read its text through `source`. +`ctx.finding(subject=model, node=node)` reports a node of that model's expanded SQL where it is in +the model file. A node inside a macro's output is reported at the macro call. + ## Declared enum types `ctx.columns.declared(model)` reports each declared column with its portable scalar type, so an @@ -86,7 +89,7 @@ sql = ctx.sql.for_model(model).expanded for node in sql.ast().walk(): if node.kind == "alias" and node.quoted and node.name != node.name.lower(): start, end = node.span - return [ctx.finding(subject=model, message=sql.source[start:end])] + return [ctx.finding(subject=model, node=node, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 798647548d..0ec799890f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1416,6 +1416,8 @@ class NativeCompiledModel: @property def authored_sql(self) -> str: ... @property + def query_place(self) -> str: ... + @property def authored_query_sql(self) -> str: ... @property def output_column_locations(self) -> dict[str, SourceLocation]: ... @@ -4321,8 +4323,9 @@ class SqlDocument: """One SQL text a custom rule reads, parsed on first use by its dialect's SQLQuack parser.""" source: str - def __init__(self, *, source: str, dialect: str) -> None: ... + def __init__(self, *, source: str, dialect: str, place: str | None = None) -> None: ... def ast(self) -> SqlTreeNode: ... + def file_position(self, *, node: object) -> tuple[int, int] | None: ... def nodes(self, *, kind: str) -> tuple[SqlNode, ...]: ... def ctes(self) -> tuple[SqlNode, ...]: ... def joins(self) -> tuple[SqlNode, ...]: ... diff --git a/src/sqlbuild/rule_engine/_helpers/host/host_payload.py b/src/sqlbuild/rule_engine/_helpers/host/host_payload.py index 46228fcec7..05766a9a0b 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/host_payload.py +++ b/src/sqlbuild/rule_engine/_helpers/host/host_payload.py @@ -101,6 +101,10 @@ def authored_sql(self) -> str: def query_sql(self) -> str: return self._payload.value(self._entry["query"]) + @cached_property + def query_place(self) -> str | None: + return self._payload.value(self._entry["place"]) + def config_value(self, name: str) -> object: """Return `config.values.get(name)` for one config value rule facts read.""" diff --git a/src/sqlbuild/rule_engine/classes/rule_context.py b/src/sqlbuild/rule_engine/classes/rule_context.py index 5700d2614b..5b76e360ba 100644 --- a/src/sqlbuild/rule_engine/classes/rule_context.py +++ b/src/sqlbuild/rule_engine/classes/rule_context.py @@ -110,6 +110,11 @@ def finding( path: Path if isinstance(subject, Model): path = subject.path + if node is not None and line is None and column is None: + in_file: tuple[int, int] | None = self.sql.for_model( + subject + ).expanded.file_position(node=node) + line, column = in_file or (None, None) elif isinstance(subject, ProjectPath): path = Path(subject.value) else: diff --git a/src/sqlbuild/rule_engine/classes/sql_facts.py b/src/sqlbuild/rule_engine/classes/sql_facts.py index 328bf3ec1c..984aa1cd8a 100644 --- a/src/sqlbuild/rule_engine/classes/sql_facts.py +++ b/src/sqlbuild/rule_engine/classes/sql_facts.py @@ -25,7 +25,11 @@ def for_model(self, model: Model) -> ModelSql: compiled: RuleFactModel = self._resolve(model) documents: ModelSql = ModelSql( authored=SqlDocument(source=compiled.authored_sql, dialect=self._dialect), - expanded=SqlDocument(source=compiled.query_sql, dialect=self._dialect), + expanded=SqlDocument( + source=compiled.query_sql, + dialect=self._dialect, + place=getattr(compiled, "query_place", None), + ), ) self._documents[model.path] = documents return documents diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py index 309675d9c4..adcbf5f610 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py @@ -63,6 +63,18 @@ ), ) +_NODE_RULE: str = custom_rule_source( + code="XSQBRTREE003", + body=( + " sql = ctx.sql.for_model(model).expanded\n" + ' aliases = [node for node in sql.ast().walk() if node.kind == "alias"]\n' + " return [\n" + " ctx.finding(subject=model, node=node, message=node.kind)\n" + " for node in (*sql.star_projections(), *aliases)\n" + " ]\n" + ), +) + @pytest.mark.parametrize( "test_case", @@ -251,3 +263,53 @@ def test_given_custom_rule_reading_node_facts_when_compiling_then_it_sees_the_au assert tuple(sorted(item["message"] for item in custom_rule_diagnostics(result))) == ( test_case.expected_messages ), result.stdout + result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + SyntaxTreeRuleE2ETestCase( + description="a multi-line MODEL header puts nodes on their file lines", + model_sql=( + 'MODEL (\n description "Orders",\n materialized table\n);\n\n' + "SELECT\n *\nFROM (SELECT 1 AS id) AS o\n" + ), + expected_messages=("Star@7:3", "alias@8:14"), + ), + SyntaxTreeRuleE2ETestCase( + description="a multi-line macro expansion above a node leaves it on its file line", + model_sql=( + 'MODEL (\n description "Orders"\n);\n' + "SELECT\n @sql.blank_to_null('a') AS a,\n *\nFROM (SELECT 'x' AS b) AS o\n" + ), + other_models=_BLANK_TO_NULL, + expected_messages=("Star@6:3", "alias@5:3", "alias@7:14"), + ), + ], + ids=lambda case: case.description, +) +def test_given_custom_rule_finding_at_a_node_when_compiling_then_it_points_at_the_model_file( + test_case: SyntaxTreeRuleE2ETestCase, tmp_path: Path +) -> None: + write_custom_rule_project( + project_dir=tmp_path, + selected_rules=("XSQBRTREE003",), + files=(("rules/nodes.py", _NODE_RULE), *test_case.other_models.items()), + configuration="\n[scopes]\nenforce_placement = false\n", + adapter=test_case.adapter, + ) + (tmp_path / "models" / "orders.sql").write_text(test_case.model_sql, encoding="utf-8") + + cold: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + warm: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + for result in (cold, warm): + assert ( + tuple( + sorted( + f"{item['message']}@{item['line']}:{item['column']}" + for item in custom_rule_diagnostics(result) + ) + ) + == test_case.expected_messages + ), result.stdout + result.stderr diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx index e715257b94..3e3e671cae 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx @@ -52,6 +52,9 @@ for star in sql.expanded.star_projections(): text, not the model file. Authored SQL is the whole model file, including its `MODEL` header, so it does not parse as SQL; read its text through `source`. +`ctx.finding(subject=model, node=node)` reports a node of that model's expanded SQL where it is in +the model file. A node inside a macro's output is reported at the macro call. + ## Declared enum types `ctx.columns.declared(model)` reports each declared column with its portable scalar type, so an @@ -74,7 +77,7 @@ sql = ctx.sql.for_model(model).expanded for node in sql.ast().walk(): if node.kind == "alias" and node.quoted and node.name != node.name.lower(): start, end = node.span - return [ctx.finding(subject=model, message=sql.source[start:end])] + return [ctx.finding(subject=model, node=node, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a From e9791a3c724c797f4f50529008332e0c1488ec5e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:22:38 +0000 Subject: [PATCH 6/7] fix(rules): place a document's node findings in one pass Each node finding rebuilt the model query's character and line tables, so reporting every column of a wide model took time proportional to findings times model size. A document now maps all its syntax tree node starts into the model file once, on the first node finding, and later findings look their node up. A node whose line or column is not an integer gets no file position again and falls back to its own line and column. --- .../src/rules/_helpers/sql/sql_document.rs | 81 ++++++++++++++--- .../src/query_facts/main/mod.rs | 2 +- .../query_facts/main/query_file_position.rs | 13 --- .../query_facts/main/query_file_positions.rs | 21 +++++ tests/unit/src/sqlbuild/rules/_test_types.py | 21 +++++ .../src/sqlbuild/rules/test_sql_document.py | 87 ++++++++++++++++++- 6 files changed, 196 insertions(+), 29 deletions(-) delete mode 100644 crates/sqlbuild-rules/src/query_facts/main/query_file_position.rs create mode 100644 crates/sqlbuild-rules/src/query_facts/main/query_file_positions.rs diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs index 3ae82b5c4e..90ecde4ab7 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs @@ -1,5 +1,6 @@ //! Custom rules' SQL documents: the SQLQuack syntax tree of one SQL text and its common structures. +use std::collections::HashMap; use std::sync::Arc; use pyo3::exceptions::{PyAttributeError, PyValueError}; @@ -13,7 +14,7 @@ use sqlbuild_analysis::sql_tree::models::{ NodeExtent, NodeLiteral, NodeName, SqlTreeNode as TreeNode, }; use sqlbuild_rules::models::ModelQueryPlace; -use sqlbuild_rules::query_facts::main::query_file_position::query_file_position; +use sqlbuild_rules::query_facts::main::query_file_positions::query_file_positions; use sqlbuild_python_base::records::main::construct::new_annotated_error::new_annotated_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -33,6 +34,14 @@ struct Tree { children: Vec>, } +/// A model query's character offsets and the file positions of its tree's node starts, built once. +struct FilePositions { + /// The character each line starts at, then the source's length. + line_starts: Vec, + /// The file `(line, column)` of every tree node's first character, by that character. + node_starts: HashMap, +} + /// One node of a rule's SQL syntax tree: an index into its shared tree, so freeing never recurses. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct SqlTreeNode { @@ -172,6 +181,7 @@ pub(crate) struct SqlDocument { /// Where `source`, a model's expanded query, sits in the model file; `None` for other SQL. place: Option, tree: PyOnceLock>, + positions: PyOnceLock, } #[pymethods] @@ -188,6 +198,7 @@ impl SqlDocument { dialect, place, tree: PyOnceLock::new(), + positions: PyOnceLock::new(), }) } @@ -228,17 +239,17 @@ impl SqlDocument { else { return Ok(None); }; - let above: usize = self - .source - .split_inclusive('\n') - .take(line - 1) - .map(|text| text.chars().count()) - .sum(); - Ok(Some(query_file_position( - &self.source, - place, - above + column - 1, - ))) + let positions: &FilePositions = self + .positions + .get_or_init(node.py(), || self.file_positions(node.py(), place)); + let character: usize = positions.character(line, column); + Ok(Some( + positions + .node_starts + .get(&character) + .copied() + .unwrap_or_else(|| query_file_positions(&self.source, place, &[character])[0]), + )) } /// Every projected star, bare or qualified; a star a function call takes, such as `COUNT(*)`, is no projection. @@ -278,6 +289,37 @@ impl SqlDocument { PyTuple::new(py, found) } + /// Line starts and the file positions of every tree node's start, mapped through `place` at once. + fn file_positions(&self, py: Python<'_>, place: &ModelQueryPlace) -> FilePositions { + let mut line_starts: Vec = vec![0]; + let mut length: usize = 0; + for character in self.source.chars() { + length += 1; + if character == '\n' { + line_starts.push(length); + } + } + line_starts.push(length); + let mut positions: FilePositions = FilePositions { + line_starts, + node_starts: HashMap::new(), + }; + let mut starts: Vec = self.parsed(py).map_or_else( + |_| Vec::new(), + |tree| { + tree.nodes + .iter() + .map(|node| positions.character(node.line, node.column)) + .collect() + }, + ); + starts.sort_unstable(); + starts.dedup(); + let mapped: Vec<(u64, u64)> = query_file_positions(&self.source, place, &starts); + positions.node_starts = starts.into_iter().zip(mapped).collect(); + positions + } + /// The parsed tree; a failure raises `RuleUsageError` each time it is read. fn parsed(&self, py: Python<'_>) -> PyResult<&Arc> { self.tree.get_or_try_init(py, || { @@ -287,10 +329,21 @@ impl SqlDocument { } } -/// A node's 1-based `line` or `column` attribute; `None` when it has none or it is not positive. +impl FilePositions { + /// The source character at 1-based `line` and `column`; a line past the end starts at the end. + fn character(&self, line: usize, column: usize) -> usize { + let last: usize = self.line_starts.len() - 1; + self.line_starts[line.saturating_sub(1).min(last)] + column.saturating_sub(1) + } +} + +/// A node's 1-based `line` or `column` attribute; `None` when it has none or it is not a positive integer. fn node_index(node: &Bound<'_, PyAny>, name: &str) -> PyResult> { let value: i64 = match native_field(node, name) { - Ok(value) => value.extract()?, + Ok(value) => match value.extract() { + Ok(value) => value, + Err(_) => return Ok(None), + }, Err(error) if error.is_instance_of::(node.py()) => return Ok(None), Err(error) => return Err(error), }; diff --git a/crates/sqlbuild-rules/src/query_facts/main/mod.rs b/crates/sqlbuild-rules/src/query_facts/main/mod.rs index fbd4506534..ad572edc3c 100644 --- a/crates/sqlbuild-rules/src/query_facts/main/mod.rs +++ b/crates/sqlbuild-rules/src/query_facts/main/mod.rs @@ -3,5 +3,5 @@ pub mod facts_available; pub mod model_shape; pub mod parse_fixture; pub mod query_facts; -pub mod query_file_position; +pub mod query_file_positions; pub mod stub_call; diff --git a/crates/sqlbuild-rules/src/query_facts/main/query_file_position.rs b/crates/sqlbuild-rules/src/query_facts/main/query_file_position.rs deleted file mode 100644 index c1dc00056a..0000000000 --- a/crates/sqlbuild-rules/src/query_facts/main/query_file_position.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! The model-file position of a character of a model's expanded query. - -use crate::models::ModelQueryPlace; -use crate::query_facts::models::ModelSource; - -/// The 1-based file line and column of `sql`'s character `character`; expansion output maps to its call. -#[must_use] -pub fn query_file_position(sql: &str, place: &ModelQueryPlace, character: usize) -> (u64, u64) { - let offsets: Vec = (0..=sql.chars().count()).collect(); - ModelSource::new(sql, &offsets, (place.line_offset, place.column_offset)) - .with_authored(&place.authored_sql, place.authored_query.as_ref()) - .line_column(character) -} diff --git a/crates/sqlbuild-rules/src/query_facts/main/query_file_positions.rs b/crates/sqlbuild-rules/src/query_facts/main/query_file_positions.rs new file mode 100644 index 0000000000..f4750564ea --- /dev/null +++ b/crates/sqlbuild-rules/src/query_facts/main/query_file_positions.rs @@ -0,0 +1,21 @@ +//! The model-file positions of characters of a model's expanded query. + +use crate::models::ModelQueryPlace; +use crate::query_facts::models::ModelSource; + +/// The 1-based file line and column of each of `sql`'s `characters`; expansion output maps to its call. +#[must_use] +pub fn query_file_positions( + sql: &str, + place: &ModelQueryPlace, + characters: &[usize], +) -> Vec<(u64, u64)> { + let offsets: Vec = (0..=sql.chars().count()).collect(); + let source: ModelSource<'_> = + ModelSource::new(sql, &offsets, (place.line_offset, place.column_offset)) + .with_authored(&place.authored_sql, place.authored_query.as_ref()); + characters + .iter() + .map(|character| source.line_column(*character)) + .collect() +} diff --git a/tests/unit/src/sqlbuild/rules/_test_types.py b/tests/unit/src/sqlbuild/rules/_test_types.py index 2cffe01e63..0b98fad832 100644 --- a/tests/unit/src/sqlbuild/rules/_test_types.py +++ b/tests/unit/src/sqlbuild/rules/_test_types.py @@ -11,3 +11,24 @@ class DeepTreeTestCase: dialect: str terms: int expected_node_count: int + + +@dataclass(frozen=True) +class FilePositionTimeLimitTestCase: + """A wide model query whose every column a rule reports at its model-file position.""" + + description: str + dialect: str + columns: int + line_offset: int + expected_max_seconds: float + + +@dataclass(frozen=True) +class UnplacedNodeTestCase: + """A node-like value a rule reports, placed in a model file only with integer positions.""" + + description: str + line: object + column: object + expected_position: tuple[int, int] | None diff --git a/tests/unit/src/sqlbuild/rules/test_sql_document.py b/tests/unit/src/sqlbuild/rules/test_sql_document.py index 77ec496732..bf0734c7e5 100644 --- a/tests/unit/src/sqlbuild/rules/test_sql_document.py +++ b/tests/unit/src/sqlbuild/rules/test_sql_document.py @@ -1,11 +1,18 @@ """Behavior tests for the syntax tree custom rules read through `SqlDocument`.""" import gc +import json +import time +from types import SimpleNamespace import pytest from sqlbuild.rules import SqlDocument, SqlTreeNode -from tests.unit.src.sqlbuild.rules._test_types import DeepTreeTestCase +from tests.unit.src.sqlbuild.rules._test_types import ( + DeepTreeTestCase, + FilePositionTimeLimitTestCase, + UnplacedNodeTestCase, +) @pytest.mark.parametrize( @@ -35,3 +42,81 @@ def test_given_deep_query_when_reading_and_freeing_its_tree_then_the_process_sur _ = gc.collect() assert walked == test_case.expected_node_count + + +@pytest.mark.parametrize( + "test_case", + [ + FilePositionTimeLimitTestCase( + description="20,000 columns, one per line, five header lines", + dialect="duckdb", + columns=20_000, + line_offset=5, + expected_max_seconds=2.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_wide_model_query_when_placing_every_column_then_time_stays_linear( + test_case: FilePositionTimeLimitTestCase, +) -> None: + source: str = ( + "SELECT\n" + ",\n".join(f" c{index}" for index in range(test_case.columns)) + "\nFROM t" + ) + place: str = json.dumps( + { + "authored_sql": source, + "line_offset": test_case.line_offset, + "column_offset": 0, + "authored_query": None, + } + ) + document: SqlDocument = SqlDocument(source=source, dialect=test_case.dialect, place=place) + + started: float = time.perf_counter() + positions: list[tuple[int, int] | None] = [ + document.file_position(node=node) for node in document.nodes(kind="Column") + ] + + assert time.perf_counter() - started < test_case.expected_max_seconds + assert positions == [ + (index + 2 + test_case.line_offset, 3) for index in range(test_case.columns) + ] + + +@pytest.mark.parametrize( + "test_case", + [ + UnplacedNodeTestCase( + description="integer line and column", + line=1, + column=8, + expected_position=(3, 8), + ), + UnplacedNodeTestCase( + description="no line or column", line=None, column=None, expected_position=None + ), + UnplacedNodeTestCase( + description="text line and column", line="1", column="8", expected_position=None + ), + ], + ids=lambda case: case.description, +) +def test_given_node_position_when_placing_it_then_only_integers_map_into_the_file( + test_case: UnplacedNodeTestCase, +) -> None: + document: SqlDocument = SqlDocument( + source="SELECT 1", + dialect="duckdb", + place=json.dumps( + { + "authored_sql": "SELECT 1", + "line_offset": 2, + "column_offset": 0, + "authored_query": None, + } + ), + ) + + node: SimpleNamespace = SimpleNamespace(line=test_case.line, column=test_case.column) + assert document.file_position(node=node) == test_case.expected_position From 9f145a37d51ba4aa26f8d6685fcaa181a2a6f2b9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:06:56 +0000 Subject: [PATCH 7/7] perf(rules): keep the custom-rule query place to a byte range and expansion passes The place of a model's expanded query repeated the authored query text in every model's custom-rule payload, about a sixth of the payload on a dense 3,000-model project. It now names the query by its byte range in the model file, which the rule's SQL document already holds, and offsets are derived only when a rule reports a node. --- .../src/rules/_helpers/sql/sql_document.rs | 26 +++++++---- .../src/engine/_helpers/project_rows.rs | 38 ++++++++++++---- crates/sqlbuild-rules/src/engine/main/mod.rs | 1 + .../src/engine/main/model_query_place.rs | 35 ++++++++------- .../src/engine/main/query_file_positions.rs | 44 +++++++++++++++++++ crates/sqlbuild-rules/src/models.rs | 22 ++++++---- .../src/query_facts/main/mod.rs | 1 - .../query_facts/main/query_file_positions.rs | 21 --------- src/sqlbuild/_native.pyi | 4 +- src/sqlbuild/rule_engine/classes/sql_facts.py | 1 + .../src/sqlbuild/rules/test_sql_document.py | 22 +++------- 11 files changed, 136 insertions(+), 79 deletions(-) create mode 100644 crates/sqlbuild-rules/src/engine/main/query_file_positions.rs delete mode 100644 crates/sqlbuild-rules/src/query_facts/main/query_file_positions.rs diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs index 90ecde4ab7..33d7ed7990 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs @@ -13,8 +13,8 @@ use sqlbuild_analysis::sql_tree::main::sql_tree_nodes::sql_tree_nodes; use sqlbuild_analysis::sql_tree::models::{ NodeExtent, NodeLiteral, NodeName, SqlTreeNode as TreeNode, }; +use sqlbuild_rules::engine::main::query_file_positions::query_file_positions; use sqlbuild_rules::models::ModelQueryPlace; -use sqlbuild_rules::query_facts::main::query_file_positions::query_file_positions; use sqlbuild_python_base::records::main::construct::new_annotated_error::new_annotated_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -180,6 +180,8 @@ pub(crate) struct SqlDocument { dialect: String, /// Where `source`, a model's expanded query, sits in the model file; `None` for other SQL. place: Option, + /// The model file `place` points into. + file: Option, tree: PyOnceLock>, positions: PyOnceLock, } @@ -187,8 +189,13 @@ pub(crate) struct SqlDocument { #[pymethods] impl SqlDocument { #[new] - #[pyo3(signature = (*, source, dialect, place=None))] - fn new(source: String, dialect: String, place: Option<&str>) -> PyResult { + #[pyo3(signature = (*, source, dialect, place=None, file=None))] + fn new( + source: String, + dialect: String, + place: Option<&str>, + file: Option, + ) -> PyResult { let place: Option = place .map(serde_json::from_str) .transpose() @@ -197,6 +204,7 @@ impl SqlDocument { source, dialect, place, + file, tree: PyOnceLock::new(), positions: PyOnceLock::new(), }) @@ -232,7 +240,7 @@ impl SqlDocument { /// Where `node`, read from this model query, is in the model file; `None` for other SQL. #[pyo3(signature = (*, node))] fn file_position(&self, node: &Bound<'_, PyAny>) -> PyResult> { - let Some(place) = self.place.as_ref() else { + let (Some(place), Some(file)) = (self.place.as_ref(), self.file.as_deref()) else { return Ok(None); }; let (Some(line), Some(column)) = (node_index(node, "line")?, node_index(node, "column")?) @@ -241,14 +249,16 @@ impl SqlDocument { }; let positions: &FilePositions = self .positions - .get_or_init(node.py(), || self.file_positions(node.py(), place)); + .get_or_init(node.py(), || self.file_positions(node.py(), (file, place))); let character: usize = positions.character(line, column); Ok(Some( positions .node_starts .get(&character) .copied() - .unwrap_or_else(|| query_file_positions(&self.source, place, &[character])[0]), + .unwrap_or_else(|| { + query_file_positions(&self.source, (file, place), &[character])[0] + }), )) } @@ -290,7 +300,7 @@ impl SqlDocument { } /// Line starts and the file positions of every tree node's start, mapped through `place` at once. - fn file_positions(&self, py: Python<'_>, place: &ModelQueryPlace) -> FilePositions { + fn file_positions(&self, py: Python<'_>, located: (&str, &ModelQueryPlace)) -> FilePositions { let mut line_starts: Vec = vec![0]; let mut length: usize = 0; for character in self.source.chars() { @@ -315,7 +325,7 @@ impl SqlDocument { ); starts.sort_unstable(); starts.dedup(); - let mapped: Vec<(u64, u64)> = query_file_positions(&self.source, place, &starts); + let mapped: Vec<(u64, u64)> = query_file_positions(&self.source, located, &starts); positions.node_starts = starts.into_iter().zip(mapped).collect(); positions } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index c4e5ed2fc0..3b49dc96d9 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -118,24 +118,44 @@ pub(crate) fn declaration(fact: &DeclarationFact) -> Declaration { } /// The expansion passes back to the authored query and where that query starts in the file. -pub(crate) fn authored_query_map(facts: &CompiledModelFacts) -> Option { +fn authored_query_map(facts: &CompiledModelFacts) -> Option { + if facts.authored_query_sql.is_empty() || facts.expansion_passes.iter().all(Vec::is_empty) { + return None; + } + located_query_map( + &facts.authored_sql, + &facts.authored_query_sql, + expansion_spans(&facts.expansion_passes), + ) +} + +/// `passes` in characters, as the rules request carries them. +pub(crate) fn expansion_spans(passes: &[Vec]) -> Vec> { let span = |span: &CharSpan| ExpansionSpan { source_start: character_offset(span.source_start), source_end: character_offset(span.source_end), output_start: character_offset(span.output_start), output_end: character_offset(span.output_end), }; - if facts.authored_query_sql.is_empty() || facts.expansion_passes.iter().all(Vec::is_empty) { + passes + .iter() + .map(|pass| pass.iter().map(span).collect()) + .collect() +} + +/// `passes` back to `authored_query` and where that query starts in `file`. +pub(crate) fn located_query_map( + file: &str, + authored_query: &str, + passes: Vec>, +) -> Option { + if authored_query.is_empty() || passes.iter().all(Vec::is_empty) { return None; } - let start: usize = facts.authored_sql.rfind(&*facts.authored_query_sql)?; - let above: &str = &facts.authored_sql[..start]; + let start: usize = file.rfind(authored_query)?; + let above: &str = &file[..start]; Some(AuthoredQueryMap { - passes: facts - .expansion_passes - .iter() - .map(|pass| pass.iter().map(span).collect()) - .collect(), + passes, line_offset: above.matches('\n').count() as u64, column_offset: above .rsplit('\n') diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index 6f601523a8..f0eba93ac2 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -9,5 +9,6 @@ pub mod json_grains; pub mod model_query_place; pub mod needs_json_grains; pub mod project_model_rows; +pub mod query_file_positions; pub mod retained_request; pub mod start_speculative_rows; diff --git a/crates/sqlbuild-rules/src/engine/main/model_query_place.rs b/crates/sqlbuild-rules/src/engine/main/model_query_place.rs index aab857992e..3628a3d3c1 100644 --- a/crates/sqlbuild-rules/src/engine/main/model_query_place.rs +++ b/crates/sqlbuild-rules/src/engine/main/model_query_place.rs @@ -1,26 +1,31 @@ -//! Where a compiled model's expanded query sits in its file. +//! What places a compiled model's expanded query in its file. use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; -use crate::engine::_helpers::project_rows::{ - authored_query_map, query_column_offset, query_line_offset, -}; -use crate::models::ModelQueryPlace; +use crate::engine::_helpers::project_rows::expansion_spans; +use crate::models::{AuthoredQuery, ModelQueryPlace}; -/// The file place of `facts`' expanded query, read the way built-in model rules read it. +/// Where `facts`' authored query is and its expansion passes; finding the query is a suffix check. #[must_use] pub fn model_query_place(facts: &CompiledModelFacts) -> ModelQueryPlace { let file: &str = &facts.authored_sql; - let authored_query: &str = &facts.authored_query_sql; + let query: &str = &facts.authored_query_sql; + let start: Option = if file.ends_with(query) { + Some(file.len() - query.len()) + } else { + file.rfind(query) + }; ModelQueryPlace { - authored_sql: if authored_query.is_empty() { - file + authored_query: (!query.is_empty()).then(|| { + start.map_or_else( + || AuthoredQuery::Text(query.to_owned()), + |start| AuthoredQuery::Range(start, start + query.len()), + ) + }), + passes: if facts.expansion_passes.iter().all(Vec::is_empty) { + Vec::new() } else { - authored_query - } - .to_owned(), - line_offset: query_line_offset(file, authored_query, &facts.query_sql), - column_offset: query_column_offset(file, authored_query, &facts.query_sql), - authored_query: authored_query_map(facts), + expansion_spans(&facts.expansion_passes) + }, } } diff --git a/crates/sqlbuild-rules/src/engine/main/query_file_positions.rs b/crates/sqlbuild-rules/src/engine/main/query_file_positions.rs new file mode 100644 index 0000000000..5e65292cc6 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/query_file_positions.rs @@ -0,0 +1,44 @@ +//! The model-file positions of characters of a model's expanded query. + +use crate::engine::_helpers::project_rows::{ + located_query_map, query_column_offset, query_line_offset, +}; +use crate::models::{AuthoredQuery, AuthoredQueryMap, ModelQueryPlace}; +use crate::query_facts::models::ModelSource; + +/// The 1-based `file` line and column of each of `sql`'s `characters`, as built-in rules map them. +#[must_use] +pub fn query_file_positions( + sql: &str, + (file, place): (&str, &ModelQueryPlace), + characters: &[usize], +) -> Vec<(u64, u64)> { + let authored_query: &str = match &place.authored_query { + None => "", + Some(AuthoredQuery::Range(start, end)) => file.get(*start..*end).unwrap_or_default(), + Some(AuthoredQuery::Text(text)) => text, + }; + let map: Option = + located_query_map(file, authored_query, place.passes.clone()); + let offsets: Vec = (0..=sql.chars().count()).collect(); + let source: ModelSource<'_> = ModelSource::new( + sql, + &offsets, + ( + query_line_offset(file, authored_query, sql), + query_column_offset(file, authored_query, sql), + ), + ) + .with_authored( + if authored_query.is_empty() { + file + } else { + authored_query + }, + map.as_ref(), + ); + characters + .iter() + .map(|character| source.line_column(*character)) + .collect() +} diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 96ea05491c..6e4b4fc618 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -952,17 +952,21 @@ impl fensu_policy::policy::types::PolicyRule for RuleMetadata { } } -/// Where a model's expanded query sits in its file, as built-in rules map query positions. +/// What places a model's expanded query in its file, mapped only when a rule reports a node. #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] pub struct ModelQueryPlace { - /// The authored query, or the whole file when the model has none. - pub authored_sql: String, - /// The lines of the model file above the query. - pub line_offset: u64, - /// The file-line characters before the query's first content that the query itself lacks. - pub column_offset: u64, - /// The expansion passes back to the authored query, when expansion changed it. - pub authored_query: Option, + /// The authored query below the model's header; `None` when the model has none. + pub authored_query: Option, + /// The expansion passes from the authored query to the expanded query; empty when unknown. + pub passes: Vec>, +} + +/// A model's authored query: its byte range in the model file, or its text when the file lacks it. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuthoredQuery { + Range(usize, usize), + Text(String), } /// The expansion passes from a model's authored query to `query_sql`, and the query's file place. diff --git a/crates/sqlbuild-rules/src/query_facts/main/mod.rs b/crates/sqlbuild-rules/src/query_facts/main/mod.rs index ad572edc3c..a4b8b0f0c7 100644 --- a/crates/sqlbuild-rules/src/query_facts/main/mod.rs +++ b/crates/sqlbuild-rules/src/query_facts/main/mod.rs @@ -3,5 +3,4 @@ pub mod facts_available; pub mod model_shape; pub mod parse_fixture; pub mod query_facts; -pub mod query_file_positions; pub mod stub_call; diff --git a/crates/sqlbuild-rules/src/query_facts/main/query_file_positions.rs b/crates/sqlbuild-rules/src/query_facts/main/query_file_positions.rs deleted file mode 100644 index f4750564ea..0000000000 --- a/crates/sqlbuild-rules/src/query_facts/main/query_file_positions.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! The model-file positions of characters of a model's expanded query. - -use crate::models::ModelQueryPlace; -use crate::query_facts::models::ModelSource; - -/// The 1-based file line and column of each of `sql`'s `characters`; expansion output maps to its call. -#[must_use] -pub fn query_file_positions( - sql: &str, - place: &ModelQueryPlace, - characters: &[usize], -) -> Vec<(u64, u64)> { - let offsets: Vec = (0..=sql.chars().count()).collect(); - let source: ModelSource<'_> = - ModelSource::new(sql, &offsets, (place.line_offset, place.column_offset)) - .with_authored(&place.authored_sql, place.authored_query.as_ref()); - characters - .iter() - .map(|character| source.line_column(*character)) - .collect() -} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 0ec799890f..a740e94411 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4323,7 +4323,9 @@ class SqlDocument: """One SQL text a custom rule reads, parsed on first use by its dialect's SQLQuack parser.""" source: str - def __init__(self, *, source: str, dialect: str, place: str | None = None) -> None: ... + def __init__( + self, *, source: str, dialect: str, place: str | None = None, file: str | None = None + ) -> None: ... def ast(self) -> SqlTreeNode: ... def file_position(self, *, node: object) -> tuple[int, int] | None: ... def nodes(self, *, kind: str) -> tuple[SqlNode, ...]: ... diff --git a/src/sqlbuild/rule_engine/classes/sql_facts.py b/src/sqlbuild/rule_engine/classes/sql_facts.py index 984aa1cd8a..458a1b3cec 100644 --- a/src/sqlbuild/rule_engine/classes/sql_facts.py +++ b/src/sqlbuild/rule_engine/classes/sql_facts.py @@ -29,6 +29,7 @@ def for_model(self, model: Model) -> ModelSql: source=compiled.query_sql, dialect=self._dialect, place=getattr(compiled, "query_place", None), + file=compiled.authored_sql, ), ) self._documents[model.path] = documents diff --git a/tests/unit/src/sqlbuild/rules/test_sql_document.py b/tests/unit/src/sqlbuild/rules/test_sql_document.py index bf0734c7e5..1275590e2e 100644 --- a/tests/unit/src/sqlbuild/rules/test_sql_document.py +++ b/tests/unit/src/sqlbuild/rules/test_sql_document.py @@ -63,15 +63,13 @@ def test_given_wide_model_query_when_placing_every_column_then_time_stays_linear source: str = ( "SELECT\n" + ",\n".join(f" c{index}" for index in range(test_case.columns)) + "\nFROM t" ) + file: str = "\n" * test_case.line_offset + source place: str = json.dumps( - { - "authored_sql": source, - "line_offset": test_case.line_offset, - "column_offset": 0, - "authored_query": None, - } + {"authored_query": {"range": [test_case.line_offset, len(file)]}, "passes": []} + ) + document: SqlDocument = SqlDocument( + source=source, dialect=test_case.dialect, place=place, file=file ) - document: SqlDocument = SqlDocument(source=source, dialect=test_case.dialect, place=place) started: float = time.perf_counter() positions: list[tuple[int, int] | None] = [ @@ -108,14 +106,8 @@ def test_given_node_position_when_placing_it_then_only_integers_map_into_the_fil document: SqlDocument = SqlDocument( source="SELECT 1", dialect="duckdb", - place=json.dumps( - { - "authored_sql": "SELECT 1", - "line_offset": 2, - "column_offset": 0, - "authored_query": None, - } - ), + place=json.dumps({"authored_query": {"range": [2, 10]}, "passes": []}), + file="\n\nSELECT 1", ) node: SimpleNamespace = SimpleNamespace(line=test_case.line, column=test_case.column)