diff --git a/benchmarks/duckshop/project/rules/quoted_aliases.py b/benchmarks/duckshop/project/rules/quoted_aliases.py index bfb855b1ce..292cf6ee24 100644 --- a/benchmarks/duckshop/project/rules/quoted_aliases.py +++ b/benchmarks/duckshop/project/rules/quoted_aliases.py @@ -29,6 +29,7 @@ def mixed_case_names_stay_in_exports(*, model: Model, ctx: RuleContext) -> list[ findings.append( ctx.finding( subject=model, + node=node, message=f"Quoted mixed-case name {node.name!r} in `{text}`", ) ) diff --git a/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py index 9a6079175b..1303b6da7f 100644 --- a/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py +++ b/benchmarks/duckshop/project/tests/test_rule_quoted_aliases.py @@ -30,6 +30,7 @@ def test_given_quoted_mixed_case_alias_outside_an_export_when_evaluating_then_re ) assert result.finding_count == 1 + assert (result.findings[0].line, result.findings[0].column) == (3, 3) assert result.findings[0].message == ( "Quoted mixed-case name 'listingId' in `l.listing_id AS \"listingId\"`" ) diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs index 0a6cc63b53..5e4ab4b24c 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_host_payload.rs @@ -212,6 +212,8 @@ impl PayloadWriter { self.part(TEXT_TAG, authored.as_bytes())?, ); model.insert("query".to_owned(), self.part(TEXT_TAG, query.as_bytes())?); + let place: String = view.query_place(py)?; + model.insert("place".to_owned(), self.part(TEXT_TAG, place.as_bytes())?); let inferred: Value = match view.inferred_rows(py)? { Some(rows) => self.part(INFERRED_TAG, &json_bytes(&rows)?)?, None => Value::Null, @@ -242,6 +244,7 @@ impl PayloadWriter { model.insert("authored".to_owned(), authored); let query: Value = self.text(&native_field(record, "query_sql")?)?; model.insert("query".to_owned(), query); + model.insert("place".to_owned(), Value::Null); let inferred: Bound<'_, PyAny> = native_field(record, "inferred_columns")?; let inferred: Value = if inferred.is_none() { Value::Null diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs index a12a8f9472..c49145ecd6 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/fact/fact_replay.rs @@ -431,6 +431,7 @@ impl<'py> FactReplay<'py> { JsonValue::Array(vec![ JsonValue::String(authored), JsonValue::String(expanded), + JsonValue::String(view.query_place(py)?), ]) } None => { diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs index 9350081971..33d7ed7990 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs @@ -1,8 +1,10 @@ //! Custom rules' SQL documents: the SQLQuack syntax tree of one SQL text and its common structures. +use std::collections::HashMap; use std::sync::Arc; -use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyResult, Python}; +use pyo3::exceptions::{PyAttributeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; use pyo3::{IntoPyObject, pyclass, pymethods}; @@ -11,9 +13,12 @@ use sqlbuild_analysis::sql_tree::main::sql_tree_nodes::sql_tree_nodes; use sqlbuild_analysis::sql_tree::models::{ NodeExtent, NodeLiteral, NodeName, SqlTreeNode as TreeNode, }; +use sqlbuild_rules::engine::main::query_file_positions::query_file_positions; +use sqlbuild_rules::models::ModelQueryPlace; use sqlbuild_python_base::records::main::construct::new_annotated_error::new_annotated_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; @@ -29,6 +34,14 @@ struct Tree { children: Vec>, } +/// A model query's character offsets and the file positions of its tree's node starts, built once. +struct FilePositions { + /// The character each line starts at, then the source's length. + line_starts: Vec, + /// The file `(line, column)` of every tree node's first character, by that character. + node_starts: HashMap, +} + /// One node of a rule's SQL syntax tree: an index into its shared tree, so freeing never recurses. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct SqlTreeNode { @@ -165,19 +178,36 @@ pub(crate) struct SqlDocument { #[pyo3(get)] source: String, dialect: String, + /// Where `source`, a model's expanded query, sits in the model file; `None` for other SQL. + place: Option, + /// The model file `place` points into. + file: Option, tree: PyOnceLock>, + positions: PyOnceLock, } #[pymethods] impl SqlDocument { #[new] - #[pyo3(signature = (*, source, dialect))] - fn new(source: String, dialect: String) -> Self { - Self { + #[pyo3(signature = (*, source, dialect, place=None, file=None))] + fn new( + source: String, + dialect: String, + place: Option<&str>, + file: Option, + ) -> PyResult { + let place: Option = place + .map(serde_json::from_str) + .transpose() + .map_err(|error| PyValueError::new_err(format!("invalid query place: {error}")))?; + Ok(Self { source, dialect, + place, + file, tree: PyOnceLock::new(), - } + positions: PyOnceLock::new(), + }) } /// The syntax tree: a `document` node whose children are the statements. @@ -207,6 +237,31 @@ impl SqlDocument { self.nodes(py, "Subquery") } + /// Where `node`, read from this model query, is in the model file; `None` for other SQL. + #[pyo3(signature = (*, node))] + fn file_position(&self, node: &Bound<'_, PyAny>) -> PyResult> { + let (Some(place), Some(file)) = (self.place.as_ref(), self.file.as_deref()) else { + return Ok(None); + }; + let (Some(line), Some(column)) = (node_index(node, "line")?, node_index(node, "column")?) + else { + return Ok(None); + }; + let positions: &FilePositions = self + .positions + .get_or_init(node.py(), || self.file_positions(node.py(), (file, place))); + let character: usize = positions.character(line, column); + Ok(Some( + positions + .node_starts + .get(&character) + .copied() + .unwrap_or_else(|| { + query_file_positions(&self.source, (file, place), &[character])[0] + }), + )) + } + /// Every projected star, bare or qualified; a star a function call takes, such as `COUNT(*)`, is no projection. fn star_projections<'py>(&self, py: Python<'py>) -> PyResult> { self.source_nodes(py, "Star", |nodes, node| { @@ -244,6 +299,37 @@ impl SqlDocument { PyTuple::new(py, found) } + /// Line starts and the file positions of every tree node's start, mapped through `place` at once. + fn file_positions(&self, py: Python<'_>, located: (&str, &ModelQueryPlace)) -> FilePositions { + let mut line_starts: Vec = vec![0]; + let mut length: usize = 0; + for character in self.source.chars() { + length += 1; + if character == '\n' { + line_starts.push(length); + } + } + line_starts.push(length); + let mut positions: FilePositions = FilePositions { + line_starts, + node_starts: HashMap::new(), + }; + let mut starts: Vec = self.parsed(py).map_or_else( + |_| Vec::new(), + |tree| { + tree.nodes + .iter() + .map(|node| positions.character(node.line, node.column)) + .collect() + }, + ); + starts.sort_unstable(); + starts.dedup(); + let mapped: Vec<(u64, u64)> = query_file_positions(&self.source, located, &starts); + positions.node_starts = starts.into_iter().zip(mapped).collect(); + positions + } + /// The parsed tree; a failure raises `RuleUsageError` each time it is read. fn parsed(&self, py: Python<'_>) -> PyResult<&Arc> { self.tree.get_or_try_init(py, || { @@ -253,6 +339,30 @@ impl SqlDocument { } } +impl FilePositions { + /// The source character at 1-based `line` and `column`; a line past the end starts at the end. + fn character(&self, line: usize, column: usize) -> usize { + let last: usize = self.line_starts.len() - 1; + self.line_starts[line.saturating_sub(1).min(last)] + column.saturating_sub(1) + } +} + +/// A node's 1-based `line` or `column` attribute; `None` when it has none or it is not a positive integer. +fn node_index(node: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let value: i64 = match native_field(node, name) { + Ok(value) => match value.extract() { + Ok(value) => value, + Err(_) => return Ok(None), + }, + Err(error) if error.is_instance_of::(node.py()) => return Ok(None), + Err(error) => return Err(error), + }; + Ok(match usize::try_from(value) { + Ok(index) if index > 0 => Some(index), + _ => None, + }) +} + /// `nodes` with each node's children listed, in ascending index order, under its parent. fn tree(nodes: Vec) -> Arc { let mut children: Vec> = vec![Vec::new(); nodes.len() + 1]; diff --git a/crates/sqlbuild-python-project/src/compiled_views/models.rs b/crates/sqlbuild-python-project/src/compiled_views/models.rs index 28438e56d5..dc513bdecb 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/models.rs @@ -1,5 +1,8 @@ //! Public records of compiled views. +use sqlbuild_rules::engine::main::model_query_place::model_query_place; +use sqlbuild_rules::models::ModelQueryPlace; + use crate::compiled_project::main::reference_row_tuple::reference_row_tuple; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_project::models::ReferenceTuple; @@ -420,12 +423,23 @@ impl NativeCompiledModel { /// The model's authored file text and expanded query, from its recorded facts. pub fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> { + self.with_facts(py, |facts| { + (facts.authored_sql.to_string(), facts.query_sql.to_string()) + }) + } + + /// `read` applied to the model's recorded facts. + pub fn with_facts( + &self, + py: Python<'_>, + read: impl FnOnce(&CompiledModelFacts) -> T, + ) -> PyResult { let name: String = self.name(py)?; let owner = self.owner.bind(py).borrow(); let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") })?; - Ok((facts.authored_sql.to_string(), facts.query_sql.to_string())) + Ok(read(facts)) } pub(crate) fn model_file( @@ -685,6 +699,14 @@ impl NativeCompiledModel { self.model_file(py, |file| Ok(file.contents(py))) } + /// Where the expanded query sits in the model file, as JSON a rule's SQL document maps positions through. + #[getter] + pub fn query_place(&self, py: Python<'_>) -> PyResult { + let place: ModelQueryPlace = self.with_facts(py, model_query_place)?; + serde_json::to_string(&place) + .map_err(|error| pyo3::exceptions::PyRuntimeError::new_err(error.to_string())) + } + #[getter] pub fn authored_query_sql<'py>(&self, py: Python<'py>) -> PyResult> { self.model_file(py, |file| Ok(file.query_sql(py))) diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index cffac3626c..3b49dc96d9 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -119,23 +119,43 @@ pub(crate) fn declaration(fact: &DeclarationFact) -> Declaration { /// The expansion passes back to the authored query and where that query starts in the file. fn authored_query_map(facts: &CompiledModelFacts) -> Option { + if facts.authored_query_sql.is_empty() || facts.expansion_passes.iter().all(Vec::is_empty) { + return None; + } + located_query_map( + &facts.authored_sql, + &facts.authored_query_sql, + expansion_spans(&facts.expansion_passes), + ) +} + +/// `passes` in characters, as the rules request carries them. +pub(crate) fn expansion_spans(passes: &[Vec]) -> Vec> { let span = |span: &CharSpan| ExpansionSpan { source_start: character_offset(span.source_start), source_end: character_offset(span.source_end), output_start: character_offset(span.output_start), output_end: character_offset(span.output_end), }; - if facts.authored_query_sql.is_empty() || facts.expansion_passes.iter().all(Vec::is_empty) { + passes + .iter() + .map(|pass| pass.iter().map(span).collect()) + .collect() +} + +/// `passes` back to `authored_query` and where that query starts in `file`. +pub(crate) fn located_query_map( + file: &str, + authored_query: &str, + passes: Vec>, +) -> Option { + if authored_query.is_empty() || passes.iter().all(Vec::is_empty) { return None; } - let start: usize = facts.authored_sql.rfind(&*facts.authored_query_sql)?; - let above: &str = &facts.authored_sql[..start]; + let start: usize = file.rfind(authored_query)?; + let above: &str = &file[..start]; Some(AuthoredQueryMap { - passes: facts - .expansion_passes - .iter() - .map(|pass| pass.iter().map(span).collect()) - .collect(), + passes, line_offset: above.matches('\n').count() as u64, column_offset: above .rsplit('\n') @@ -151,7 +171,7 @@ fn character_offset(offset: usize) -> i64 { } /// The file-line characters before the query's first content that the query itself lacks. -fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 { +pub(crate) fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 { let line_prefix = |text: &str, at: usize| { text[..at] .rsplit('\n') @@ -172,7 +192,7 @@ fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 { } /// The lines of `file` above `query`, the rendered form of its `authored_query`; 0 if unknown. -fn query_line_offset(file: &str, authored_query: &str, query: &str) -> u64 { +pub(crate) fn query_line_offset(file: &str, authored_query: &str, query: &str) -> u64 { let leading_lines = |text: &str| { let body: &str = text.trim_start(); text[..text.len() - body.len()].matches('\n').count() diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index 1acb27f0d2..f0eba93ac2 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -6,7 +6,9 @@ pub mod evaluate_rows; pub mod finalize_rows; pub mod json_grain_skipped_models; pub mod json_grains; +pub mod model_query_place; pub mod needs_json_grains; pub mod project_model_rows; +pub mod query_file_positions; pub mod retained_request; pub mod start_speculative_rows; diff --git a/crates/sqlbuild-rules/src/engine/main/model_query_place.rs b/crates/sqlbuild-rules/src/engine/main/model_query_place.rs new file mode 100644 index 0000000000..3628a3d3c1 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/model_query_place.rs @@ -0,0 +1,31 @@ +//! What places a compiled model's expanded query in its file. + +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; + +use crate::engine::_helpers::project_rows::expansion_spans; +use crate::models::{AuthoredQuery, ModelQueryPlace}; + +/// Where `facts`' authored query is and its expansion passes; finding the query is a suffix check. +#[must_use] +pub fn model_query_place(facts: &CompiledModelFacts) -> ModelQueryPlace { + let file: &str = &facts.authored_sql; + let query: &str = &facts.authored_query_sql; + let start: Option = if file.ends_with(query) { + Some(file.len() - query.len()) + } else { + file.rfind(query) + }; + ModelQueryPlace { + authored_query: (!query.is_empty()).then(|| { + start.map_or_else( + || AuthoredQuery::Text(query.to_owned()), + |start| AuthoredQuery::Range(start, start + query.len()), + ) + }), + passes: if facts.expansion_passes.iter().all(Vec::is_empty) { + Vec::new() + } else { + expansion_spans(&facts.expansion_passes) + }, + } +} diff --git a/crates/sqlbuild-rules/src/engine/main/query_file_positions.rs b/crates/sqlbuild-rules/src/engine/main/query_file_positions.rs new file mode 100644 index 0000000000..5e65292cc6 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/query_file_positions.rs @@ -0,0 +1,44 @@ +//! The model-file positions of characters of a model's expanded query. + +use crate::engine::_helpers::project_rows::{ + located_query_map, query_column_offset, query_line_offset, +}; +use crate::models::{AuthoredQuery, AuthoredQueryMap, ModelQueryPlace}; +use crate::query_facts::models::ModelSource; + +/// The 1-based `file` line and column of each of `sql`'s `characters`, as built-in rules map them. +#[must_use] +pub fn query_file_positions( + sql: &str, + (file, place): (&str, &ModelQueryPlace), + characters: &[usize], +) -> Vec<(u64, u64)> { + let authored_query: &str = match &place.authored_query { + None => "", + Some(AuthoredQuery::Range(start, end)) => file.get(*start..*end).unwrap_or_default(), + Some(AuthoredQuery::Text(text)) => text, + }; + let map: Option = + located_query_map(file, authored_query, place.passes.clone()); + let offsets: Vec = (0..=sql.chars().count()).collect(); + let source: ModelSource<'_> = ModelSource::new( + sql, + &offsets, + ( + query_line_offset(file, authored_query, sql), + query_column_offset(file, authored_query, sql), + ), + ) + .with_authored( + if authored_query.is_empty() { + file + } else { + authored_query + }, + map.as_ref(), + ); + characters + .iter() + .map(|character| source.line_column(*character)) + .collect() +} diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index bd71cf5cbe..6e4b4fc618 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -952,6 +952,23 @@ impl fensu_policy::policy::types::PolicyRule for RuleMetadata { } } +/// What places a model's expanded query in its file, mapped only when a rule reports a node. +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct ModelQueryPlace { + /// The authored query below the model's header; `None` when the model has none. + pub authored_query: Option, + /// The expansion passes from the authored query to the expanded query; empty when unknown. + pub passes: Vec>, +} + +/// A model's authored query: its byte range in the model file, or its text when the file lacks it. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuthoredQuery { + Range(usize, usize), + Text(String), +} + /// The expansion passes from a model's authored query to `query_sql`, and the query's file place. #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] pub struct AuthoredQueryMap { diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md index b855a669d6..3293adb007 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/rule-context.md @@ -64,6 +64,9 @@ for star in sql.expanded.star_projections(): text, not the model file. Authored SQL is the whole model file, including its `MODEL` header, so it does not parse as SQL; read its text through `source`. +`ctx.finding(subject=model, node=node)` reports a node of that model's expanded SQL where it is in +the model file. A node inside a macro's output is reported at the macro call. + ## Declared enum types `ctx.columns.declared(model)` reports each declared column with its portable scalar type, so an @@ -86,7 +89,7 @@ sql = ctx.sql.for_model(model).expanded for node in sql.ast().walk(): if node.kind == "alias" and node.quoted and node.name != node.name.lower(): start, end = node.span - return [ctx.finding(subject=model, message=sql.source[start:end])] + return [ctx.finding(subject=model, node=node, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 798647548d..a740e94411 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1416,6 +1416,8 @@ class NativeCompiledModel: @property def authored_sql(self) -> str: ... @property + def query_place(self) -> str: ... + @property def authored_query_sql(self) -> str: ... @property def output_column_locations(self) -> dict[str, SourceLocation]: ... @@ -4321,8 +4323,11 @@ class SqlDocument: """One SQL text a custom rule reads, parsed on first use by its dialect's SQLQuack parser.""" source: str - def __init__(self, *, source: str, dialect: str) -> None: ... + def __init__( + self, *, source: str, dialect: str, place: str | None = None, file: str | None = None + ) -> None: ... def ast(self) -> SqlTreeNode: ... + def file_position(self, *, node: object) -> tuple[int, int] | None: ... def nodes(self, *, kind: str) -> tuple[SqlNode, ...]: ... def ctes(self) -> tuple[SqlNode, ...]: ... def joins(self) -> tuple[SqlNode, ...]: ... diff --git a/src/sqlbuild/rule_engine/_helpers/host/host_payload.py b/src/sqlbuild/rule_engine/_helpers/host/host_payload.py index 46228fcec7..05766a9a0b 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/host_payload.py +++ b/src/sqlbuild/rule_engine/_helpers/host/host_payload.py @@ -101,6 +101,10 @@ def authored_sql(self) -> str: def query_sql(self) -> str: return self._payload.value(self._entry["query"]) + @cached_property + def query_place(self) -> str | None: + return self._payload.value(self._entry["place"]) + def config_value(self, name: str) -> object: """Return `config.values.get(name)` for one config value rule facts read.""" diff --git a/src/sqlbuild/rule_engine/classes/rule_context.py b/src/sqlbuild/rule_engine/classes/rule_context.py index 5700d2614b..5b76e360ba 100644 --- a/src/sqlbuild/rule_engine/classes/rule_context.py +++ b/src/sqlbuild/rule_engine/classes/rule_context.py @@ -110,6 +110,11 @@ def finding( path: Path if isinstance(subject, Model): path = subject.path + if node is not None and line is None and column is None: + in_file: tuple[int, int] | None = self.sql.for_model( + subject + ).expanded.file_position(node=node) + line, column = in_file or (None, None) elif isinstance(subject, ProjectPath): path = Path(subject.value) else: diff --git a/src/sqlbuild/rule_engine/classes/sql_facts.py b/src/sqlbuild/rule_engine/classes/sql_facts.py index 328bf3ec1c..458a1b3cec 100644 --- a/src/sqlbuild/rule_engine/classes/sql_facts.py +++ b/src/sqlbuild/rule_engine/classes/sql_facts.py @@ -25,7 +25,12 @@ def for_model(self, model: Model) -> ModelSql: compiled: RuleFactModel = self._resolve(model) documents: ModelSql = ModelSql( authored=SqlDocument(source=compiled.authored_sql, dialect=self._dialect), - expanded=SqlDocument(source=compiled.query_sql, dialect=self._dialect), + expanded=SqlDocument( + source=compiled.query_sql, + dialect=self._dialect, + place=getattr(compiled, "query_place", None), + file=compiled.authored_sql, + ), ) self._documents[model.path] = documents return documents diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py index 309675d9c4..adcbf5f610 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py @@ -63,6 +63,18 @@ ), ) +_NODE_RULE: str = custom_rule_source( + code="XSQBRTREE003", + body=( + " sql = ctx.sql.for_model(model).expanded\n" + ' aliases = [node for node in sql.ast().walk() if node.kind == "alias"]\n' + " return [\n" + " ctx.finding(subject=model, node=node, message=node.kind)\n" + " for node in (*sql.star_projections(), *aliases)\n" + " ]\n" + ), +) + @pytest.mark.parametrize( "test_case", @@ -251,3 +263,53 @@ def test_given_custom_rule_reading_node_facts_when_compiling_then_it_sees_the_au assert tuple(sorted(item["message"] for item in custom_rule_diagnostics(result))) == ( test_case.expected_messages ), result.stdout + result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + SyntaxTreeRuleE2ETestCase( + description="a multi-line MODEL header puts nodes on their file lines", + model_sql=( + 'MODEL (\n description "Orders",\n materialized table\n);\n\n' + "SELECT\n *\nFROM (SELECT 1 AS id) AS o\n" + ), + expected_messages=("Star@7:3", "alias@8:14"), + ), + SyntaxTreeRuleE2ETestCase( + description="a multi-line macro expansion above a node leaves it on its file line", + model_sql=( + 'MODEL (\n description "Orders"\n);\n' + "SELECT\n @sql.blank_to_null('a') AS a,\n *\nFROM (SELECT 'x' AS b) AS o\n" + ), + other_models=_BLANK_TO_NULL, + expected_messages=("Star@6:3", "alias@5:3", "alias@7:14"), + ), + ], + ids=lambda case: case.description, +) +def test_given_custom_rule_finding_at_a_node_when_compiling_then_it_points_at_the_model_file( + test_case: SyntaxTreeRuleE2ETestCase, tmp_path: Path +) -> None: + write_custom_rule_project( + project_dir=tmp_path, + selected_rules=("XSQBRTREE003",), + files=(("rules/nodes.py", _NODE_RULE), *test_case.other_models.items()), + configuration="\n[scopes]\nenforce_placement = false\n", + adapter=test_case.adapter, + ) + (tmp_path / "models" / "orders.sql").write_text(test_case.model_sql, encoding="utf-8") + + cold: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + warm: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + for result in (cold, warm): + assert ( + tuple( + sorted( + f"{item['message']}@{item['line']}:{item['column']}" + for item in custom_rule_diagnostics(result) + ) + ) + == test_case.expected_messages + ), result.stdout + result.stderr diff --git a/tests/unit/src/sqlbuild/rules/_test_types.py b/tests/unit/src/sqlbuild/rules/_test_types.py index 2cffe01e63..0b98fad832 100644 --- a/tests/unit/src/sqlbuild/rules/_test_types.py +++ b/tests/unit/src/sqlbuild/rules/_test_types.py @@ -11,3 +11,24 @@ class DeepTreeTestCase: dialect: str terms: int expected_node_count: int + + +@dataclass(frozen=True) +class FilePositionTimeLimitTestCase: + """A wide model query whose every column a rule reports at its model-file position.""" + + description: str + dialect: str + columns: int + line_offset: int + expected_max_seconds: float + + +@dataclass(frozen=True) +class UnplacedNodeTestCase: + """A node-like value a rule reports, placed in a model file only with integer positions.""" + + description: str + line: object + column: object + expected_position: tuple[int, int] | None diff --git a/tests/unit/src/sqlbuild/rules/test_sql_document.py b/tests/unit/src/sqlbuild/rules/test_sql_document.py index 77ec496732..1275590e2e 100644 --- a/tests/unit/src/sqlbuild/rules/test_sql_document.py +++ b/tests/unit/src/sqlbuild/rules/test_sql_document.py @@ -1,11 +1,18 @@ """Behavior tests for the syntax tree custom rules read through `SqlDocument`.""" import gc +import json +import time +from types import SimpleNamespace import pytest from sqlbuild.rules import SqlDocument, SqlTreeNode -from tests.unit.src.sqlbuild.rules._test_types import DeepTreeTestCase +from tests.unit.src.sqlbuild.rules._test_types import ( + DeepTreeTestCase, + FilePositionTimeLimitTestCase, + UnplacedNodeTestCase, +) @pytest.mark.parametrize( @@ -35,3 +42,73 @@ def test_given_deep_query_when_reading_and_freeing_its_tree_then_the_process_sur _ = gc.collect() assert walked == test_case.expected_node_count + + +@pytest.mark.parametrize( + "test_case", + [ + FilePositionTimeLimitTestCase( + description="20,000 columns, one per line, five header lines", + dialect="duckdb", + columns=20_000, + line_offset=5, + expected_max_seconds=2.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_wide_model_query_when_placing_every_column_then_time_stays_linear( + test_case: FilePositionTimeLimitTestCase, +) -> None: + source: str = ( + "SELECT\n" + ",\n".join(f" c{index}" for index in range(test_case.columns)) + "\nFROM t" + ) + file: str = "\n" * test_case.line_offset + source + place: str = json.dumps( + {"authored_query": {"range": [test_case.line_offset, len(file)]}, "passes": []} + ) + document: SqlDocument = SqlDocument( + source=source, dialect=test_case.dialect, place=place, file=file + ) + + started: float = time.perf_counter() + positions: list[tuple[int, int] | None] = [ + document.file_position(node=node) for node in document.nodes(kind="Column") + ] + + assert time.perf_counter() - started < test_case.expected_max_seconds + assert positions == [ + (index + 2 + test_case.line_offset, 3) for index in range(test_case.columns) + ] + + +@pytest.mark.parametrize( + "test_case", + [ + UnplacedNodeTestCase( + description="integer line and column", + line=1, + column=8, + expected_position=(3, 8), + ), + UnplacedNodeTestCase( + description="no line or column", line=None, column=None, expected_position=None + ), + UnplacedNodeTestCase( + description="text line and column", line="1", column="8", expected_position=None + ), + ], + ids=lambda case: case.description, +) +def test_given_node_position_when_placing_it_then_only_integers_map_into_the_file( + test_case: UnplacedNodeTestCase, +) -> None: + document: SqlDocument = SqlDocument( + source="SELECT 1", + dialect="duckdb", + place=json.dumps({"authored_query": {"range": [2, 10]}, "passes": []}), + file="\n\nSELECT 1", + ) + + node: SimpleNamespace = SimpleNamespace(line=test_case.line, column=test_case.column) + assert document.file_position(node=node) == test_case.expected_position diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx index e715257b94..3e3e671cae 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/rule-context.mdx @@ -52,6 +52,9 @@ for star in sql.expanded.star_projections(): text, not the model file. Authored SQL is the whole model file, including its `MODEL` header, so it does not parse as SQL; read its text through `source`. +`ctx.finding(subject=model, node=node)` reports a node of that model's expanded SQL where it is in +the model file. A node inside a macro's output is reported at the macro call. + ## Declared enum types `ctx.columns.declared(model)` reports each declared column with its portable scalar type, so an @@ -74,7 +77,7 @@ sql = ctx.sql.for_model(model).expanded for node in sql.ast().walk(): if node.kind == "alias" and node.quoted and node.name != node.name.lower(): start, end = node.span - return [ctx.finding(subject=model, message=sql.source[start:end])] + return [ctx.finding(subject=model, node=node, message=sql.source[start:end])] ``` `ast()` returns a `document` node whose children are the statements. Every `SqlTreeNode` has a