Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions benchmarks/duckshop/project/rules/quoted_aliases.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def mixed_case_names_stay_in_exports(*, model: Model, ctx: RuleContext) -> list[
findings.append(
ctx.finding(
subject=model,
node=node,
message=f"Quoted mixed-case name {node.name!r} in `{text}`",
)
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ def test_given_quoted_mixed_case_alias_outside_an_export_when_evaluating_then_re
)

assert result.finding_count == 1
assert (result.findings[0].line, result.findings[0].column) == (3, 3)
assert result.findings[0].message == (
"Quoted mixed-case name 'listingId' in `l.listing_id AS \"listingId\"`"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,8 @@ impl<W: Write> PayloadWriter<W> {
self.part(TEXT_TAG, authored.as_bytes())?,
);
model.insert("query".to_owned(), self.part(TEXT_TAG, query.as_bytes())?);
let place: String = view.query_place(py)?;
model.insert("place".to_owned(), self.part(TEXT_TAG, place.as_bytes())?);
let inferred: Value = match view.inferred_rows(py)? {
Some(rows) => self.part(INFERRED_TAG, &json_bytes(&rows)?)?,
None => Value::Null,
Expand Down Expand Up @@ -242,6 +244,7 @@ impl<W: Write> PayloadWriter<W> {
model.insert("authored".to_owned(), authored);
let query: Value = self.text(&native_field(record, "query_sql")?)?;
model.insert("query".to_owned(), query);
model.insert("place".to_owned(), Value::Null);
let inferred: Bound<'_, PyAny> = native_field(record, "inferred_columns")?;
let inferred: Value = if inferred.is_none() {
Value::Null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,7 @@ impl<'py> FactReplay<'py> {
JsonValue::Array(vec![
JsonValue::String(authored),
JsonValue::String(expanded),
JsonValue::String(view.query_place(py)?),
])
}
None => {
Expand Down
120 changes: 115 additions & 5 deletions crates/sqlbuild-python-compile/src/rules/_helpers/sql/sql_document.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
//! Custom rules' SQL documents: the SQLQuack syntax tree of one SQL text and its common structures.

use std::collections::HashMap;
use std::sync::Arc;

use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyResult, Python};
use pyo3::exceptions::{PyAttributeError, PyValueError};
use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python};
use pyo3::sync::PyOnceLock;
use pyo3::types::PyTuple;
use pyo3::{IntoPyObject, pyclass, pymethods};
Expand All @@ -11,9 +13,12 @@ use sqlbuild_analysis::sql_tree::main::sql_tree_nodes::sql_tree_nodes;
use sqlbuild_analysis::sql_tree::models::{
NodeExtent, NodeLiteral, NodeName, SqlTreeNode as TreeNode,
};
use sqlbuild_rules::engine::main::query_file_positions::query_file_positions;
use sqlbuild_rules::models::ModelQueryPlace;

use sqlbuild_python_base::records::main::construct::new_annotated_error::new_annotated_error;
use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with;
use sqlbuild_python_base::records::main::fields::field::field as native_field;
use sqlbuild_python_base::records::types::ErrorId;
use sqlbuild_python_base::records::types::RecordId;

Expand All @@ -29,6 +34,14 @@ struct Tree {
children: Vec<Vec<usize>>,
}

/// A model query's character offsets and the file positions of its tree's node starts, built once.
struct FilePositions {
/// The character each line starts at, then the source's length.
line_starts: Vec<usize>,
/// The file `(line, column)` of every tree node's first character, by that character.
node_starts: HashMap<usize, (u64, u64)>,
}

/// One node of a rule's SQL syntax tree: an index into its shared tree, so freeing never recurses.
#[pyclass(module = "sqlbuild._native", frozen)]
pub(crate) struct SqlTreeNode {
Expand Down Expand Up @@ -165,19 +178,36 @@ pub(crate) struct SqlDocument {
#[pyo3(get)]
source: String,
dialect: String,
/// Where `source`, a model's expanded query, sits in the model file; `None` for other SQL.
place: Option<ModelQueryPlace>,
/// The model file `place` points into.
file: Option<String>,
tree: PyOnceLock<Arc<Tree>>,
positions: PyOnceLock<FilePositions>,
}

#[pymethods]
impl SqlDocument {
#[new]
#[pyo3(signature = (*, source, dialect))]
fn new(source: String, dialect: String) -> Self {
Self {
#[pyo3(signature = (*, source, dialect, place=None, file=None))]
fn new(
source: String,
dialect: String,
place: Option<&str>,
file: Option<String>,
) -> PyResult<Self> {
let place: Option<ModelQueryPlace> = place
.map(serde_json::from_str)
.transpose()
.map_err(|error| PyValueError::new_err(format!("invalid query place: {error}")))?;
Ok(Self {
source,
dialect,
place,
file,
tree: PyOnceLock::new(),
}
positions: PyOnceLock::new(),
})
}

/// The syntax tree: a `document` node whose children are the statements.
Expand Down Expand Up @@ -207,6 +237,31 @@ impl SqlDocument {
self.nodes(py, "Subquery")
}

/// Where `node`, read from this model query, is in the model file; `None` for other SQL.
#[pyo3(signature = (*, node))]
fn file_position(&self, node: &Bound<'_, PyAny>) -> PyResult<Option<(u64, u64)>> {
let (Some(place), Some(file)) = (self.place.as_ref(), self.file.as_deref()) else {
return Ok(None);
};
let (Some(line), Some(column)) = (node_index(node, "line")?, node_index(node, "column")?)
else {
return Ok(None);
};
let positions: &FilePositions = self
.positions
.get_or_init(node.py(), || self.file_positions(node.py(), (file, place)));
let character: usize = positions.character(line, column);
Ok(Some(
positions
.node_starts
.get(&character)
.copied()
.unwrap_or_else(|| {
query_file_positions(&self.source, (file, place), &[character])[0]
}),
))
}

/// Every projected star, bare or qualified; a star a function call takes, such as `COUNT(*)`, is no projection.
fn star_projections<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
self.source_nodes(py, "Star", |nodes, node| {
Expand Down Expand Up @@ -244,6 +299,37 @@ impl SqlDocument {
PyTuple::new(py, found)
}

/// Line starts and the file positions of every tree node's start, mapped through `place` at once.
fn file_positions(&self, py: Python<'_>, located: (&str, &ModelQueryPlace)) -> FilePositions {
let mut line_starts: Vec<usize> = vec![0];
let mut length: usize = 0;
for character in self.source.chars() {
length += 1;
if character == '\n' {
line_starts.push(length);
}
}
line_starts.push(length);
let mut positions: FilePositions = FilePositions {
line_starts,
node_starts: HashMap::new(),
};
let mut starts: Vec<usize> = self.parsed(py).map_or_else(
|_| Vec::new(),
|tree| {
tree.nodes
.iter()
.map(|node| positions.character(node.line, node.column))
.collect()
},
);
starts.sort_unstable();
starts.dedup();
let mapped: Vec<(u64, u64)> = query_file_positions(&self.source, located, &starts);
positions.node_starts = starts.into_iter().zip(mapped).collect();
positions
}

/// The parsed tree; a failure raises `RuleUsageError` each time it is read.
fn parsed(&self, py: Python<'_>) -> PyResult<&Arc<Tree>> {
self.tree.get_or_try_init(py, || {
Expand All @@ -253,6 +339,30 @@ impl SqlDocument {
}
}

impl FilePositions {
/// The source character at 1-based `line` and `column`; a line past the end starts at the end.
fn character(&self, line: usize, column: usize) -> usize {
let last: usize = self.line_starts.len() - 1;
self.line_starts[line.saturating_sub(1).min(last)] + column.saturating_sub(1)
}
}

/// A node's 1-based `line` or `column` attribute; `None` when it has none or it is not a positive integer.
fn node_index(node: &Bound<'_, PyAny>, name: &str) -> PyResult<Option<usize>> {
let value: i64 = match native_field(node, name) {
Ok(value) => match value.extract() {
Ok(value) => value,
Err(_) => return Ok(None),
},
Err(error) if error.is_instance_of::<PyAttributeError>(node.py()) => return Ok(None),
Err(error) => return Err(error),
};
Ok(match usize::try_from(value) {
Ok(index) if index > 0 => Some(index),
_ => None,
})
}

/// `nodes` with each node's children listed, in ascending index order, under its parent.
fn tree(nodes: Vec<TreeNode>) -> Arc<Tree> {
let mut children: Vec<Vec<usize>> = vec![Vec::new(); nodes.len() + 1];
Expand Down
24 changes: 23 additions & 1 deletion crates/sqlbuild-python-project/src/compiled_views/models.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
//! Public records of compiled views.

use sqlbuild_rules::engine::main::model_query_place::model_query_place;
use sqlbuild_rules::models::ModelQueryPlace;

use crate::compiled_project::main::reference_row_tuple::reference_row_tuple;
use crate::compiled_project::models::NativeCompiledProject;
use crate::compiled_project::models::ReferenceTuple;
Expand Down Expand Up @@ -420,12 +423,23 @@ impl NativeCompiledModel {

/// The model's authored file text and expanded query, from its recorded facts.
pub fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> {
self.with_facts(py, |facts| {
(facts.authored_sql.to_string(), facts.query_sql.to_string())
})
}

/// `read` applied to the model's recorded facts.
pub fn with_facts<T>(
&self,
py: Python<'_>,
read: impl FnOnce(&CompiledModelFacts) -> T,
) -> PyResult<T> {
let name: String = self.name(py)?;
let owner = self.owner.bind(py).borrow();
let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| {
pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled")
})?;
Ok((facts.authored_sql.to_string(), facts.query_sql.to_string()))
Ok(read(facts))
}

pub(crate) fn model_file<T>(
Expand Down Expand Up @@ -685,6 +699,14 @@ impl NativeCompiledModel {
self.model_file(py, |file| Ok(file.contents(py)))
}

/// Where the expanded query sits in the model file, as JSON a rule's SQL document maps positions through.
#[getter]
pub fn query_place(&self, py: Python<'_>) -> PyResult<String> {
let place: ModelQueryPlace = self.with_facts(py, model_query_place)?;
serde_json::to_string(&place)
.map_err(|error| pyo3::exceptions::PyRuntimeError::new_err(error.to_string()))
}

#[getter]
pub fn authored_query_sql<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyString>> {
self.model_file(py, |file| Ok(file.query_sql(py)))
Expand Down
40 changes: 30 additions & 10 deletions crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,23 +119,43 @@ pub(crate) fn declaration(fact: &DeclarationFact) -> Declaration {

/// The expansion passes back to the authored query and where that query starts in the file.
fn authored_query_map(facts: &CompiledModelFacts) -> Option<AuthoredQueryMap> {
if facts.authored_query_sql.is_empty() || facts.expansion_passes.iter().all(Vec::is_empty) {
return None;
}
located_query_map(
&facts.authored_sql,
&facts.authored_query_sql,
expansion_spans(&facts.expansion_passes),
)
}

/// `passes` in characters, as the rules request carries them.
pub(crate) fn expansion_spans(passes: &[Vec<CharSpan>]) -> Vec<Vec<ExpansionSpan>> {
let span = |span: &CharSpan| ExpansionSpan {
source_start: character_offset(span.source_start),
source_end: character_offset(span.source_end),
output_start: character_offset(span.output_start),
output_end: character_offset(span.output_end),
};
if facts.authored_query_sql.is_empty() || facts.expansion_passes.iter().all(Vec::is_empty) {
passes
.iter()
.map(|pass| pass.iter().map(span).collect())
.collect()
}

/// `passes` back to `authored_query` and where that query starts in `file`.
pub(crate) fn located_query_map(
file: &str,
authored_query: &str,
passes: Vec<Vec<ExpansionSpan>>,
) -> Option<AuthoredQueryMap> {
if authored_query.is_empty() || passes.iter().all(Vec::is_empty) {
return None;
}
let start: usize = facts.authored_sql.rfind(&*facts.authored_query_sql)?;
let above: &str = &facts.authored_sql[..start];
let start: usize = file.rfind(authored_query)?;
let above: &str = &file[..start];
Some(AuthoredQueryMap {
passes: facts
.expansion_passes
.iter()
.map(|pass| pass.iter().map(span).collect())
.collect(),
passes,
line_offset: above.matches('\n').count() as u64,
column_offset: above
.rsplit('\n')
Expand All @@ -151,7 +171,7 @@ fn character_offset(offset: usize) -> i64 {
}

/// The file-line characters before the query's first content that the query itself lacks.
fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 {
pub(crate) fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 {
let line_prefix = |text: &str, at: usize| {
text[..at]
.rsplit('\n')
Expand All @@ -172,7 +192,7 @@ fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 {
}

/// The lines of `file` above `query`, the rendered form of its `authored_query`; 0 if unknown.
fn query_line_offset(file: &str, authored_query: &str, query: &str) -> u64 {
pub(crate) fn query_line_offset(file: &str, authored_query: &str, query: &str) -> u64 {
let leading_lines = |text: &str| {
let body: &str = text.trim_start();
text[..text.len() - body.len()].matches('\n').count()
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlbuild-rules/src/engine/main/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ pub mod evaluate_rows;
pub mod finalize_rows;
pub mod json_grain_skipped_models;
pub mod json_grains;
pub mod model_query_place;
pub mod needs_json_grains;
pub mod project_model_rows;
pub mod query_file_positions;
pub mod retained_request;
pub mod start_speculative_rows;
31 changes: 31 additions & 0 deletions crates/sqlbuild-rules/src/engine/main/model_query_place.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
//! What places a compiled model's expanded query in its file.

use sqlbuild_analysis::compiled_project::models::CompiledModelFacts;

use crate::engine::_helpers::project_rows::expansion_spans;
use crate::models::{AuthoredQuery, ModelQueryPlace};

/// Where `facts`' authored query is and its expansion passes; finding the query is a suffix check.
#[must_use]
pub fn model_query_place(facts: &CompiledModelFacts) -> ModelQueryPlace {
let file: &str = &facts.authored_sql;
let query: &str = &facts.authored_query_sql;
let start: Option<usize> = if file.ends_with(query) {
Some(file.len() - query.len())
} else {
file.rfind(query)
};
ModelQueryPlace {
authored_query: (!query.is_empty()).then(|| {
start.map_or_else(
|| AuthoredQuery::Text(query.to_owned()),
|start| AuthoredQuery::Range(start, start + query.len()),
)
}),
passes: if facts.expansion_passes.iter().all(Vec::is_empty) {
Vec::new()
} else {
expansion_spans(&facts.expansion_passes)
},
}
}
Loading
Loading