diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index ee0aa9b30f..93913583f3 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -118,8 +118,22 @@ pub(crate) struct SqlTestPatterns { pub(crate) identifier: Regex, } +/// The SQL-test patterns, compiled once; each scenario and test clones them with its syntax. +static COMPILED_PATTERNS: OnceLock> = OnceLock::new(); + impl SqlTestPatterns { pub(crate) fn new(lexical: LexicalSyntax) -> Result { + let compiled: &Self = COMPILED_PATTERNS + .get_or_init(|| Self::compiled(LexicalSyntax::default())) + .as_ref() + .map_err(Clone::clone)?; + Ok(Self { + lexical, + ..compiled.clone() + }) + } + + fn compiled(lexical: LexicalSyntax) -> Result { Ok(Self { reference: compile_pattern(r#"(?i)__ref\(\"([^\"]+)\"\)"#)?, source: compile_pattern(r#"(?i)__source\(\"([^\"]+)\"\)"#)?, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs index 88bdb221dd..9ab2f625d6 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs @@ -1,5 +1,8 @@ //! A call's argument list: split, bound to parameters and checked against their types. +use std::cell::RefCell; +use std::collections::HashMap; + use crate::sql_macros::_helpers::parsing::header::{split_top_level, unquoted}; use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, closing_token, tokens}; use crate::sql_macros::_helpers::rendering::slot_mappings::slot_mappings; @@ -531,8 +534,34 @@ pub(crate) fn is_atomic(sql: &str) -> bool { false } +thread_local! { + /// Each declared type's family by adapter, since normalizing a type parses it. + static FAMILIES: RefCell>> = + RefCell::new(HashMap::new()); +} + /// The family of a declared type, normalized as schema declaration types are for the adapter. pub(crate) fn type_family(data_type: &str, adapter: &str) -> TypeFamily { + FAMILIES.with(|families| { + if let Some(family) = families + .borrow() + .get(adapter) + .and_then(|by_type| by_type.get(data_type)) + { + return *family; + } + let family: TypeFamily = normalized_family(data_type, adapter); + families + .borrow_mut() + .entry(adapter.to_owned()) + .or_default() + .insert(data_type.to_owned(), family); + family + }) +} + +/// The family `normalize_type` gives a declared type for the adapter's dialect. +fn normalized_family(data_type: &str, adapter: &str) -> TypeFamily { use crate::type_system::models::TypeFamily as Normalized; let dialect: &str = adapter_dialect(adapter); let normalized = normalize_type(data_type, dialect) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs index 512dffbee3..e0d8e6ef93 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs @@ -25,6 +25,8 @@ use crate::type_system::models::TypeNormalization; /// Each body's type, by the probe that typed it, so a macro is typed once per process. static BODY_TYPES: OnceLock>> = OnceLock::new(); +/// `returns` verdicts by dialect, adapter, definition and body, so each is probed once per process. +static RETURNS_MISMATCHES: OnceLock>>> = OnceLock::new(); /// Each body's first type error, by the probe that found it. static TYPE_ERRORS: OnceLock>>> = OnceLock::new(); @@ -159,27 +161,19 @@ fn remembered( verdict } -/// Rejects an expression macro whose body's known type is not its declared `returns`. -pub(crate) fn check_body_returns( +/// The body's known type when it is not `returns`, typed over stand-in arguments. +fn returns_mismatch( (parser, definition): (&dyn NativeDialect, &SqlMacroDefinition), body: &ExpandedText, - (adapter, location): (&str, SqlMacroLocation), -) -> Result<(), SqlMacroError> { - if definition.form != SqlMacroForm::Expression { - return Ok(()); - } - let Some(returns) = definition.returns.as_deref() else { - return Ok(()); - }; - let Some((probe, table)) = body_probe((parser, definition), body, false) else { - return Ok(()); - }; + (returns, adapter): (&str, &str), +) -> Option { + let (probe, table) = body_probe((parser, definition), body, false)?; let key: String = format!("returns\n{}\n{table:?}\n{probe}", parser.name()); let body_type: BodyType = remembered(&BODY_TYPES, key, || { inferred_body_type(parser, &probe, &table) }); let (shown, wrong): (String, bool) = match body_type { - BodyType::Unknown => return Ok(()), + BodyType::Unknown => return None, BodyType::Known(inferred) if inferred.eq_ignore_ascii_case(INFERRED_TEXT_TYPE) => ( VARCHAR_TYPE.to_owned(), disagrees(returns, VARCHAR_TYPE, adapter), @@ -189,9 +183,28 @@ pub(crate) fn check_body_returns( (inferred, wrong) } }; - if !wrong { + wrong.then_some(shown) +} + +/// Rejects an expression macro whose body's known type is not its declared `returns`. +pub(crate) fn check_body_returns( + (parser, definition): (&dyn NativeDialect, &SqlMacroDefinition), + body: &ExpandedText, + (adapter, location): (&str, SqlMacroLocation), +) -> Result<(), SqlMacroError> { + if definition.form != SqlMacroForm::Expression { return Ok(()); } + let Some(returns) = definition.returns.as_deref() else { + return Ok(()); + }; + let key: String = format!("{}\n{adapter}\n{definition:?}\n{body:?}", parser.name()); + let mismatch: Option = remembered(&RETURNS_MISMATCHES, key, || { + returns_mismatch((parser, definition), body, (returns, adapter)) + }); + let Some(shown) = mismatch else { + return Ok(()); + }; Err(SqlMacroError::new( RETURNS_MISMATCH_CODE, format!( diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs b/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs deleted file mode 100644 index a7139e3c65..0000000000 --- a/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; -use crate::sql_macros::constants::INVALID_DEFINITION_CODE; -use crate::sql_macros::errors::ExpansionFailure; -use crate::sql_macros::errors::SqlMacroError; -use crate::sql_macros::models::{ExpandedText, InsertedNames, SqlMacroCall, TextContext}; -use crate::sql_macros::types::SqlMacroHost; - -/// One outer SQL macro call's replacement text with its typed calls, and the names inserted so far. -pub fn expand_sql_macro_call( - host: &H, - context: &TextContext<'_>, - inserted: InsertedNames, - call: &SqlMacroCall, -) -> Result<(ExpandedText, InsertedNames), ExpansionFailure> { - let scanned: Vec = tokens(context.text).map_err(|_| { - SqlMacroError::new( - INVALID_DEFINITION_CODE, - "SQL has an unclosed quote or comment", - ) - })?; - crate::sql_macros::_helpers::rendering::expansion::expand_call( - host, - context, - (&scanned, inserted), - call, - ) -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_calls.rs new file mode 100644 index 0000000000..5643c1aef1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_calls.rs @@ -0,0 +1,40 @@ +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; +use crate::sql_macros::constants::INVALID_DEFINITION_CODE; +use crate::sql_macros::errors::ExpansionFailure; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ExpandedText, InsertedNames, SqlMacroCall, TextContext}; +use crate::sql_macros::types::SqlMacroHost; + +/// Each outer SQL macro call's expansion in order, or the index of the call that failed and why. +pub fn expand_sql_macro_calls( + host: &H, + context: &TextContext<'_>, + calls: &[&SqlMacroCall], +) -> Result, (usize, ExpansionFailure)> { + if calls.is_empty() { + return Ok(Vec::new()); + } + let scanned: Vec = tokens(context.text).map_err(|_| { + ( + 0, + ExpansionFailure::from(SqlMacroError::new( + INVALID_DEFINITION_CODE, + "SQL has an unclosed quote or comment", + )), + ) + })?; + let mut inserted: InsertedNames = InsertedNames::default(); + let mut texts: Vec = Vec::with_capacity(calls.len()); + for (index, call) in calls.iter().enumerate() { + let (text, updated) = crate::sql_macros::_helpers::rendering::expansion::expand_call( + host, + context, + (&scanned, inserted), + call, + ) + .map_err(|failure| (index, failure))?; + texts.push(text); + inserted = updated; + } + Ok(texts) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs index 45b48d9884..13b003d717 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs @@ -2,7 +2,7 @@ pub mod bare_macro_calls; pub mod byte_text_splices; pub mod check_sql_macro_arguments; pub mod declaration_visible_from_macro; -pub mod expand_sql_macro_call; +pub mod expand_sql_macro_calls; pub mod marker_free_text; pub mod moved_typed_calls; pub mod parse_sql_macro; diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs index e2c332f855..8eda163feb 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs @@ -5,6 +5,7 @@ use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_s use crate::sql_macros::tests::helpers::{adapter_parses, expanded_for}; use crate::sql_macros::tests::test_types::{ AsciiArguments, ExpansionErrorTestCase, ExpansionTestCase, PythonArgumentTestCase, + ReturnsSequenceTestCase, }; const CENTS_TO_DOLLARS: (&str, &str) = ( @@ -137,6 +138,10 @@ const SHOUTED_WRONG: (&str, &str) = ( "shouted", "MACRO (description \"Shouted\", arguments (value VARCHAR), returns BIGINT);\n\nUPPER(@value)\n", ); +const SHOUTED_TEXT: (&str, &str) = ( + "shouted", + "MACRO (description \"Shouted\", arguments (value VARCHAR), returns VARCHAR);\n\nUPPER(@value)\n", +); const SHOUTED_UNTYPED: (&str, &str) = ( "shouted", "MACRO (description \"Shouted\", arguments (value VARCHAR));\n\nUPPER(@value)\n", @@ -821,6 +826,38 @@ fn given_a_body_whose_type_breaks_its_declaration_when_expanding_then_the_error_ } } +#[test] +fn given_one_body_under_different_returns_when_expanding_in_turn_then_each_returns_is_checked() { + let test_cases = [ + ReturnsSequenceTestCase { + description: "a fitting declaration, then a breaking one, alternating", + definitions: &[SHOUTED_TEXT, SHOUTED_WRONG, SHOUTED_TEXT, SHOUTED_WRONG], + sql: "SELECT @sql.shouted(o.status) FROM orders AS o", + expected_codes: &[None, Some("P024"), None, Some("P024")], + }, + ReturnsSequenceTestCase { + description: "a breaking declaration, then a fitting one", + definitions: &[SHOUTED_WRONG, SHOUTED_TEXT], + sql: "SELECT @sql.shouted(o.status) FROM orders AS o", + expected_codes: &[Some("P024"), None], + }, + ]; + + for test_case in test_cases { + let codes: Vec> = test_case + .definitions + .iter() + .map(|definition| { + expanded_for("duckdb", &[*definition], test_case.sql) + .err() + .map(|error| error.code) + }) + .collect(); + + assert_eq!(codes, test_case.expected_codes, "{}", test_case.description); + } +} + #[test] fn given_a_sql_macro_call_in_python_macro_arguments_when_expanding_then_python_receives_its_expansion() { diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index 81d73c007f..26fdebc00a 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -22,6 +22,13 @@ pub(super) struct ExpansionErrorTestCase { pub(super) expected_message: &'static str, } +pub(super) struct ReturnsSequenceTestCase { + pub(super) description: &'static str, + pub(super) definitions: &'static [(&'static str, &'static str)], + pub(super) sql: &'static str, + pub(super) expected_codes: &'static [Option<&'static str>], +} + pub(super) struct DefinitionErrorTestCase { pub(super) description: &'static str, pub(super) source: &'static str, diff --git a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs index 5cc5adb91f..97d90be26b 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs @@ -17,13 +17,13 @@ use sqlbuild_analysis::sql_macros::errors::SqlMacroError; use sqlbuild_analysis::sql_macros::main::bare_macro_calls::bare_macro_calls; use sqlbuild_analysis::sql_macros::main::byte_text_splices::byte_text_splices; use sqlbuild_analysis::sql_macros::main::declaration_visible_from_macro::declaration_visible_from_macro; -use sqlbuild_analysis::sql_macros::main::expand_sql_macro_call::expand_sql_macro_call; +use sqlbuild_analysis::sql_macros::main::expand_sql_macro_calls::expand_sql_macro_calls; use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; use sqlbuild_analysis::sql_macros::main::render_inline_collection::render_inline_collection; use sqlbuild_analysis::sql_macros::main::sql_macro_calls::sql_macro_calls; use sqlbuild_analysis::sql_macros::models::{ - ExpandedText, InsertedNames, RenderedDeclarations, SqlMacroCall, SqlMacroCheck, - SqlMacroDefinition, SqlMacroLocation, TextContext, TextSplice, + ExpandedText, RenderedDeclarations, SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, + SqlMacroLocation, TextContext, TextSplice, }; use sqlbuild_analysis::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; @@ -68,6 +68,10 @@ struct PyHost<'r, 'a, 'py> { request: &'r DeclarationExpansion<'a, 'py>, source: SqlMacroSource<'a, 'py>, facts: RefCell>, + /// The byte of the authored file where its query starts, once per pass rather than per call. + query_byte: usize, + /// Each macro this pass parsed, by name, path and source, so repeated calls parse it once. + definitions: RefCell>>, adapter: String, analysis_dialect: Option, } @@ -100,6 +104,8 @@ pub(crate) fn expanded_sql_macro_calls<'py>( request, source, facts: RefCell::new(SqlMacroFacts::default()), + query_byte: source.origin.map_or(0, query_start), + definitions: RefCell::new(HashMap::new()), adapter: request.rendering.adapter_name(request.context.py()), analysis_dialect: request.rendering.analysis_dialect(request.context.py())?, }; @@ -110,59 +116,61 @@ pub(crate) fn expanded_sql_macro_calls<'py>( stack: &[], owner: &owner, }; - let mut inserted: InsertedNames = InsertedNames::default(); - let mut replacements: Vec = Vec::with_capacity(starts.len()); - for start in starts { - let Some(call) = by_char.get(start) else { - return Err(macro_error( - request, - &SqlMacroError { - code: UNKNOWN_CODE, - message: "a SQL macro call could not be read".to_owned(), - locations: Vec::new(), - help: None, - }, - )); - }; - match expand_sql_macro_call(&host, &context, inserted, call) { - Ok((text, updated)) => { - replacements.push(text); - inserted = updated; - } - Err(ExpansionFailure::Host(error)) => return Err(error), - Err(ExpansionFailure::Macro(error)) => { - let located: SqlMacroError = match host.call_location((call.start, call.end)) { - Some(location) if is_macro_file_error(&error) => error.at(location), - Some(location) => error.at_call(location), - None => error, - }; - return Err(macro_error(request, &located)); - } + let read: Vec<&SqlMacroCall> = starts + .iter() + .map_while(|start| by_char.get(start).copied()) + .collect(); + let replacements: Vec = match expand_sql_macro_calls(&host, &context, &read) { + Ok(replacements) => replacements, + Err((_, ExpansionFailure::Host(error))) => return Err(error), + Err((index, ExpansionFailure::Macro(error))) => { + let call: &SqlMacroCall = read[index]; + let located: SqlMacroError = match host.call_location((call.start, call.end)) { + Some(location) if is_macro_file_error(&error) => error.at(location), + Some(location) => error.at_call(location), + None => error, + }; + return Err(macro_error(request, &located)); } + }; + if read.len() < starts.len() { + return Err(macro_error( + request, + &SqlMacroError { + code: UNKNOWN_CODE, + message: "a SQL macro call could not be read".to_owned(), + locations: Vec::new(), + help: None, + }, + )); } Ok((replacements, host.facts.into_inner())) } +/// The byte of the authored file `origin` where its query starts. +fn query_start(origin: SqlCallOrigin<'_>) -> usize { + if origin.contents.ends_with(origin.query) { + origin.contents.len() - origin.query.len() + } else { + origin.contents.find(origin.query).unwrap_or(0) + } +} + /// Where the call spanning bytes `span` of `sql` starts in the authored file `origin`. fn authored_location( - origin: SqlCallOrigin<'_>, + (origin, query_byte): (SqlCallOrigin<'_>, usize), sql: &str, span: (usize, usize), ) -> SqlMacroLocation { - authored_span(origin, sql, span).0 + authored_span((origin, query_byte), sql, span).0 } /// Where the call at bytes `start..end` of `sql` starts and ends in the authored file `origin`. fn authored_span( - origin: SqlCallOrigin<'_>, + (origin, query_byte): (SqlCallOrigin<'_>, usize), sql: &str, (start, end): (usize, usize), ) -> (SqlMacroLocation, (usize, usize)) { - let query_byte: usize = if origin.contents.ends_with(origin.query) { - origin.contents.len() - origin.query.len() - } else { - origin.contents.find(origin.query).unwrap_or(0) - }; let snippet: &str = &sql[start..end]; let authored: usize = if sql == origin.query { query_byte + start @@ -219,7 +227,7 @@ pub(crate) fn reject_bare_macro_calls(request: &DeclarationExpansion<'_, '_>) -> "write @sql.{name}(...) to call a SQL macro or @py.{name}(...) to call a Python macro" )) .at_call(authored_location( - origin, + (origin, query_start(origin)), request.sql, (start, start + 1 + name.len()), )); @@ -302,7 +310,7 @@ impl<'py> PyHost<'_, '_, 'py> { /// The authored location of the call spanning bytes `span` of the caller's text. fn call_location(&self, span: (usize, usize)) -> Option { Some(authored_location( - self.source.origin?, + (self.source.origin?, self.query_byte), self.request.sql, span, )) @@ -458,7 +466,18 @@ impl<'py> SqlMacroHost for PyHost<'_, '_, 'py> { )); } let source: String = host(field_text(&loaded, "raw_source"))?; - let definition: Arc = Arc::new(parse_sql_macro(name, &path, &source)?); + let key: String = format!("{name}\n{path}\n{source}"); + let known: Option> = self.definitions.borrow().get(&key).cloned(); + let definition: Arc = match known { + Some(definition) => definition, + None => { + let parsed = Arc::new(parse_sql_macro(name, &path, &source)?); + self.definitions + .borrow_mut() + .insert(key, Arc::clone(&parsed)); + parsed + } + }; host(self.record_use(name))?; Ok(definition) } @@ -594,7 +613,7 @@ impl<'py> SqlMacroHost for PyHost<'_, '_, 'py> { let Some(origin) = self.source.origin else { return; }; - let (location, end) = authored_span(origin, self.request.sql, call); + let (location, end) = authored_span((origin, self.query_byte), self.request.sql, call); self.facts.borrow_mut().checks.push(SqlMacroCheck { call: Some((location, end)), ..check diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md index 810becb3f3..7e4aaf8df6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md @@ -140,7 +140,9 @@ counts the downstream uses that weren't checked. Independent errors elsewhere ar ## Turning analysis off To skip SQL analysis for one model, use `MODEL (sql_analysis false)`, a path default, or -`--no-sql-analysis` for a run. This also turns off type inference and lineage for that model. +`--no-sql-analysis` for a run. This also turns off type inference for that model. Column +lineage still runs; to turn it off, use `--lineage-mode none` (see +[Column lineage modes](../cli/compile.md#column-lineage-modes)). To allow those opt-outs only for SQL the parser can't read, set `[settings] require_sql_analysis = true`; see [Requiring SQL analysis](models/configuration.md#requiring-sql-analysis). diff --git a/website/src/content/docs/docs/concepts/semantic-compilation.mdx b/website/src/content/docs/docs/concepts/semantic-compilation.mdx index c6ea842771..ada14ff44f 100644 --- a/website/src/content/docs/docs/concepts/semantic-compilation.mdx +++ b/website/src/content/docs/docs/concepts/semantic-compilation.mdx @@ -128,7 +128,9 @@ counts the downstream uses that weren't checked. Independent errors elsewhere ar ## Turning analysis off To skip SQL analysis for one model, use `MODEL (sql_analysis false)`, a path default, or -`--no-sql-analysis` for a run. This also turns off type inference and lineage for that model. +`--no-sql-analysis` for a run. This also turns off type inference for that model. Column +lineage still runs; to turn it off, use `--lineage-mode none` (see +[Column lineage modes](/docs/cli/compile#column-lineage-modes)). To allow those opt-outs only for SQL the parser can't read, set `[settings] require_sql_analysis = true`; see [Requiring SQL analysis](/docs/concepts/models/configuration#requiring-sql-analysis).