diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 321a5f9967..728798a5cb 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -133,7 +133,7 @@ function, a hook-created JavaScript function, 192 unit tests and 4 scenarios, in | Ordering through arrays | an unordered `ARRAY_AGG` ranked under `LIMIT` by position, by name, through `IFF` and over `UNION ALL`, with fixtures whose pick does not depend on element order; order-free reads (`ARRAY_SIZE`, `ARRAY_CONTAINS`, `ARRAYS_OVERLAP`, `ARRAY_DISTINCT` of `ARRAY_CAT`, `FLATTEN` values through a CTE); `SELECT DISTINCT` over arrays repeated in different orders; `ARRAY_APPEND`, `ARRAY_INTERSECTION`, `ARRAY_SLICE` with negative bounds, `TO_ARRAY` and bracket array literals | | VARIANT comparisons | `ARRAY_MAX`, `ARRAY_MIN` and `ARRAY_REVERSE` over arrays mixing numbers, strings, booleans, JSON nulls and integers past 2^53; `MAX` across kinds and non-ASCII strings; `=` and `EQUAL_NULL` for `1`, `"1"` and `1.0`, reordered keys, JSON null and DATE or TIMESTAMP values held in a `VARIANT` against their text; `MIN_BY` and `MAX_BY` of `VARIANT` values; `PARTITION BY`, `SELECT DISTINCT` and `ARRAY_AGG(DISTINCT)` over `VARIANT`; JSON nulls in `ARRAY_REMOVE`, `ARRAY_DISTINCT` and `ARRAYS_OVERLAP`; decimals with trailing zeros in `ARRAY_CONTAINS` and `ARRAY_POSITION`; `OBJECT_CONSTRUCT(* EXCLUDE ("note"))` | | Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps; `TRY_TO_DATE` with numeric and month-name formats over unpadded fields, two-digit years, ISO `T` and `Z`, trailing letters, a long s, mixed separators and runs of spaces; text read as `TIMESTAMP_TZ` and `TIMESTAMP_LTZ` with `T`, `Z`, `+00:00`, `+0000`, `+09:00` and `-05:30`, back as the UTC instant, date, year and day start, and compared with text, `TIMESTAMP_NTZ` and `DATE`; `FIRST_VALUE` of a zoned value as a `DATE`; `DATEADD` and `TIMEADD` on `TIME(3)` and the type they return; `DATEDIFF` across units; `TIME(9)`, `FF3` and bare `FF` formats truncating; `DATE`, `TO_TIME`, `TRY_TO_TIMESTAMP` and `TO_TIMESTAMP_LTZ` of clock text, and `CURRENT_DATE` and `SYSDATE` read only for facts that hold on any day | -| Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes, and a scoped SQL macro collapsing whitespace whose body holds a comment; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | +| Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes, and a scoped SQL macro collapsing whitespace whose body opens with a block comment and holds a line comment; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | | Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; relations aliased `first` and `last`, one without `AS`, the words of Snowflake's read-order clauses; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote, and columns qualified by their reference call, whose ranges are dropped | | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results | | Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild, and a keyed cursor `delete_insert` copy of it whose rows move out of the window between loads | diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql b/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql index ab10719442..85cfe5d291 100644 --- a/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql +++ b/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql @@ -4,6 +4,8 @@ MACRO ( returns VARCHAR, ); +/* Spaces, tabs and line breaks all count as whitespace, + so a value pasted from a spreadsheet collapses too */ REGEXP_REPLACE( @text_value, -- Each run of spaces, tabs and line breaks becomes one space diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs index e6d9ffd6e6..02fcf32612 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs @@ -8,6 +8,7 @@ use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitte use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_core::text::main::python_isspace::python_isspace; use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::dsl_headers::main::declares_table_returns::declares_table_returns; @@ -17,16 +18,18 @@ use crate::dsl_headers::main::sql_body_ranges::sql_body_ranges; use crate::dsl_headers::models::HeaderSpan; use crate::lint_text::_helpers::sites::compiled; use crate::lint_text::constants::{ - DECLARATION_GROUP_DIRECTORY, DECLINED_WITHOUT_REASON, DOLLAR_QUOTE, EXPRESSION_SELECT_PREFIX, - FIXTURE_HEADER_KINDS, FORMAT_API_VERSION, FUNCTION_HEADER_KIND, GLOBAL_NAMED_ROOTS, - GROUPED_NAMED_ROLES, LINE_COMMENT_PREFIX, LINTED_EXPRESSION_ALIAS, MACRO_HEADER_KIND, - MAX_FUNCTION_CALL_DEPTH, MOVED_EXPRESSION_ALIAS, MOVED_OMITTED_SELECT, - OMITTED_CEREMONIAL_SELECT_SQL, OMITTED_SELECT_SUFFIX_PATTERN, STRING_QUOTES, + BLOCK_COMMENT_OPENER, DECLARATION_GROUP_DIRECTORY, DECLINED_WITHOUT_REASON, + EXPRESSION_SELECT_PREFIX, FIXTURE_HEADER_KINDS, FORMAT_API_VERSION, FUNCTION_HEADER_KIND, + GLOBAL_NAMED_ROOTS, GROUPED_NAMED_ROLES, LINE_COMMENT_PREFIX, LINTED_EXPRESSION_ALIAS, + MACRO_HEADER_KIND, MAX_FUNCTION_CALL_DEPTH, MOVED_EXPRESSION_ALIAS, MOVED_OMITTED_SELECT, + OMITTED_CEREMONIAL_SELECT_SQL, OMITTED_SELECT_SUFFIX_PATTERN, }; use crate::lint_text::errors::LintTextError; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; use crate::lint_text::main::restore_interpolation::restore_interpolation; -use crate::lint_text::models::{FormatFault, FormatFile, InterpolationSite, PreparedBody}; +use crate::lint_text::models::{ + FormatFault, FormatFile, FormatRun, InterpolationSite, PreparedBody, +}; use sqlclean::sql_lint::models::{FormatRequest, FormatResponse}; static OMITTED_SELECT_SUFFIX: LazyLock> = @@ -260,6 +263,7 @@ fn printed_width(text: &str) -> usize { pub(crate) fn formatted_body( prepared: &PreparedBody, response: &FormatResponse, + syntax: &LexicalSyntax, ) -> Result, String> { if !response.formatted { return Err(response @@ -287,7 +291,7 @@ pub(crate) fn formatted_body( let Some(alias) = pattern.find(sql) else { return Err(MOVED_EXPRESSION_ALIAS.to_string()); }; - unwrapped = unwrapped_expression(&sql[..alias.start()]) + unwrapped = unwrapped_expression(&sql[..alias.start()], syntax) .ok_or_else(|| MOVED_EXPRESSION_ALIAS.to_string())?; sql = &unwrapped; changed = true; @@ -305,13 +309,13 @@ pub(crate) fn splice_file( index: usize, contents: &str, bodies: &[PreparedBody], - responses: &[FormatResponse], + run: &FormatRun<'_>, ) -> (Option, Vec) { let original: Vec = contents.chars().collect(); let mut updated: Vec = original.clone(); let mut faults: Vec = Vec::new(); for body in bodies.iter().rev() { - match formatted_body(body, &responses[body.request]) { + match formatted_body(body, &run.responses[body.request], run.syntax) { Ok(Some(replacement)) => { let _ = updated.splice(body.start..body.end, replacement.chars()); } @@ -324,7 +328,7 @@ pub(crate) fn splice_file( .count() + 1, reason, - unavailable: responses[body.request].unavailable.is_some(), + unavailable: run.responses[body.request].unavailable.is_some(), }), } } @@ -332,78 +336,59 @@ pub(crate) fn splice_file( (formatted, faults) } -/// The expression of a formatted one-item `SELECT`, dedented, or `None` if it is not one. -pub(crate) fn unwrapped_expression(formatted: &str) -> Option { +/// The expression of a formatted one-item `SELECT`, dedented except inside comments and strings. +pub(crate) fn unwrapped_expression(formatted: &str, syntax: &LexicalSyntax) -> Option { let trimmed: &str = formatted.trim_start(); let keyword: &str = trimmed.get(..EXPRESSION_SELECT_PREFIX.trim_end().len())?; if !keyword.eq_ignore_ascii_case(EXPRESSION_SELECT_PREFIX.trim_end()) { return None; } let rest: &str = &trimmed[keyword.len()..]; - let lines: Vec<&str> = rest.trim_start_matches([' ', '\t']).lines().collect(); - let first: &str = lines.first().copied().unwrap_or_default(); - let (first, rest_lines): (&str, &[&str]) = if first.trim().is_empty() { - ( - lines.get(1).copied().unwrap_or_default(), - lines.get(2..).unwrap_or_default(), - ) - } else { - (first, lines.get(1..).unwrap_or_default()) - }; - let code: &str = std::iter::once(first) - .chain(rest_lines.iter().copied()) - .find(|line| !line.trim_start().starts_with(LINE_COMMENT_PREFIX)) - .unwrap_or(first); + let opaque: Vec<(usize, usize)> = dialect_non_code_ranges(rest, syntax); + let mut offset: usize = rest.len() - rest.trim_start_matches([' ', '\t']).len(); + let mut lines: Vec<(usize, &str)> = Vec::new(); + for line in rest[offset..].split('\n') { + lines.push((offset, line.strip_suffix('\r').unwrap_or(line))); + offset += line.len() + 1; + } + if lines.len() > 1 && lines[0].1.trim().is_empty() { + let _ = lines.remove(0); + } + let (_, first): (usize, &str) = lines[0]; + let code: &str = lines + .iter() + .find(|&&(start, line)| { + let lead: usize = start + line.len() - line.trim_start().len(); + !line.trim().is_empty() && !in_comment(rest, &opaque, lead, syntax) + }) + .map_or(first, |&(_, line)| line); let indent: usize = code.len() - code.trim_start().len(); + let mut ranges = opaque.iter().peekable(); let mut dedented: Vec<&str> = vec![first.trim_start()]; - let mut quote: Option = open_quote(None, first); - for line in rest_lines { - if quote.is_some() { + for &(start, line) in &lines[1..] { + while ranges.next_if(|&&(_, end)| end <= start).is_some() {} + if ranges.peek().is_some_and(|&&(open, _)| open < start) { dedented.push(line); } else { let leading: usize = line.len() - line.trim_start().len(); dedented.push(&line[leading.min(indent)..]); } - quote = open_quote(quote, line); } Some(dedented.join("\n").trim_end().to_owned()) } -/// The quote still open after `line`, given the one before it; `$` stands for `$$`, which has no escapes. -fn open_quote(mut quote: Option, line: &str) -> Option { - let mut characters = line.chars().peekable(); - let mut previous: Option = None; - while let Some(character) = characters.next() { - let after_word: bool = previous.is_some_and(|before| { - before.is_alphanumeric() || before == '_' || before == DOLLAR_QUOTE - }); - previous = Some(character); - if character == DOLLAR_QUOTE - && (quote == Some(DOLLAR_QUOTE) || (quote.is_none() && !after_word)) - && characters.peek() == Some(&DOLLAR_QUOTE) - { - let _ = characters.next(); - quote = if quote.is_some() { - None - } else { - Some(DOLLAR_QUOTE) - }; - continue; - } - match quote { - Some(DOLLAR_QUOTE) => {} - Some(open) if character == open => { - if characters.peek() == Some(&open) { - let _ = characters.next(); - } else { - quote = None; - } - } - Some(_) => {} - None if STRING_QUOTES.contains(&character) => quote = Some(character), - None if character == '-' && characters.peek() == Some(&'-') => return None, - None => {} - } - } - quote +/// Whether offset `at` of `text` lies in one of its non-code `ranges` that is a comment. +fn in_comment(text: &str, ranges: &[(usize, usize)], at: usize, syntax: &LexicalSyntax) -> bool { + ranges + .iter() + .any(|&(open, end)| open <= at && at < end && is_comment(&text[open..], syntax)) +} + +/// Whether the non-code text at the start of `text` is a comment rather than quoted text. +fn is_comment(text: &str, syntax: &LexicalSyntax) -> bool { + text.starts_with(BLOCK_COMMENT_OPENER) + || syntax + .line_comment_prefixes + .iter() + .any(|prefix| text.starts_with(prefix.as_str())) } diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index 416d21b21e..e2dac213d3 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -215,10 +215,6 @@ pub(crate) const CROSS_KEYWORD: &[u8] = b"cross"; pub(crate) const EXPRESSION_SELECT_PREFIX: &str = "SELECT "; /// The output name lint and format give an expression body; Snowflake keeps unnamed items as written. pub(crate) const LINTED_EXPRESSION_ALIAS: &str = "__sqlbuild_macro_expression__"; -/// The characters that open a quoted string or identifier in formatted SQL. -pub(crate) const STRING_QUOTES: [char; 3] = ['\'', '"', '`']; -/// Each half of the `$$` that opens and closes a dollar-quoted string in formatted SQL. -pub(crate) const DOLLAR_QUOTE: char = '$'; /// A CTE definition, capturing its `name`. pub(crate) const CTE_DEFINITION_PATTERN: &str = r#"(?is)(?:\bWITH(?:\s+RECURSIVE)?|,)\s*["`\[]?(?P[A-Za-z_][A-Za-z0-9_]*)["`\]]?(?:\s*\([^)]*\))?\s+AS\s*\("#; diff --git a/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs b/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs index 674a0de7a9..dab610089e 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs @@ -6,7 +6,9 @@ use crate::lint_text::_helpers::format_bodies::{ PendingBody, RequestTable, prepare_file, splice_file, }; use crate::lint_text::errors::LintTextError; -use crate::lint_text::models::{FormatFault, FormatFile, FormattedBodies, PreparedBody}; +use crate::lint_text::models::{FormatFault, FormatFile, FormatRun, FormattedBodies, PreparedBody}; +use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use sqlclean::sql_lint::main::format_requests::format_requests; use sqlclean::sql_lint::models::FormatResponse; @@ -31,11 +33,16 @@ pub fn format_sql_bodies( prepared.push(bodies); } let responses: Vec = format_requests(table.requests); + let syntax: LexicalSyntax = dialect_lexical_syntax(dialect); + let run: FormatRun<'_> = FormatRun { + responses: &responses, + syntax: &syntax, + }; let spliced: Vec<(Option, Vec)> = files .par_iter() .zip(prepared.par_iter()) .enumerate() - .map(|(index, (file, bodies))| splice_file(index, &file.contents, bodies, &responses)) + .map(|(index, (file, bodies))| splice_file(index, &file.contents, bodies, &run)) .collect(); let mut result: FormattedBodies = FormattedBodies::default(); for (index, (formatted, faults)) in spliced.into_iter().enumerate() { diff --git a/crates/sqlbuild-rules/src/lint_text/models.rs b/crates/sqlbuild-rules/src/lint_text/models.rs index 5c070f2047..0cf9e20fd1 100644 --- a/crates/sqlbuild-rules/src/lint_text/models.rs +++ b/crates/sqlbuild-rules/src/lint_text/models.rs @@ -83,6 +83,13 @@ pub struct FormattedBodies { pub faults: Vec, } +/// What splicing reads from one formatting run: every response, and the dialect's lexical syntax. +#[derive(Clone, Copy, Debug)] +pub(crate) struct FormatRun<'a> { + pub(crate) responses: &'a [sqlclean::sql_lint::models::FormatResponse], + pub(crate) syntax: &'a sqlbuild_sqltext::sql_scan::models::LexicalSyntax, +} + /// One authored body prepared for batched formatting; offsets are character positions. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct PreparedBody { diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index 31cc743b7a..1e180c5143 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -26,12 +26,17 @@ use crate::lint_text::tests::test_types::{ UnavailableFinding, UnavailableLintTestCase, UnwrappedExpressionTestCase, }; use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; +use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; const SUPPRESSED_FILE: &str = "-- sqb: ignore SQLCLEAN005 because kept\n\n-- note\nselect 1\n-- sqb: ignore nope\nselect 2\n"; const CROSS_JOIN_FILE: &str = "select o.order_id\nfrom orders as o\n-- sqb: ignore SQLCLEAN038 because every order meets every policy\ncross join __seed(\"order_policy\") as p\n"; const NORMALISE_NAME_MACRO: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(\n @column_name,\n '[[:space:]]+',\n ' '\n ))),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n"; +const ESCAPED_QUOTE_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(\n @v,\n 'O\\'Brien\n kept indent',\n 'tail'\n)\n"; +const ESCAPED_QUOTE_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(@v, 'O\\'Brien\n kept indent', 'tail')\n"; +const BLOCK_COMMENT_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* first\n second */\n 'x'\n)\n"; +const BLOCK_COMMENT_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* first\n second */ 'x'\n)\n"; const NORMALISE_NAME_FORMATTED: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(\n UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(@column_name, '[[:space:]]+', ' ')\n )\n ),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n"; #[test] @@ -652,26 +657,55 @@ fn given_expanded_bodies_when_linting_then_findings_map_to_authored_lines() { fn given_a_formatted_expression_select_when_unwrapping_then_strings_keep_their_lines() { let test_cases = [ UnwrappedExpressionTestCase { + dialect: "duckdb", description: "the expression is dedented one level", formatted: "SELECT\n ROUND(\n @v,\n 2\n )", expected_expression: Some("ROUND(\n @v,\n 2\n)"), }, UnwrappedExpressionTestCase { + dialect: "duckdb", description: "lines inside a multi-line string literal are not dedented", formatted: "SELECT\n CONCAT(\n 'first\n second',\n @v\n )", expected_expression: Some("CONCAT(\n 'first\n second',\n @v\n)"), }, UnwrappedExpressionTestCase { + dialect: "duckdb", description: "lines inside a dollar-quoted string are not dedented", formatted: "SELECT\n CONCAT(\n $$first\n 'second'\n $$,\n @v\n )", expected_expression: Some("CONCAT(\n $$first\n 'second'\n $$,\n @v\n)"), }, UnwrappedExpressionTestCase { + dialect: "duckdb", description: "a $$ inside an identifier does not open a string", formatted: "SELECT\n COALESCE(\n col$$x,\n @v\n )", expected_expression: Some("COALESCE(\n col$$x,\n @v\n)"), }, UnwrappedExpressionTestCase { + dialect: "snowflake", + description: "a Snowflake string continues past a backslash-escaped quote", + formatted: "SELECT\n CONCAT(\n 'O\\'Brien\n kept',\n @v\n )", + expected_expression: Some("CONCAT(\n 'O\\'Brien\n kept',\n @v\n)"), + }, + UnwrappedExpressionTestCase { + dialect: "snowflake", + description: "an apostrophe in a Snowflake // comment opens no string", + formatted: "SELECT\n COALESCE(\n @v, // it's here\n 'x'\n )", + expected_expression: Some("COALESCE(\n @v, // it's here\n 'x'\n)"), + }, + UnwrappedExpressionTestCase { + dialect: "duckdb", + description: "lines inside a block comment keep their text", + formatted: "SELECT\n COALESCE(\n @v, /* first\n second */\n 'x'\n )", + expected_expression: Some("COALESCE(\n @v, /* first\n second */\n 'x'\n)"), + }, + UnwrappedExpressionTestCase { + dialect: "duckdb", + description: "a block comment on the SELECT line does not set the indent", + formatted: "SELECT /* first\n second */\n COALESCE(\n @v,\n 'x'\n )", + expected_expression: Some("/* first\n second */\nCOALESCE(\n @v,\n 'x'\n)"), + }, + UnwrappedExpressionTestCase { + dialect: "duckdb", description: "text that is not a select is not an expression", formatted: "WITH a AS (SELECT 1)", expected_expression: None, @@ -679,7 +713,10 @@ fn given_a_formatted_expression_select_when_unwrapping_then_strings_keep_their_l ]; for test_case in test_cases { - let unwrapped: Option = unwrapped_expression(test_case.formatted); + let unwrapped: Option = unwrapped_expression( + test_case.formatted, + &dialect_lexical_syntax(test_case.dialect), + ); assert_eq!( unwrapped.as_deref(), @@ -974,18 +1011,32 @@ fn given_expression_macro_bodies_when_formatting_twice_then_every_dialect_lays_t let test_cases = [ ExpressionMacroFormatTestCase { description: "a body with a comment between nested calls", + dialects: &["duckdb", "snowflake"], contents: NORMALISE_NAME_MACRO.to_string(), expected_formatted: NORMALISE_NAME_FORMATTED.to_string(), }, ExpressionMacroFormatTestCase { description: "a body nesting fifteen calls, authored flush left", + dialects: &["duckdb", "snowflake"], contents: nested_macro(15, ""), expected_formatted: nested_macro(15, " "), }, + ExpressionMacroFormatTestCase { + description: "a multi-line string after a backslash-escaped quote", + dialects: &["snowflake"], + contents: ESCAPED_QUOTE_MACRO.to_string(), + expected_formatted: ESCAPED_QUOTE_FORMATTED.to_string(), + }, + ExpressionMacroFormatTestCase { + description: "a block comment with indented lines", + dialects: &["duckdb", "snowflake"], + contents: BLOCK_COMMENT_MACRO.to_string(), + expected_formatted: BLOCK_COMMENT_FORMATTED.to_string(), + }, ]; for test_case in test_cases { - for dialect in ["duckdb", "snowflake"] { + for dialect in test_case.dialects { let once: String = formatted_macro(&test_case.contents, dialect)?; let twice: String = formatted_macro(&once, dialect)?; diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs index 7d35354a91..767de51d4f 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs @@ -116,12 +116,14 @@ pub(crate) struct SitePatternTestCase { pub(crate) struct UnwrappedExpressionTestCase { pub(crate) description: &'static str, + pub(crate) dialect: &'static str, pub(crate) formatted: &'static str, pub(crate) expected_expression: Option<&'static str>, } pub(crate) struct ExpressionMacroFormatTestCase { pub(crate) description: &'static str, + pub(crate) dialects: &'static [&'static str], pub(crate) contents: String, pub(crate) expected_formatted: String, } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 03a8793a1b..d6ab08535c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -929,6 +929,7 @@ class ExpressionMacroFormatTestCase: description: str macro_directory: str consumer_folders: tuple[str, ...] + authored_macro: str expected_macro: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py index 195b941829..611a807e8e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py @@ -329,14 +329,25 @@ def test_given_multibyte_text_before_restorations_when_formatting_then_output_is "macros/sql", # A project macro needs consumers in more than one folder to stay a project macro. ("staging", "marts"), + _NORMALISE_NAME_HEADER + _NORMALISE_NAME_AUTHORED, _NORMALISE_NAME_HEADER + _NORMALISE_NAME_FORMATTED, ), ExpressionMacroFormatTestCase( "scoped macro", "models/staging/_sqlbuild/_macros/sql", ("staging",), + _NORMALISE_NAME_HEADER + _NORMALISE_NAME_AUTHORED, _NORMALISE_NAME_HEADER + _NORMALISE_NAME_FORMATTED, ), + ExpressionMacroFormatTestCase( + "multi-line string after a backslash-escaped quote", + "models/staging/_sqlbuild/_macros/sql", + ("staging",), + _NORMALISE_NAME_HEADER + + "CONCAT(\n @column_name,\n 'O\\'Brien\n kept indent',\n 'tail'\n)\n", + _NORMALISE_NAME_HEADER + + "CONCAT(@column_name, 'O\\'Brien\n kept indent', 'tail')\n", + ), ], ids=lambda case: case.description, ) @@ -346,7 +357,7 @@ def test_given_snowflake_expression_macro_when_formatting_then_body_is_laid_out_ (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') macro: Path = tmp_path / test_case.macro_directory / "normalise_name.sql" macro.parent.mkdir(parents=True) - macro.write_text(_NORMALISE_NAME_HEADER + _NORMALISE_NAME_AUTHORED) + macro.write_text(test_case.authored_macro) for folder in test_case.consumer_folders: model: Path = tmp_path / "models" / folder / f"{folder}_customer_names.sql" model.parent.mkdir(parents=True, exist_ok=True)