From 16fc2bf0beb67fe185a1ac220a329fa85968f5f8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 15 Nov 2026 09:00:40 +0000 Subject: [PATCH] fix: lay out Snowflake expression macro bodies instead of dedenting them On Snowflake, `sqb format` removed one indentation level from an expression macro or scalar function body on every run, so `--check` never settled. The body was formatted as an unaliased SELECT item, which the Snowflake formatter keeps exactly as written, and the dedent that removes the SELECT again then stripped an authored level from every line. Expression bodies are now formatted as an aliased item, which every dialect lays out, and the alias is removed again; a result without it is a fault that leaves the file unchanged. The dedent also leaves lines inside a $$ dollar-quoted string alone instead of changing the string. DuckShop gains a scoped Snowflake expression macro whose body holds a comment. --- benchmarks/duckshop/README.md | 2 +- .../_macros/sql/collapse_whitespace.sql | 12 +++ .../edge_cases/stg_edge_ticket_notes.sql | 9 +- .../src/lint_text/_helpers/format_bodies.rs | 56 ++++++++++-- .../sqlbuild-rules/src/lint_text/constants.rs | 6 +- .../src/lint_text/tests/helpers.rs | 40 ++++++++- .../src/lint_text/tests/test_lint_text.rs | 87 ++++++++++++++++++- .../src/lint_text/tests/test_types.rs | 6 ++ .../sqlbuild/cli/commands/main/_test_types.py | 10 +++ .../commands/main/test_format_preservation.py | 64 +++++++++++++- 10 files changed, 271 insertions(+), 21 deletions(-) create mode 100644 benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index cdaa718f59..321a5f9967 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -133,7 +133,7 @@ function, a hook-created JavaScript function, 192 unit tests and 4 scenarios, in | Ordering through arrays | an unordered `ARRAY_AGG` ranked under `LIMIT` by position, by name, through `IFF` and over `UNION ALL`, with fixtures whose pick does not depend on element order; order-free reads (`ARRAY_SIZE`, `ARRAY_CONTAINS`, `ARRAYS_OVERLAP`, `ARRAY_DISTINCT` of `ARRAY_CAT`, `FLATTEN` values through a CTE); `SELECT DISTINCT` over arrays repeated in different orders; `ARRAY_APPEND`, `ARRAY_INTERSECTION`, `ARRAY_SLICE` with negative bounds, `TO_ARRAY` and bracket array literals | | VARIANT comparisons | `ARRAY_MAX`, `ARRAY_MIN` and `ARRAY_REVERSE` over arrays mixing numbers, strings, booleans, JSON nulls and integers past 2^53; `MAX` across kinds and non-ASCII strings; `=` and `EQUAL_NULL` for `1`, `"1"` and `1.0`, reordered keys, JSON null and DATE or TIMESTAMP values held in a `VARIANT` against their text; `MIN_BY` and `MAX_BY` of `VARIANT` values; `PARTITION BY`, `SELECT DISTINCT` and `ARRAY_AGG(DISTINCT)` over `VARIANT`; JSON nulls in `ARRAY_REMOVE`, `ARRAY_DISTINCT` and `ARRAYS_OVERLAP`; decimals with trailing zeros in `ARRAY_CONTAINS` and `ARRAY_POSITION`; `OBJECT_CONSTRUCT(* EXCLUDE ("note"))` | | Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps; `TRY_TO_DATE` with numeric and month-name formats over unpadded fields, two-digit years, ISO `T` and `Z`, trailing letters, a long s, mixed separators and runs of spaces; text read as `TIMESTAMP_TZ` and `TIMESTAMP_LTZ` with `T`, `Z`, `+00:00`, `+0000`, `+09:00` and `-05:30`, back as the UTC instant, date, year and day start, and compared with text, `TIMESTAMP_NTZ` and `DATE`; `FIRST_VALUE` of a zoned value as a `DATE`; `DATEADD` and `TIMEADD` on `TIME(3)` and the type they return; `DATEDIFF` across units; `TIME(9)`, `FF3` and bare `FF` formats truncating; `DATE`, `TO_TIME`, `TRY_TO_TIMESTAMP` and `TO_TIMESTAMP_LTZ` of clock text, and `CURRENT_DATE` and `SYSDATE` read only for facts that hold on any day | -| Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | +| Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes, and a scoped SQL macro collapsing whitespace whose body holds a comment; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | | Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; relations aliased `first` and `last`, one without `AS`, the words of Snowflake's read-order clauses; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote, and columns qualified by their reference call, whose ranges are dropped | | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results | | Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild, and a keyed cursor `delete_insert` copy of it whose rows move out of the window between loads | diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql b/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql new file mode 100644 index 0000000000..ab10719442 --- /dev/null +++ b/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql @@ -0,0 +1,12 @@ +MACRO ( + description "Text with each run of whitespace collapsed to one space", + arguments (text_value VARCHAR), + returns VARCHAR, +); + +REGEXP_REPLACE( + @text_value, + -- Each run of spaces, tabs and line breaks becomes one space + '\\s+', + ' ' +)::VARCHAR diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql index bd9fd1d85d..f71accae7e 100644 --- a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql @@ -1,9 +1,10 @@ MODEL ( description """ One row per support ticket note with order references and reasons found by regular expressions, - written with escaped and dollar-quoted patterns, a SKU normalised by a macro after an escaped - quote, campaign and discount codes matched by pattern lists and escapes, and SKUs split at edge - positions; the note text and tags are passed on as written + written with escaped and dollar-quoted patterns, whitespace collapsed by a macro whose body + holds a comment, a SKU normalised by a macro after an escaped quote, campaign and discount + codes matched by pattern lists and escapes, and SKUs split at edge positions; the note text and + tags are passed on as written """, tags [edge_case], columns ( @@ -51,7 +52,7 @@ final AS ( REGEXP_LIKE(s.body, '.*ds-o-\\d+.*', 'is') AS mentions_order_regexp_like, REGEXP_COUNT(s.body, 'DS-O-\\d+')::INT AS order_ref_count, REGEXP_SUBSTR(s.body, 'refund(ed)?', 1, 1, 'i')::VARCHAR AS refund_word, - REGEXP_REPLACE(s.body, '\\s+', ' ')::VARCHAR AS body_collapsed, + @sql.collapse_whitespace(s.body) AS body_collapsed, // A backslash-escaped quote inside a single-quoted literal, then a macro call to expand CONTAINS(s.body, 'O\'Brien')::BOOLEAN AS quotes_customer, @sql.normalise_sku(s.sku) AS sku_canonical, diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs index c1691a80c8..e6d9ffd6e6 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs @@ -17,11 +17,11 @@ use crate::dsl_headers::main::sql_body_ranges::sql_body_ranges; use crate::dsl_headers::models::HeaderSpan; use crate::lint_text::_helpers::sites::compiled; use crate::lint_text::constants::{ - DECLARATION_GROUP_DIRECTORY, DECLINED_WITHOUT_REASON, EXPRESSION_SELECT_PREFIX, + DECLARATION_GROUP_DIRECTORY, DECLINED_WITHOUT_REASON, DOLLAR_QUOTE, EXPRESSION_SELECT_PREFIX, FIXTURE_HEADER_KINDS, FORMAT_API_VERSION, FUNCTION_HEADER_KIND, GLOBAL_NAMED_ROOTS, - GROUPED_NAMED_ROLES, LINE_COMMENT_PREFIX, MACRO_HEADER_KIND, MAX_FUNCTION_CALL_DEPTH, - MOVED_OMITTED_SELECT, OMITTED_CEREMONIAL_SELECT_SQL, OMITTED_SELECT_SUFFIX_PATTERN, - STRING_QUOTES, + GROUPED_NAMED_ROLES, LINE_COMMENT_PREFIX, LINTED_EXPRESSION_ALIAS, MACRO_HEADER_KIND, + MAX_FUNCTION_CALL_DEPTH, MOVED_EXPRESSION_ALIAS, MOVED_OMITTED_SELECT, + OMITTED_CEREMONIAL_SELECT_SQL, OMITTED_SELECT_SUFFIX_PATTERN, STRING_QUOTES, }; use crate::lint_text::errors::LintTextError; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; @@ -31,6 +31,9 @@ use sqlclean::sql_lint::models::{FormatRequest, FormatResponse}; static OMITTED_SELECT_SUFFIX: LazyLock> = LazyLock::new(|| Regex::new(OMITTED_SELECT_SUFFIX_PATTERN)); +/// The alias that closes a formatted expression body, with or without an `AS` before it. +static EXPRESSION_ALIAS_SUFFIX: LazyLock> = + LazyLock::new(|| Regex::new(&format!(r"(?i)\s+(?:AS\s+)?{LINTED_EXPRESSION_ALIAS}\s*\z"))); /// Deduplicated format requests, keyed by neutralized SQL and sentinel widths. #[derive(Default)] @@ -157,16 +160,26 @@ pub(crate) fn prepare_file( trailing = characters[offset..].iter().collect::() + &trailing; core = characters[..offset].iter().collect(); } + if wraps_expression && let Some(statement) = core.strip_suffix(';') { + trailing = format!(";{trailing}"); + core = statement.trim_end().to_string(); + } let neutralized = neutralize_interpolation(&core, dialect, true)?; let mut sql: String = if wraps_expression { - format!("{EXPRESSION_SELECT_PREFIX}{}", neutralized.text) + format!( + "{EXPRESSION_SELECT_PREFIX}{}\n{LINTED_EXPRESSION_ALIAS}", + neutralized.text + ) } else { neutralized.text }; if omitted.is_some() { sql.push_str(OMITTED_CEREMONIAL_SELECT_SQL); } - let widths: BTreeMap = sentinel_widths(&neutralized.sites); + let mut widths: BTreeMap = sentinel_widths(&neutralized.sites); + if wraps_expression { + let _ = widths.insert(LINTED_EXPRESSION_ALIAS.to_uppercase(), 0); + } let body: PreparedBody = PreparedBody { start, end, @@ -187,7 +200,7 @@ enum MacroBodyForm { CteList, /// A `SELECT` list, formatted as it stands. ColumnList, - /// An expression, formatted as the one item of a `SELECT` that is removed again. + /// An expression, formatted as the one aliased item of a `SELECT` that is removed again. Expression, } @@ -269,8 +282,15 @@ pub(crate) fn formatted_body( } let unwrapped: String; if prepared.wraps_expression { - unwrapped = unwrapped_expression(sql).ok_or_else(|| MOVED_OMITTED_SELECT.to_string())?; + let pattern: &Regex = + compiled(&EXPRESSION_ALIAS_SUFFIX).map_err(|error| error.to_string())?; + let Some(alias) = pattern.find(sql) else { + return Err(MOVED_EXPRESSION_ALIAS.to_string()); + }; + unwrapped = unwrapped_expression(&sql[..alias.start()]) + .ok_or_else(|| MOVED_EXPRESSION_ALIAS.to_string())?; sql = &unwrapped; + changed = true; } if !changed { return Ok(None); @@ -349,11 +369,29 @@ pub(crate) fn unwrapped_expression(formatted: &str) -> Option { Some(dedented.join("\n").trim_end().to_owned()) } -/// The quote still open after `line`, given the one open before it; a doubled quote escapes. +/// The quote still open after `line`, given the one before it; `$` stands for `$$`, which has no escapes. fn open_quote(mut quote: Option, line: &str) -> Option { let mut characters = line.chars().peekable(); + let mut previous: Option = None; while let Some(character) = characters.next() { + let after_word: bool = previous.is_some_and(|before| { + before.is_alphanumeric() || before == '_' || before == DOLLAR_QUOTE + }); + previous = Some(character); + if character == DOLLAR_QUOTE + && (quote == Some(DOLLAR_QUOTE) || (quote.is_none() && !after_word)) + && characters.peek() == Some(&DOLLAR_QUOTE) + { + let _ = characters.next(); + quote = if quote.is_some() { + None + } else { + Some(DOLLAR_QUOTE) + }; + continue; + } match quote { + Some(DOLLAR_QUOTE) => {} Some(open) if character == open => { if characters.peek() == Some(&open) { let _ = characters.next(); diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index a629cf9be8..416d21b21e 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -79,6 +79,8 @@ pub(crate) const LINE_COMMENT_PREFIX: &str = "--"; pub(crate) const DECLINED_WITHOUT_REASON: &str = "native formatter declined without a reason"; pub(crate) const MOVED_OMITTED_SELECT: &str = "native formatter moved the omitted ceremonial SELECT 1"; +pub(crate) const MOVED_EXPRESSION_ALIAS: &str = + "native formatter moved the alias it formats an expression body under"; pub(crate) const DECLARATION_GROUP_DIRECTORY: &str = "_sqlbuild"; /// `GLOBAL_NAMED_DECLARATION_ROOTS`, in Python's dict order, with whether each holds audits. pub(crate) const GLOBAL_NAMED_ROOTS: [(&[&str], bool); 5] = [ @@ -211,10 +213,12 @@ pub(crate) const GROUPED_NAMED_ROLE_KINDS: [(&[&str], &str); 9] = [ pub(crate) const CROSS_KEYWORD: &[u8] = b"cross"; /// What a SQL macro expression body is formatted after, and removed from again. pub(crate) const EXPRESSION_SELECT_PREFIX: &str = "SELECT "; -/// The output name lint gives an expression macro body, so it is not an unnamed projection. +/// The output name lint and format give an expression body; Snowflake keeps unnamed items as written. pub(crate) const LINTED_EXPRESSION_ALIAS: &str = "__sqlbuild_macro_expression__"; /// The characters that open a quoted string or identifier in formatted SQL. pub(crate) const STRING_QUOTES: [char; 3] = ['\'', '"', '`']; +/// Each half of the `$$` that opens and closes a dollar-quoted string in formatted SQL. +pub(crate) const DOLLAR_QUOTE: char = '$'; /// A CTE definition, capturing its `name`. pub(crate) const CTE_DEFINITION_PATTERN: &str = r#"(?is)(?:\bWITH(?:\s+RECURSIVE)?|,)\s*["`\[]?(?P[A-Za-z_][A-Za-z0-9_]*)["`\]]?(?:\s*\([^)]*\))?\s+AS\s*\("#; diff --git a/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs b/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs index 96a8aef7f0..4c59e34e9a 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs @@ -1,5 +1,7 @@ use crate::lint_text::constants::{DIRECTIVE_FIX_REASON, DIRECTIVE_MESSAGE}; -use crate::lint_text::models::SuppressionFinding; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::main::format_sql_bodies::format_sql_bodies; +use crate::lint_text::models::{FormatFile, SuppressionFinding}; const INDENT: &str = "\n "; @@ -23,3 +25,39 @@ pub(crate) fn directive_finding(line: usize, rule: &str, rest: &[&str]) -> Suppr fix_unavailable_reason: Some(DIRECTIVE_FIX_REASON), } } + +/// A macro nesting `depth` calls, each with a comment that keeps it on several lines, `step` per level. +pub(crate) fn nested_macro(depth: usize, step: &str) -> String { + let mut body: String = String::new(); + for level in 0..depth { + body.push_str(&format!("{}COALESCE(\n", step.repeat(level))); + } + body.push_str(&format!("{}@value_text", step.repeat(depth))); + for level in (0..depth).rev() { + let inner: String = step.repeat(level + 1); + body.push_str(&format!( + ",\n{inner}-- Level {level}\n{inner}'level {level}'\n{})", + step.repeat(level) + )); + } + format!("MACRO (arguments (value_text VARCHAR), returns VARCHAR);\n\n{body}\n") +} + +/// The project macro file `contents` as one `sqb format` run on `dialect` leaves it, without faults. +pub(crate) fn formatted_macro(contents: &str, dialect: &str) -> Result { + let files = [FormatFile { + relative_parts: Some(vec![ + "macros".to_string(), + "sql".to_string(), + "m.sql".to_string(), + ]), + contents: contents.to_string(), + }]; + let result = format_sql_bodies(&files, dialect, 100)?; + assert_eq!(result.faults, Vec::new(), "{dialect}"); + Ok(result + .formatted + .into_iter() + .next() + .map_or_else(|| contents.to_string(), |(_, text)| text)) +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index 4416a617e3..31cc743b7a 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -18,11 +18,12 @@ use crate::lint_text::main::neutralize_test_parameters::neutralize_test_paramete use crate::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use crate::lint_text::main::restore_interpolation::restore_interpolation; use crate::lint_text::models::{ExpansionSpan, FormatFile, LintSqlBody, LintSqlOptions}; -use crate::lint_text::tests::helpers::directive_finding; +use crate::lint_text::tests::helpers::{directive_finding, formatted_macro, nested_macro}; use crate::lint_text::tests::test_types::{ - ExitCountTestCase, Finding, FormatBodiesTestCase, LintBodiesTestCase, NeutralizeTestCase, - ParameterTestCase, RestoreTestCase, SitePatternTestCase, SlotReadTestCase, SuppressionTestCase, - TokenAtTestCase, UnavailableFinding, UnavailableLintTestCase, UnwrappedExpressionTestCase, + ExitCountTestCase, ExpressionMacroFormatTestCase, Finding, FormatBodiesTestCase, + LintBodiesTestCase, NeutralizeTestCase, ParameterTestCase, RestoreTestCase, + SitePatternTestCase, SlotReadTestCase, SuppressionTestCase, TokenAtTestCase, + UnavailableFinding, UnavailableLintTestCase, UnwrappedExpressionTestCase, }; use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; @@ -30,6 +31,9 @@ const SUPPRESSED_FILE: &str = "-- sqb: ignore SQLCLEAN005 because kept\n\n-- note\nselect 1\n-- sqb: ignore nope\nselect 2\n"; const CROSS_JOIN_FILE: &str = "select o.order_id\nfrom orders as o\n-- sqb: ignore SQLCLEAN038 because every order meets every policy\ncross join __seed(\"order_policy\") as p\n"; +const NORMALISE_NAME_MACRO: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(\n @column_name,\n '[[:space:]]+',\n ' '\n ))),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n"; +const NORMALISE_NAME_FORMATTED: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(\n UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(@column_name, '[[:space:]]+', ' ')\n )\n ),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n"; + #[test] fn given_site_patterns_when_text_opens_with_another_byte_then_no_site_starts_there() -> Result<(), LintTextError> { @@ -509,6 +513,36 @@ fn given_sql_files_when_formatting_bodies_then_sites_and_fixture_ceremony_surviv )], expected_fault_lines: vec![], }, + FormatBodiesTestCase { + description: "an expression macro body ending in a semicolon", + dialect: "snowflake", + relative_parts: None, + contents: "MACRO (arguments (q INTEGER), returns INTEGER);\n\n@q + 1 ;\n", + expected_formatted: vec![( + 0, + "MACRO (arguments (q INTEGER), returns INTEGER);\n\n@q + 1;\n".to_string(), + )], + expected_fault_lines: vec![], + }, + FormatBodiesTestCase { + description: "an expression macro body keeps the line comment that ends it", + dialect: "snowflake", + relative_parts: None, + contents: "MACRO (arguments (q INTEGER), returns INTEGER);\n\n@q + 1 -- note\n", + expected_formatted: vec![( + 0, + "MACRO (arguments (q INTEGER), returns INTEGER);\n\n@q + 1 -- note\n".to_string(), + )], + expected_fault_lines: vec![], + }, + FormatBodiesTestCase { + description: "an unparseable Snowflake expression macro body is a fault, not rewritten", + dialect: "snowflake", + relative_parts: None, + contents: "MACRO (arguments (q VARCHAR), returns VARCHAR);\n\nTRIM(\n UPPER(@q,,\n )\n", + expected_formatted: vec![], + expected_fault_lines: vec![3], + }, FormatBodiesTestCase { description: "an unparseable body is a fault on its first line", dialect: "duckdb", @@ -627,6 +661,16 @@ fn given_a_formatted_expression_select_when_unwrapping_then_strings_keep_their_l formatted: "SELECT\n CONCAT(\n 'first\n second',\n @v\n )", expected_expression: Some("CONCAT(\n 'first\n second',\n @v\n)"), }, + UnwrappedExpressionTestCase { + description: "lines inside a dollar-quoted string are not dedented", + formatted: "SELECT\n CONCAT(\n $$first\n 'second'\n $$,\n @v\n )", + expected_expression: Some("CONCAT(\n $$first\n 'second'\n $$,\n @v\n)"), + }, + UnwrappedExpressionTestCase { + description: "a $$ inside an identifier does not open a string", + formatted: "SELECT\n COALESCE(\n col$$x,\n @v\n )", + expected_expression: Some("COALESCE(\n col$$x,\n @v\n)"), + }, UnwrappedExpressionTestCase { description: "text that is not a select is not an expression", formatted: "WITH a AS (SELECT 1)", @@ -923,3 +967,38 @@ fn given_findings_when_deciding_exit_then_only_lint_unavailable_never_fails_a_co ); } } + +#[test] +fn given_expression_macro_bodies_when_formatting_twice_then_every_dialect_lays_them_out_stably() +-> Result<(), LintTextError> { + let test_cases = [ + ExpressionMacroFormatTestCase { + description: "a body with a comment between nested calls", + contents: NORMALISE_NAME_MACRO.to_string(), + expected_formatted: NORMALISE_NAME_FORMATTED.to_string(), + }, + ExpressionMacroFormatTestCase { + description: "a body nesting fifteen calls, authored flush left", + contents: nested_macro(15, ""), + expected_formatted: nested_macro(15, " "), + }, + ]; + + for test_case in test_cases { + for dialect in ["duckdb", "snowflake"] { + let once: String = formatted_macro(&test_case.contents, dialect)?; + let twice: String = formatted_macro(&once, dialect)?; + + assert_eq!( + (once.as_str(), twice.as_str()), + ( + test_case.expected_formatted.as_str(), + test_case.expected_formatted.as_str() + ), + "{} on {dialect}", + test_case.description + ); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs index b2c56ed338..7d35354a91 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs @@ -119,3 +119,9 @@ pub(crate) struct UnwrappedExpressionTestCase { pub(crate) formatted: &'static str, pub(crate) expected_expression: Option<&'static str>, } + +pub(crate) struct ExpressionMacroFormatTestCase { + pub(crate) description: &'static str, + pub(crate) contents: String, + pub(crate) expected_formatted: String, +} diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index a847abb71f..03a8793a1b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -922,6 +922,16 @@ class SqlReadabilityTestCase: rule_code: str = "SQLCLEAN041" +@dataclass(frozen=True) +class ExpressionMacroFormatTestCase: + """A Snowflake expression macro in one location, and the file formatting leaves.""" + + description: str + macro_directory: str + consumer_folders: tuple[str, ...] + expected_macro: str + + @dataclass(frozen=True) class FormatterSyntaxTestCase: """Authored constructs that must survive canonical layout formatting.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py index f7e146fec1..195b941829 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py @@ -6,7 +6,10 @@ import pytest from sqlbuild._native import run_cli as main -from tests.integration.src.sqlbuild.cli.commands.main._test_types import FormatterSyntaxTestCase +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + ExpressionMacroFormatTestCase, + FormatterSyntaxTestCase, +) # The window syntax cases order by a non-unique column, which SQLCLEAN018 reports. _WINDOW_SYNTAX_PROJECT_TOML: str = ( @@ -14,6 +17,22 @@ ) +_NORMALISE_NAME_HEADER: str = ( + 'MACRO (\n description "Normalise a name",\n arguments (column_name VARCHAR),\n' + " returns VARCHAR,\n);\n\n" +) +_NORMALISE_NAME_AUTHORED: str = ( + "REGEXP_REPLACE(\n TRIM(UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(\n" + " @column_name,\n '[[:space:]]+',\n ' '\n ))),\n '(^| )([A-Z]) ',\n" + " '\\\\1\\\\2'\n)\n" +) +_NORMALISE_NAME_FORMATTED: str = ( + "REGEXP_REPLACE(\n TRIM(\n UPPER(\n -- Collapse repeated spaces\n" + " REGEXP_REPLACE(@column_name, '[[:space:]]+', ' ')\n )\n ),\n" + " '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n" +) + + @pytest.mark.parametrize( "test_case", [ @@ -302,5 +321,48 @@ def test_given_multibyte_text_before_restorations_when_formatting_then_output_is assert model.read_text(encoding="utf-8") == formatted +@pytest.mark.parametrize( + "test_case", + [ + ExpressionMacroFormatTestCase( + "project macro", + "macros/sql", + # A project macro needs consumers in more than one folder to stay a project macro. + ("staging", "marts"), + _NORMALISE_NAME_HEADER + _NORMALISE_NAME_FORMATTED, + ), + ExpressionMacroFormatTestCase( + "scoped macro", + "models/staging/_sqlbuild/_macros/sql", + ("staging",), + _NORMALISE_NAME_HEADER + _NORMALISE_NAME_FORMATTED, + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_expression_macro_when_formatting_then_body_is_laid_out_once( + test_case: ExpressionMacroFormatTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') + macro: Path = tmp_path / test_case.macro_directory / "normalise_name.sql" + macro.parent.mkdir(parents=True) + macro.write_text(_NORMALISE_NAME_HEADER + _NORMALISE_NAME_AUTHORED) + for folder in test_case.consumer_folders: + model: Path = tmp_path / "models" / folder / f"{folder}_customer_names.sql" + model.parent.mkdir(parents=True, exist_ok=True) + model.write_text( + 'MODEL (description "Customer names", database warehouse, schema analytics);\n' + "SELECT @sql.normalise_name('ada lovelace') AS customer_name\n" + ) + assert main(["--project-dir", str(tmp_path), "format", "--json"]) == 0 + payload: dict[str, object] = json.loads(capsys.readouterr().out) + assert payload["faults"] == 0 + assert macro.read_text() == test_case.expected_macro + assert main(["--project-dir", str(tmp_path), "format", "--check", "--json"]) == 0 + assert main(["--project-dir", str(tmp_path), "format", "--json"]) == 0 + assert macro.read_text() == test_case.expected_macro + assert main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) == 0 + + if __name__ == "__main__": pytest.main([__file__, "-vv"])