diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index cdaa718f59..321a5f9967 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -133,7 +133,7 @@ function, a hook-created JavaScript function, 192 unit tests and 4 scenarios, in | Ordering through arrays | an unordered `ARRAY_AGG` ranked under `LIMIT` by position, by name, through `IFF` and over `UNION ALL`, with fixtures whose pick does not depend on element order; order-free reads (`ARRAY_SIZE`, `ARRAY_CONTAINS`, `ARRAYS_OVERLAP`, `ARRAY_DISTINCT` of `ARRAY_CAT`, `FLATTEN` values through a CTE); `SELECT DISTINCT` over arrays repeated in different orders; `ARRAY_APPEND`, `ARRAY_INTERSECTION`, `ARRAY_SLICE` with negative bounds, `TO_ARRAY` and bracket array literals | | VARIANT comparisons | `ARRAY_MAX`, `ARRAY_MIN` and `ARRAY_REVERSE` over arrays mixing numbers, strings, booleans, JSON nulls and integers past 2^53; `MAX` across kinds and non-ASCII strings; `=` and `EQUAL_NULL` for `1`, `"1"` and `1.0`, reordered keys, JSON null and DATE or TIMESTAMP values held in a `VARIANT` against their text; `MIN_BY` and `MAX_BY` of `VARIANT` values; `PARTITION BY`, `SELECT DISTINCT` and `ARRAY_AGG(DISTINCT)` over `VARIANT`; JSON nulls in `ARRAY_REMOVE`, `ARRAY_DISTINCT` and `ARRAYS_OVERLAP`; decimals with trailing zeros in `ARRAY_CONTAINS` and `ARRAY_POSITION`; `OBJECT_CONSTRUCT(* EXCLUDE ("note"))` | | Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps; `TRY_TO_DATE` with numeric and month-name formats over unpadded fields, two-digit years, ISO `T` and `Z`, trailing letters, a long s, mixed separators and runs of spaces; text read as `TIMESTAMP_TZ` and `TIMESTAMP_LTZ` with `T`, `Z`, `+00:00`, `+0000`, `+09:00` and `-05:30`, back as the UTC instant, date, year and day start, and compared with text, `TIMESTAMP_NTZ` and `DATE`; `FIRST_VALUE` of a zoned value as a `DATE`; `DATEADD` and `TIMEADD` on `TIME(3)` and the type they return; `DATEDIFF` across units; `TIME(9)`, `FF3` and bare `FF` formats truncating; `DATE`, `TO_TIME`, `TRY_TO_TIMESTAMP` and `TO_TIMESTAMP_LTZ` of clock text, and `CURRENT_DATE` and `SYSDATE` read only for facts that hold on any day | -| Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | +| Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes, and a scoped SQL macro collapsing whitespace whose body holds a comment; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | | Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; relations aliased `first` and `last`, one without `AS`, the words of Snowflake's read-order clauses; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote, and columns qualified by their reference call, whose ranges are dropped | | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results | | Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild, and a keyed cursor `delete_insert` copy of it whose rows move out of the window between loads | diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql b/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql new file mode 100644 index 0000000000..ab10719442 --- /dev/null +++ b/benchmarks/duckshop/project/models/staging/edge_cases/_sqlbuild/_macros/sql/collapse_whitespace.sql @@ -0,0 +1,12 @@ +MACRO ( + description "Text with each run of whitespace collapsed to one space", + arguments (text_value VARCHAR), + returns VARCHAR, +); + +REGEXP_REPLACE( + @text_value, + -- Each run of spaces, tabs and line breaks becomes one space + '\\s+', + ' ' +)::VARCHAR diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql index bd9fd1d85d..f71accae7e 100644 --- a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql @@ -1,9 +1,10 @@ MODEL ( description """ One row per support ticket note with order references and reasons found by regular expressions, - written with escaped and dollar-quoted patterns, a SKU normalised by a macro after an escaped - quote, campaign and discount codes matched by pattern lists and escapes, and SKUs split at edge - positions; the note text and tags are passed on as written + written with escaped and dollar-quoted patterns, whitespace collapsed by a macro whose body + holds a comment, a SKU normalised by a macro after an escaped quote, campaign and discount + codes matched by pattern lists and escapes, and SKUs split at edge positions; the note text and + tags are passed on as written """, tags [edge_case], columns ( @@ -51,7 +52,7 @@ final AS ( REGEXP_LIKE(s.body, '.*ds-o-\\d+.*', 'is') AS mentions_order_regexp_like, REGEXP_COUNT(s.body, 'DS-O-\\d+')::INT AS order_ref_count, REGEXP_SUBSTR(s.body, 'refund(ed)?', 1, 1, 'i')::VARCHAR AS refund_word, - REGEXP_REPLACE(s.body, '\\s+', ' ')::VARCHAR AS body_collapsed, + @sql.collapse_whitespace(s.body) AS body_collapsed, // A backslash-escaped quote inside a single-quoted literal, then a macro call to expand CONTAINS(s.body, 'O\'Brien')::BOOLEAN AS quotes_customer, @sql.normalise_sku(s.sku) AS sku_canonical, diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs index c1691a80c8..e6d9ffd6e6 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs @@ -17,11 +17,11 @@ use crate::dsl_headers::main::sql_body_ranges::sql_body_ranges; use crate::dsl_headers::models::HeaderSpan; use crate::lint_text::_helpers::sites::compiled; use crate::lint_text::constants::{ - DECLARATION_GROUP_DIRECTORY, DECLINED_WITHOUT_REASON, EXPRESSION_SELECT_PREFIX, + DECLARATION_GROUP_DIRECTORY, DECLINED_WITHOUT_REASON, DOLLAR_QUOTE, EXPRESSION_SELECT_PREFIX, FIXTURE_HEADER_KINDS, FORMAT_API_VERSION, FUNCTION_HEADER_KIND, GLOBAL_NAMED_ROOTS, - GROUPED_NAMED_ROLES, LINE_COMMENT_PREFIX, MACRO_HEADER_KIND, MAX_FUNCTION_CALL_DEPTH, - MOVED_OMITTED_SELECT, OMITTED_CEREMONIAL_SELECT_SQL, OMITTED_SELECT_SUFFIX_PATTERN, - STRING_QUOTES, + GROUPED_NAMED_ROLES, LINE_COMMENT_PREFIX, LINTED_EXPRESSION_ALIAS, MACRO_HEADER_KIND, + MAX_FUNCTION_CALL_DEPTH, MOVED_EXPRESSION_ALIAS, MOVED_OMITTED_SELECT, + OMITTED_CEREMONIAL_SELECT_SQL, OMITTED_SELECT_SUFFIX_PATTERN, STRING_QUOTES, }; use crate::lint_text::errors::LintTextError; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; @@ -31,6 +31,9 @@ use sqlclean::sql_lint::models::{FormatRequest, FormatResponse}; static OMITTED_SELECT_SUFFIX: LazyLock> = LazyLock::new(|| Regex::new(OMITTED_SELECT_SUFFIX_PATTERN)); +/// The alias that closes a formatted expression body, with or without an `AS` before it. +static EXPRESSION_ALIAS_SUFFIX: LazyLock> = + LazyLock::new(|| Regex::new(&format!(r"(?i)\s+(?:AS\s+)?{LINTED_EXPRESSION_ALIAS}\s*\z"))); /// Deduplicated format requests, keyed by neutralized SQL and sentinel widths. #[derive(Default)] @@ -157,16 +160,26 @@ pub(crate) fn prepare_file( trailing = characters[offset..].iter().collect::() + &trailing; core = characters[..offset].iter().collect(); } + if wraps_expression && let Some(statement) = core.strip_suffix(';') { + trailing = format!(";{trailing}"); + core = statement.trim_end().to_string(); + } let neutralized = neutralize_interpolation(&core, dialect, true)?; let mut sql: String = if wraps_expression { - format!("{EXPRESSION_SELECT_PREFIX}{}", neutralized.text) + format!( + "{EXPRESSION_SELECT_PREFIX}{}\n{LINTED_EXPRESSION_ALIAS}", + neutralized.text + ) } else { neutralized.text }; if omitted.is_some() { sql.push_str(OMITTED_CEREMONIAL_SELECT_SQL); } - let widths: BTreeMap = sentinel_widths(&neutralized.sites); + let mut widths: BTreeMap = sentinel_widths(&neutralized.sites); + if wraps_expression { + let _ = widths.insert(LINTED_EXPRESSION_ALIAS.to_uppercase(), 0); + } let body: PreparedBody = PreparedBody { start, end, @@ -187,7 +200,7 @@ enum MacroBodyForm { CteList, /// A `SELECT` list, formatted as it stands. ColumnList, - /// An expression, formatted as the one item of a `SELECT` that is removed again. + /// An expression, formatted as the one aliased item of a `SELECT` that is removed again. Expression, } @@ -269,8 +282,15 @@ pub(crate) fn formatted_body( } let unwrapped: String; if prepared.wraps_expression { - unwrapped = unwrapped_expression(sql).ok_or_else(|| MOVED_OMITTED_SELECT.to_string())?; + let pattern: &Regex = + compiled(&EXPRESSION_ALIAS_SUFFIX).map_err(|error| error.to_string())?; + let Some(alias) = pattern.find(sql) else { + return Err(MOVED_EXPRESSION_ALIAS.to_string()); + }; + unwrapped = unwrapped_expression(&sql[..alias.start()]) + .ok_or_else(|| MOVED_EXPRESSION_ALIAS.to_string())?; sql = &unwrapped; + changed = true; } if !changed { return Ok(None); @@ -349,11 +369,29 @@ pub(crate) fn unwrapped_expression(formatted: &str) -> Option { Some(dedented.join("\n").trim_end().to_owned()) } -/// The quote still open after `line`, given the one open before it; a doubled quote escapes. +/// The quote still open after `line`, given the one before it; `$` stands for `$$`, which has no escapes. fn open_quote(mut quote: Option, line: &str) -> Option { let mut characters = line.chars().peekable(); + let mut previous: Option = None; while let Some(character) = characters.next() { + let after_word: bool = previous.is_some_and(|before| { + before.is_alphanumeric() || before == '_' || before == DOLLAR_QUOTE + }); + previous = Some(character); + if character == DOLLAR_QUOTE + && (quote == Some(DOLLAR_QUOTE) || (quote.is_none() && !after_word)) + && characters.peek() == Some(&DOLLAR_QUOTE) + { + let _ = characters.next(); + quote = if quote.is_some() { + None + } else { + Some(DOLLAR_QUOTE) + }; + continue; + } match quote { + Some(DOLLAR_QUOTE) => {} Some(open) if character == open => { if characters.peek() == Some(&open) { let _ = characters.next(); diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index a629cf9be8..416d21b21e 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -79,6 +79,8 @@ pub(crate) const LINE_COMMENT_PREFIX: &str = "--"; pub(crate) const DECLINED_WITHOUT_REASON: &str = "native formatter declined without a reason"; pub(crate) const MOVED_OMITTED_SELECT: &str = "native formatter moved the omitted ceremonial SELECT 1"; +pub(crate) const MOVED_EXPRESSION_ALIAS: &str = + "native formatter moved the alias it formats an expression body under"; pub(crate) const DECLARATION_GROUP_DIRECTORY: &str = "_sqlbuild"; /// `GLOBAL_NAMED_DECLARATION_ROOTS`, in Python's dict order, with whether each holds audits. pub(crate) const GLOBAL_NAMED_ROOTS: [(&[&str], bool); 5] = [ @@ -211,10 +213,12 @@ pub(crate) const GROUPED_NAMED_ROLE_KINDS: [(&[&str], &str); 9] = [ pub(crate) const CROSS_KEYWORD: &[u8] = b"cross"; /// What a SQL macro expression body is formatted after, and removed from again. pub(crate) const EXPRESSION_SELECT_PREFIX: &str = "SELECT "; -/// The output name lint gives an expression macro body, so it is not an unnamed projection. +/// The output name lint and format give an expression body; Snowflake keeps unnamed items as written. pub(crate) const LINTED_EXPRESSION_ALIAS: &str = "__sqlbuild_macro_expression__"; /// The characters that open a quoted string or identifier in formatted SQL. pub(crate) const STRING_QUOTES: [char; 3] = ['\'', '"', '`']; +/// Each half of the `$$` that opens and closes a dollar-quoted string in formatted SQL. +pub(crate) const DOLLAR_QUOTE: char = '$'; /// A CTE definition, capturing its `name`. pub(crate) const CTE_DEFINITION_PATTERN: &str = r#"(?is)(?:\bWITH(?:\s+RECURSIVE)?|,)\s*["`\[]?(?P[A-Za-z_][A-Za-z0-9_]*)["`\]]?(?:\s*\([^)]*\))?\s+AS\s*\("#; diff --git a/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs b/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs index 96a8aef7f0..4c59e34e9a 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs @@ -1,5 +1,7 @@ use crate::lint_text::constants::{DIRECTIVE_FIX_REASON, DIRECTIVE_MESSAGE}; -use crate::lint_text::models::SuppressionFinding; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::main::format_sql_bodies::format_sql_bodies; +use crate::lint_text::models::{FormatFile, SuppressionFinding}; const INDENT: &str = "\n "; @@ -23,3 +25,39 @@ pub(crate) fn directive_finding(line: usize, rule: &str, rest: &[&str]) -> Suppr fix_unavailable_reason: Some(DIRECTIVE_FIX_REASON), } } + +/// A macro nesting `depth` calls, each with a comment that keeps it on several lines, `step` per level. +pub(crate) fn nested_macro(depth: usize, step: &str) -> String { + let mut body: String = String::new(); + for level in 0..depth { + body.push_str(&format!("{}COALESCE(\n", step.repeat(level))); + } + body.push_str(&format!("{}@value_text", step.repeat(depth))); + for level in (0..depth).rev() { + let inner: String = step.repeat(level + 1); + body.push_str(&format!( + ",\n{inner}-- Level {level}\n{inner}'level {level}'\n{})", + step.repeat(level) + )); + } + format!("MACRO (arguments (value_text VARCHAR), returns VARCHAR);\n\n{body}\n") +} + +/// The project macro file `contents` as one `sqb format` run on `dialect` leaves it, without faults. +pub(crate) fn formatted_macro(contents: &str, dialect: &str) -> Result { + let files = [FormatFile { + relative_parts: Some(vec![ + "macros".to_string(), + "sql".to_string(), + "m.sql".to_string(), + ]), + contents: contents.to_string(), + }]; + let result = format_sql_bodies(&files, dialect, 100)?; + assert_eq!(result.faults, Vec::new(), "{dialect}"); + Ok(result + .formatted + .into_iter() + .next() + .map_or_else(|| contents.to_string(), |(_, text)| text)) +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index 4416a617e3..31cc743b7a 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -18,11 +18,12 @@ use crate::lint_text::main::neutralize_test_parameters::neutralize_test_paramete use crate::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use crate::lint_text::main::restore_interpolation::restore_interpolation; use crate::lint_text::models::{ExpansionSpan, FormatFile, LintSqlBody, LintSqlOptions}; -use crate::lint_text::tests::helpers::directive_finding; +use crate::lint_text::tests::helpers::{directive_finding, formatted_macro, nested_macro}; use crate::lint_text::tests::test_types::{ - ExitCountTestCase, Finding, FormatBodiesTestCase, LintBodiesTestCase, NeutralizeTestCase, - ParameterTestCase, RestoreTestCase, SitePatternTestCase, SlotReadTestCase, SuppressionTestCase, - TokenAtTestCase, UnavailableFinding, UnavailableLintTestCase, UnwrappedExpressionTestCase, + ExitCountTestCase, ExpressionMacroFormatTestCase, Finding, FormatBodiesTestCase, + LintBodiesTestCase, NeutralizeTestCase, ParameterTestCase, RestoreTestCase, + SitePatternTestCase, SlotReadTestCase, SuppressionTestCase, TokenAtTestCase, + UnavailableFinding, UnavailableLintTestCase, UnwrappedExpressionTestCase, }; use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; @@ -30,6 +31,9 @@ const SUPPRESSED_FILE: &str = "-- sqb: ignore SQLCLEAN005 because kept\n\n-- note\nselect 1\n-- sqb: ignore nope\nselect 2\n"; const CROSS_JOIN_FILE: &str = "select o.order_id\nfrom orders as o\n-- sqb: ignore SQLCLEAN038 because every order meets every policy\ncross join __seed(\"order_policy\") as p\n"; +const NORMALISE_NAME_MACRO: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(\n @column_name,\n '[[:space:]]+',\n ' '\n ))),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n"; +const NORMALISE_NAME_FORMATTED: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(\n UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(@column_name, '[[:space:]]+', ' ')\n )\n ),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n"; + #[test] fn given_site_patterns_when_text_opens_with_another_byte_then_no_site_starts_there() -> Result<(), LintTextError> { @@ -509,6 +513,36 @@ fn given_sql_files_when_formatting_bodies_then_sites_and_fixture_ceremony_surviv )], expected_fault_lines: vec![], }, + FormatBodiesTestCase { + description: "an expression macro body ending in a semicolon", + dialect: "snowflake", + relative_parts: None, + contents: "MACRO (arguments (q INTEGER), returns INTEGER);\n\n@q + 1 ;\n", + expected_formatted: vec![( + 0, + "MACRO (arguments (q INTEGER), returns INTEGER);\n\n@q + 1;\n".to_string(), + )], + expected_fault_lines: vec![], + }, + FormatBodiesTestCase { + description: "an expression macro body keeps the line comment that ends it", + dialect: "snowflake", + relative_parts: None, + contents: "MACRO (arguments (q INTEGER), returns INTEGER);\n\n@q + 1 -- note\n", + expected_formatted: vec![( + 0, + "MACRO (arguments (q INTEGER), returns INTEGER);\n\n@q + 1 -- note\n".to_string(), + )], + expected_fault_lines: vec![], + }, + FormatBodiesTestCase { + description: "an unparseable Snowflake expression macro body is a fault, not rewritten", + dialect: "snowflake", + relative_parts: None, + contents: "MACRO (arguments (q VARCHAR), returns VARCHAR);\n\nTRIM(\n UPPER(@q,,\n )\n", + expected_formatted: vec![], + expected_fault_lines: vec![3], + }, FormatBodiesTestCase { description: "an unparseable body is a fault on its first line", dialect: "duckdb", @@ -627,6 +661,16 @@ fn given_a_formatted_expression_select_when_unwrapping_then_strings_keep_their_l formatted: "SELECT\n CONCAT(\n 'first\n second',\n @v\n )", expected_expression: Some("CONCAT(\n 'first\n second',\n @v\n)"), }, + UnwrappedExpressionTestCase { + description: "lines inside a dollar-quoted string are not dedented", + formatted: "SELECT\n CONCAT(\n $$first\n 'second'\n $$,\n @v\n )", + expected_expression: Some("CONCAT(\n $$first\n 'second'\n $$,\n @v\n)"), + }, + UnwrappedExpressionTestCase { + description: "a $$ inside an identifier does not open a string", + formatted: "SELECT\n COALESCE(\n col$$x,\n @v\n )", + expected_expression: Some("COALESCE(\n col$$x,\n @v\n)"), + }, UnwrappedExpressionTestCase { description: "text that is not a select is not an expression", formatted: "WITH a AS (SELECT 1)", @@ -923,3 +967,38 @@ fn given_findings_when_deciding_exit_then_only_lint_unavailable_never_fails_a_co ); } } + +#[test] +fn given_expression_macro_bodies_when_formatting_twice_then_every_dialect_lays_them_out_stably() +-> Result<(), LintTextError> { + let test_cases = [ + ExpressionMacroFormatTestCase { + description: "a body with a comment between nested calls", + contents: NORMALISE_NAME_MACRO.to_string(), + expected_formatted: NORMALISE_NAME_FORMATTED.to_string(), + }, + ExpressionMacroFormatTestCase { + description: "a body nesting fifteen calls, authored flush left", + contents: nested_macro(15, ""), + expected_formatted: nested_macro(15, " "), + }, + ]; + + for test_case in test_cases { + for dialect in ["duckdb", "snowflake"] { + let once: String = formatted_macro(&test_case.contents, dialect)?; + let twice: String = formatted_macro(&once, dialect)?; + + assert_eq!( + (once.as_str(), twice.as_str()), + ( + test_case.expected_formatted.as_str(), + test_case.expected_formatted.as_str() + ), + "{} on {dialect}", + test_case.description + ); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs index b2c56ed338..7d35354a91 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs @@ -119,3 +119,9 @@ pub(crate) struct UnwrappedExpressionTestCase { pub(crate) formatted: &'static str, pub(crate) expected_expression: Option<&'static str>, } + +pub(crate) struct ExpressionMacroFormatTestCase { + pub(crate) description: &'static str, + pub(crate) contents: String, + pub(crate) expected_formatted: String, +} diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index a847abb71f..03a8793a1b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -922,6 +922,16 @@ class SqlReadabilityTestCase: rule_code: str = "SQLCLEAN041" +@dataclass(frozen=True) +class ExpressionMacroFormatTestCase: + """A Snowflake expression macro in one location, and the file formatting leaves.""" + + description: str + macro_directory: str + consumer_folders: tuple[str, ...] + expected_macro: str + + @dataclass(frozen=True) class FormatterSyntaxTestCase: """Authored constructs that must survive canonical layout formatting.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py index f7e146fec1..195b941829 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py @@ -6,7 +6,10 @@ import pytest from sqlbuild._native import run_cli as main -from tests.integration.src.sqlbuild.cli.commands.main._test_types import FormatterSyntaxTestCase +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + ExpressionMacroFormatTestCase, + FormatterSyntaxTestCase, +) # The window syntax cases order by a non-unique column, which SQLCLEAN018 reports. _WINDOW_SYNTAX_PROJECT_TOML: str = ( @@ -14,6 +17,22 @@ ) +_NORMALISE_NAME_HEADER: str = ( + 'MACRO (\n description "Normalise a name",\n arguments (column_name VARCHAR),\n' + " returns VARCHAR,\n);\n\n" +) +_NORMALISE_NAME_AUTHORED: str = ( + "REGEXP_REPLACE(\n TRIM(UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(\n" + " @column_name,\n '[[:space:]]+',\n ' '\n ))),\n '(^| )([A-Z]) ',\n" + " '\\\\1\\\\2'\n)\n" +) +_NORMALISE_NAME_FORMATTED: str = ( + "REGEXP_REPLACE(\n TRIM(\n UPPER(\n -- Collapse repeated spaces\n" + " REGEXP_REPLACE(@column_name, '[[:space:]]+', ' ')\n )\n ),\n" + " '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n" +) + + @pytest.mark.parametrize( "test_case", [ @@ -302,5 +321,48 @@ def test_given_multibyte_text_before_restorations_when_formatting_then_output_is assert model.read_text(encoding="utf-8") == formatted +@pytest.mark.parametrize( + "test_case", + [ + ExpressionMacroFormatTestCase( + "project macro", + "macros/sql", + # A project macro needs consumers in more than one folder to stay a project macro. + ("staging", "marts"), + _NORMALISE_NAME_HEADER + _NORMALISE_NAME_FORMATTED, + ), + ExpressionMacroFormatTestCase( + "scoped macro", + "models/staging/_sqlbuild/_macros/sql", + ("staging",), + _NORMALISE_NAME_HEADER + _NORMALISE_NAME_FORMATTED, + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_expression_macro_when_formatting_then_body_is_laid_out_once( + test_case: ExpressionMacroFormatTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') + macro: Path = tmp_path / test_case.macro_directory / "normalise_name.sql" + macro.parent.mkdir(parents=True) + macro.write_text(_NORMALISE_NAME_HEADER + _NORMALISE_NAME_AUTHORED) + for folder in test_case.consumer_folders: + model: Path = tmp_path / "models" / folder / f"{folder}_customer_names.sql" + model.parent.mkdir(parents=True, exist_ok=True) + model.write_text( + 'MODEL (description "Customer names", database warehouse, schema analytics);\n' + "SELECT @sql.normalise_name('ada lovelace') AS customer_name\n" + ) + assert main(["--project-dir", str(tmp_path), "format", "--json"]) == 0 + payload: dict[str, object] = json.loads(capsys.readouterr().out) + assert payload["faults"] == 0 + assert macro.read_text() == test_case.expected_macro + assert main(["--project-dir", str(tmp_path), "format", "--check", "--json"]) == 0 + assert main(["--project-dir", str(tmp_path), "format", "--json"]) == 0 + assert macro.read_text() == test_case.expected_macro + assert main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) == 0 + + if __name__ == "__main__": pytest.main([__file__, "-vv"])