Skip to content

Commit f98235d

Browse files
authored
fix: read Databricks and DuckDB strings with their own escape rules (#1502)
SQLBuild's lexical rules gave Databricks and DuckDB one shared entry with no backslash escapes, no raw strings and flat block comments. SQLQuack and both adapters read them differently: Databricks honours a backslash in '...' and "..." outside r'...' raw strings, DuckDB honours it inside E'...', and both nest block comments. The expression-macro dedent in sqb format reads the formatted body with these rules. A multi-line string after a backslash-escaped quote looked closed early, so its next line lost two spaces on every run and the string's value changed. Databricks now has its own entry, and DuckDB uses the PostgreSQL rules its adapter already declares. The same rules decide quoted text for semantic-validation normalisation, fast lineage, dynamic pivot, SQL tree nodes and query-facts shapes.
1 parent d15ee25 commit f98235d

5 files changed

Lines changed: 91 additions & 7 deletions

File tree

‎crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ const CROSS_JOIN_FILE: &str = "select o.order_id\nfrom orders as o\n-- sqb: igno
3535
const NORMALISE_NAME_MACRO: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(\n @column_name,\n '[[:space:]]+',\n ' '\n ))),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n";
3636
const ESCAPED_QUOTE_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(\n @v,\n 'O\\'Brien\n kept indent',\n 'tail'\n)\n";
3737
const ESCAPED_QUOTE_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(@v, 'O\\'Brien\n kept indent', 'tail')\n";
38+
const E_STRING_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(\n @v,\n E'O\\'Brien\n kept indent',\n 'tail'\n)\n";
39+
const E_STRING_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(@v, E'O\\'Brien\n kept indent', 'tail')\n";
40+
const NESTED_COMMENT_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* outer /* inner */ it's\n kept */\n 'x'\n)\n";
41+
const NESTED_COMMENT_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* outer /* inner */ it's\n kept */ 'x'\n)\n";
3842
const BLOCK_COMMENT_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* first\n second */\n 'x'\n)\n";
3943
const BLOCK_COMMENT_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* first\n second */ 'x'\n)\n";
4044
const NORMALISE_NAME_FORMATTED: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(\n UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(@column_name, '[[:space:]]+', ' ')\n )\n ),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n";
@@ -1011,22 +1015,34 @@ fn given_expression_macro_bodies_when_formatting_twice_then_every_dialect_lays_t
10111015
let test_cases = [
10121016
ExpressionMacroFormatTestCase {
10131017
description: "a body with a comment between nested calls",
1014-
dialects: &["duckdb", "snowflake"],
1018+
dialects: &["duckdb", "snowflake", "databricks"],
10151019
contents: NORMALISE_NAME_MACRO.to_string(),
10161020
expected_formatted: NORMALISE_NAME_FORMATTED.to_string(),
10171021
},
10181022
ExpressionMacroFormatTestCase {
10191023
description: "a body nesting fifteen calls, authored flush left",
1020-
dialects: &["duckdb", "snowflake"],
1024+
dialects: &["duckdb", "snowflake", "databricks"],
10211025
contents: nested_macro(15, ""),
10221026
expected_formatted: nested_macro(15, " "),
10231027
},
10241028
ExpressionMacroFormatTestCase {
10251029
description: "a multi-line string after a backslash-escaped quote",
1026-
dialects: &["snowflake"],
1030+
dialects: &["snowflake", "databricks"],
10271031
contents: ESCAPED_QUOTE_MACRO.to_string(),
10281032
expected_formatted: ESCAPED_QUOTE_FORMATTED.to_string(),
10291033
},
1034+
ExpressionMacroFormatTestCase {
1035+
description: "a multi-line E-string after a backslash-escaped quote",
1036+
dialects: &["duckdb"],
1037+
contents: E_STRING_MACRO.to_string(),
1038+
expected_formatted: E_STRING_FORMATTED.to_string(),
1039+
},
1040+
ExpressionMacroFormatTestCase {
1041+
description: "a nested block comment holding an apostrophe",
1042+
dialects: &["duckdb", "databricks"],
1043+
contents: NESTED_COMMENT_MACRO.to_string(),
1044+
expected_formatted: NESTED_COMMENT_FORMATTED.to_string(),
1045+
},
10301046
ExpressionMacroFormatTestCase {
10311047
description: "a block comment with indented lines",
10321048
dialects: &["duckdb", "snowflake"],

‎crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ pub fn dialect_lexical_syntax(dialect: &str) -> LexicalSyntax {
2424
line_comment_prefixes: lines(&["--", "# ", "#!", "//"]),
2525
..LexicalSyntax::default()
2626
},
27-
"databricks" | "duckdb" => LexicalSyntax {
27+
"databricks" => LexicalSyntax {
28+
backslash_escape_quotes: strings(&["'", "\""]),
29+
raw_string_prefix: true,
30+
nested_block_comments: true,
2831
line_comment_prefixes: lines(&["--"]),
2932
..LexicalSyntax::default()
3033
},

‎crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,48 @@ fn given_dialect_lexical_syntax_when_scanning_non_code_then_dialect_decides_boun
200200
syntax: single_quote_backslashes(),
201201
expected_first_range: Some((0, 4)),
202202
},
203+
DialectFirstNonCodeRangeTestCase {
204+
description: "a Databricks string continues past a backslash-escaped quote",
205+
sql: r"'a\'b' x",
206+
syntax: dialect_lexical_syntax("databricks"),
207+
expected_first_range: Some((0, 6)),
208+
},
209+
DialectFirstNonCodeRangeTestCase {
210+
description: "Databricks backslash escapes hold in double-quoted strings",
211+
sql: r#""a\"b" x"#,
212+
syntax: dialect_lexical_syntax("databricks"),
213+
expected_first_range: Some((0, 6)),
214+
},
215+
DialectFirstNonCodeRangeTestCase {
216+
description: "a Databricks raw string ends at the quote after a backslash",
217+
sql: r"r'a\' x",
218+
syntax: dialect_lexical_syntax("databricks"),
219+
expected_first_range: Some((1, 5)),
220+
},
221+
DialectFirstNonCodeRangeTestCase {
222+
description: "Databricks block comments nest",
223+
sql: "/* a /* b */ c */ x",
224+
syntax: dialect_lexical_syntax("databricks"),
225+
expected_first_range: Some((0, 17)),
226+
},
227+
DialectFirstNonCodeRangeTestCase {
228+
description: "a DuckDB E-string continues past a backslash-escaped quote",
229+
sql: r"E'a\'b' x",
230+
syntax: dialect_lexical_syntax("duckdb"),
231+
expected_first_range: Some((1, 7)),
232+
},
233+
DialectFirstNonCodeRangeTestCase {
234+
description: "a plain DuckDB string ends at the quote after a backslash",
235+
sql: r"'a\' x",
236+
syntax: dialect_lexical_syntax("duckdb"),
237+
expected_first_range: Some((0, 4)),
238+
},
239+
DialectFirstNonCodeRangeTestCase {
240+
description: "DuckDB block comments nest",
241+
sql: "/* a /* b */ c */ x",
242+
syntax: dialect_lexical_syntax("duckdb"),
243+
expected_first_range: Some((0, 17)),
244+
},
203245
];
204246
for test_case in test_cases {
205247
let ranges: Vec<(usize, usize)> = dialect_non_code_ranges(test_case.sql, &test_case.syntax);

‎tests/integration/src/sqlbuild/cli/commands/main/_test_types.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -924,13 +924,14 @@ class SqlReadabilityTestCase:
924924

925925
@dataclass(frozen=True)
926926
class ExpressionMacroFormatTestCase:
927-
"""A Snowflake expression macro in one location, and the file formatting leaves."""
927+
"""An expression macro in one location, and the file formatting leaves on an adapter."""
928928

929929
description: str
930930
macro_directory: str
931931
consumer_folders: tuple[str, ...]
932932
authored_macro: str
933933
expected_macro: str
934+
adapter: str = "snowflake"
934935

935936

936937
@dataclass(frozen=True)

‎tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -348,13 +348,35 @@ def test_given_multibyte_text_before_restorations_when_formatting_then_output_is
348348
_NORMALISE_NAME_HEADER
349349
+ "CONCAT(@column_name, 'O\\'Brien\n kept indent', 'tail')\n",
350350
),
351+
ExpressionMacroFormatTestCase(
352+
"Databricks multi-line string after a backslash-escaped quote",
353+
"models/staging/_sqlbuild/_macros/sql",
354+
("staging",),
355+
_NORMALISE_NAME_HEADER
356+
+ "CONCAT(\n @column_name,\n 'O\\'Brien\n kept indent',\n 'tail'\n)\n",
357+
_NORMALISE_NAME_HEADER
358+
+ "CONCAT(@column_name, 'O\\'Brien\n kept indent', 'tail')\n",
359+
adapter="databricks",
360+
),
361+
ExpressionMacroFormatTestCase(
362+
"DuckDB multi-line E-string after a backslash-escaped quote",
363+
"models/staging/_sqlbuild/_macros/sql",
364+
("staging",),
365+
_NORMALISE_NAME_HEADER
366+
+ "CONCAT(\n @column_name,\n E'O\\'Brien\n kept indent',\n 'tail'\n)\n",
367+
_NORMALISE_NAME_HEADER
368+
+ "CONCAT(@column_name, E'O\\'Brien\n kept indent', 'tail')\n",
369+
adapter="duckdb",
370+
),
351371
],
352372
ids=lambda case: case.description,
353373
)
354-
def test_given_snowflake_expression_macro_when_formatting_then_body_is_laid_out_once(
374+
def test_given_expression_macro_when_formatting_then_body_is_laid_out_once(
355375
test_case: ExpressionMacroFormatTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str]
356376
) -> None:
357-
(tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n')
377+
(tmp_path / "sqlbuild_project.toml").write_text(
378+
f'name = "orders"\nadapter = "{test_case.adapter}"\n'
379+
)
358380
macro: Path = tmp_path / test_case.macro_directory / "normalise_name.sql"
359381
macro.parent.mkdir(parents=True)
360382
macro.write_text(test_case.authored_macro)

0 commit comments

Comments
 (0)