Skip to content

Commit 911882f

Browse files
authored
feat(sql-macros)!: relation slots with COLUMNS groups, replacing COLUMN OF (#1254)
SQL macros take relation slots: single-column slots and COLUMNS groups declared in the RELATION type and mapped at the call. A plain group read expands to bare column names; aliased reads prefix every column. A capture check rejects plain group reads that would bind to another relation in a correlated subquery. BREAKING CHANGE: COLUMN OF SQL macro arguments are removed; use relation slots.
1 parent 3cb8197 commit 911882f

107 files changed

Lines changed: 3554 additions & 279 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
generator 1 files 10874 sha256 4d276ad2428841efa2a1b851c83863bea70505d9e085a71390401399b5667667
1+
generator 2 files 10874 sha256 7dd651a573bce90f9816889cdeece75b9f837137813a033125daafa28b7b70eb

‎benchmarks/duckshop-10k/duckshop10k/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@
44
regenerate checksum.txt, whenever a change alters any generated file.
55
"""
66

7-
GENERATOR_VERSION = "1"
7+
GENERATOR_VERSION = "2"
88
SEED = 20261024
99
PROJECT_NAME = "duckshop_10k"

‎benchmarks/duckshop-10k/duckshop10k/declarations.py‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,13 @@
88
SQL_MACROS: dict[str, str] = {
99
"cdc_latest": """MACRO (
1010
description "Latest surviving version of each key in an append-only change log",
11-
arguments (
12-
src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ),
13-
key COLUMN OF src,
14-
),
11+
arguments (src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ, key COLUMNS)),
1512
);
1613
1714
WITH cdc_versions AS (
1815
SELECT *
1916
FROM @src
20-
QUALIFY ROW_NUMBER() OVER (PARTITION BY @key ORDER BY _seq DESC) = 1
17+
QUALIFY ROW_NUMBER() OVER (PARTITION BY @src.key ORDER BY _seq DESC) = 1
2118
),
2219
2320
cdc_latest AS (
@@ -28,16 +25,13 @@
2825
""",
2926
"cdc_history": """MACRO (
3027
description "Every version of each key in a change log, keeping the last one of each load",
31-
arguments (
32-
src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ),
33-
key COLUMN OF src,
34-
),
28+
arguments (src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ, key COLUMNS)),
3529
);
3630
3731
WITH cdc_history AS (
3832
SELECT *
3933
FROM @src
40-
QUALIFY ROW_NUMBER() OVER (PARTITION BY @key, _loaded_at ORDER BY _seq DESC) = 1
34+
QUALIFY ROW_NUMBER() OVER (PARTITION BY @src.key, _loaded_at ORDER BY _seq DESC) = 1
4135
)
4236
""",
4337
"load_metadata": """MACRO (

‎benchmarks/duckshop-10k/duckshop10k/rules_gen.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,10 @@ def _is_change_log(ctx: RuleContext, source: str) -> bool:
3232
code="XSQBRDUCK001",
3333
message="Staging models over a change-log source must deduplicate it with a change-log macro",
3434
remediation=(
35-
"Read the source in an import CTE and follow it with @sql.cdc_latest(<cte>, <key>), or "
36-
"@sql.cdc_history(<cte>, <key>) for a model that keeps every version, so deletes and "
37-
"superseded versions are handled the same way everywhere."
35+
"Read the source in an import CTE and follow it with "
36+
"@sql.cdc_latest(<cte>, src.key := (<key>)), or "
37+
"@sql.cdc_history(<cte>, src.key := (<key>)) for a model that keeps every version, so "
38+
"deletes and superseded versions are handled the same way everywhere."
3839
),
3940
)
4041
def staging_uses_change_log_macro(*, model: Model, ctx: RuleContext) -> list[Finding]:

‎benchmarks/duckshop-10k/duckshop10k/staging.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ def base_model(project: Project, instance: Instance, archetype: Archetype, folde
202202
)
203203
relation = "deduplicated"
204204
else:
205-
ctes.append(f"@sql.cdc_latest(src, {key})")
205+
ctes.append(f"@sql.cdc_latest(src, src.key := ({key}))")
206206
relation = "cdc_latest"
207207
items: list[str] = []
208208
columns: list[Column] = []
@@ -480,7 +480,7 @@ def history_model(project: Project, instance: Instance, archetype: Archetype, fo
480480
)
481481
ctes = [
482482
import_cte("src", f'__source("{source}")'),
483-
f"@sql.cdc_history(src, {key_physical})",
483+
f"@sql.cdc_history(src, src.key := ({key_physical}))",
484484
cte("versions", f"SELECT\n{select_list(versions)}\nFROM cdc_history AS s"),
485485
cte("final", f"SELECT\n{select_list(items)}\nFROM versions AS h"),
486486
]

‎benchmarks/duckshop/project/models/staging/_sqlbuild/macros/sql/cdc_latest.sql‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,12 @@
11
MACRO (
22
description "Latest surviving version of each key in an append-only change log",
3-
arguments (
4-
src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ),
5-
key COLUMN OF src,
6-
),
3+
arguments (src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ, key COLUMNS)),
74
);
85

96
WITH cdc_versions AS (
107
SELECT *
118
FROM @src
12-
QUALIFY ROW_NUMBER() OVER (PARTITION BY @key ORDER BY _seq DESC) = 1
9+
QUALIFY ROW_NUMBER() OVER (PARTITION BY @src.key ORDER BY _seq DESC) = 1
1310
),
1411

1512
cdc_latest AS (

‎benchmarks/duckshop/project/models/staging/catalogue/stg_brands.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ WITH src AS (
1616
FROM __source("catalogue__brands")
1717
),
1818

19-
@sql.cdc_latest(src, brand_code)
19+
@sql.cdc_latest(src, src.key := (brand_code))
2020

2121
final AS (
2222
SELECT

‎benchmarks/duckshop/project/models/staging/catalogue/stg_bundle_components.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ WITH src AS (
1515
FROM __source("catalogue__bundle_components")
1616
),
1717

18-
@sql.cdc_latest(src, component_id)
18+
@sql.cdc_latest(src, src.key := (component_id))
1919

2020
final AS (
2121
SELECT

‎benchmarks/duckshop/project/models/staging/catalogue/stg_bundles.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ WITH src AS (
2020
FROM __source("catalogue__bundles")
2121
),
2222

23-
@sql.cdc_latest(src, bundle_sku)
23+
@sql.cdc_latest(src, src.key := (bundle_sku))
2424

2525
final AS (
2626
SELECT

‎benchmarks/duckshop/project/models/staging/catalogue/stg_categories.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ WITH src AS (
1818
FROM __source("catalogue__categories")
1919
),
2020

21-
@sql.cdc_latest(src, category_id)
21+
@sql.cdc_latest(src, src.key := (category_id))
2222

2323
final AS (
2424
SELECT

0 commit comments

Comments
 (0)