|
14 | 14 | import pytest |
15 | 15 |
|
16 | 16 | import sqlbuild._native as native_module |
| 17 | +import sqlbuild.compiler.analysis_session.classes.native_model_analysis as native_model_analysis |
17 | 18 | import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly |
| 19 | +import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly |
18 | 20 | from sqlbuild.adapter.contract.models import ExpressionInferenceProfile |
19 | 21 | from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter |
20 | 22 | from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( |
|
35 | 37 | from sqlbuild.compiler.compile.models import ( |
36 | 38 | CompactLineageFacts, |
37 | 39 | CompileAdapterContext, |
| 40 | + CompiledLineageColumnFact, |
38 | 41 | CompileProjectInputs, |
39 | 42 | DynamicColumnContractProof, |
40 | 43 | ModelSqlAnalysis, |
|
64 | 67 | description: Customers feed. |
65 | 68 | expression: >- |
66 | 69 | (SELECT 10 AS customer_id, 'Ada' AS Customer_Name, 'east' AS region) |
| 70 | + - name: raw_payments |
| 71 | + description: Typed payments feed. |
| 72 | + expression: >- |
| 73 | + (SELECT 10 AS customer_id, 'east' AS region, CAST(5 AS DECIMAL(10, 2)) AS amount) |
| 74 | + columns: |
| 75 | + - name: customer_id |
| 76 | + type: INTEGER |
| 77 | + - name: region |
| 78 | + type: VARCHAR |
| 79 | + - name: amount |
| 80 | + type: DECIMAL(10, 2) |
67 | 81 | - name: raw_events |
68 | 82 | description: Untyped events feed. |
69 | 83 | columns: |
|
113 | 127 | 'SELECT * FROM __source("raw_orders") o\nJOIN __source("raw_customers") c USING (customer_id)', |
114 | 128 | ) |
115 | 129 | _PIVOT_AGGREGATES: tuple[str, ...] = ("MAX", "MIN", "ANY_VALUE", "SUM") |
| 130 | +_TYPED_PIVOT: str = ( |
| 131 | + _PIVOT_HEADER.format(pivot="status", value="amount", aggregate="MAX") |
| 132 | + + _PIVOT_BODIES[0].format(aggregate="MAX") |
| 133 | + + "\n" |
| 134 | +) |
| 135 | +_TYPED_PIVOT_PASSTHROUGH: str = ( |
| 136 | + _PIVOT_HEADER.format(pivot="status", value="amount", aggregate="MAX") |
| 137 | + + 'SELECT * FROM __ref("status_amounts")\n' |
| 138 | +) |
| 139 | +_CONTRACT_CTE_MODELS: dict[str, str] = { |
| 140 | + "models/marts/payments_by_customer.sql": ( |
| 141 | + 'MODEL (\n description "Payments passed through a CTE",\n contract enforced,\n' |
| 142 | + " columns (customer_id (type INTEGER), region (type VARCHAR)),\n);\n\n" |
| 143 | + 'WITH base AS (SELECT customer_id, region FROM __source("raw_payments"))\n' |
| 144 | + "SELECT customer_id, region FROM base\n" |
| 145 | + ), |
| 146 | + "models/marts/payment_totals.sql": ( |
| 147 | + 'MODEL (\n description "Payment totals aggregated in a CTE",\n contract enforced,\n' |
| 148 | + " columns (total_amount (type DECIMAL(38, 2)), region (type VARCHAR)),\n);\n\n" |
| 149 | + "WITH base AS (\n SELECT SUM(amount) AS total_amount, region\n" |
| 150 | + ' FROM __source("raw_payments")\n GROUP BY region\n)\n' |
| 151 | + "SELECT total_amount, region FROM base\n" |
| 152 | + ), |
| 153 | +} |
116 | 154 |
|
117 | 155 |
|
118 | 156 | def _pivot_models(rng: random.Random) -> dict[str, str]: |
119 | 157 | """A typed pivot every passthrough reads, and seeded pivot shapes proven or refused.""" |
120 | 158 |
|
121 | 159 | files: dict[str, str] = { |
122 | | - "models/marts/status_amounts.sql": _PIVOT_HEADER.format( |
123 | | - pivot="status", value="amount", aggregate="MAX" |
124 | | - ) |
125 | | - + _PIVOT_BODIES[0].format(aggregate="MAX") |
126 | | - + "\n", |
127 | | - "models/marts/status_passthrough.sql": _PIVOT_HEADER.format( |
128 | | - pivot="status", value="amount", aggregate="MAX" |
129 | | - ) |
130 | | - + 'SELECT * FROM __ref("status_amounts")\n', |
| 160 | + "models/marts/status_amounts.sql": _TYPED_PIVOT, |
| 161 | + "models/marts/status_passthrough.sql": _TYPED_PIVOT_PASSTHROUGH, |
131 | 162 | } |
132 | 163 | for index, body in enumerate(rng.sample(_PIVOT_BODIES, k=4)): |
133 | 164 | aggregate: str = rng.choice(_PIVOT_AGGREGATES) |
@@ -256,13 +287,55 @@ def _subquery(rng: random.Random, inputs: list[_Relation]) -> tuple[str, str, tu |
256 | 287 | ) |
257 | 288 |
|
258 | 289 |
|
| 290 | +def pivot_project_files() -> dict[str, str]: |
| 291 | + """A typed pivot with its passthrough and one plain model, for selective assembly.""" |
| 292 | + |
| 293 | + return { |
| 294 | + "sqlbuild_project.toml": _PROJECT_TOML, |
| 295 | + "sources/raw.yml": _SOURCES, |
| 296 | + "models/marts/status_amounts.sql": _TYPED_PIVOT, |
| 297 | + "models/marts/status_passthrough.sql": _TYPED_PIVOT_PASSTHROUGH, |
| 298 | + "models/staging/orders_list.sql": ( |
| 299 | + 'MODEL (description "Order ids");\n\nSELECT order_id FROM __source("raw_orders")\n' |
| 300 | + ), |
| 301 | + } |
| 302 | + |
| 303 | + |
| 304 | +def native_pivot_proofs( |
| 305 | + *, monkeypatch: pytest.MonkeyPatch |
| 306 | +) -> list[DynamicColumnContractProof | None]: |
| 307 | + """Record every standalone native pivot proof assembly takes, failing any wheel pivot proof.""" |
| 308 | + |
| 309 | + proofs: list[DynamicColumnContractProof | None] = [] |
| 310 | + native_proof: Callable[..., DynamicColumnContractProof | None] = ( |
| 311 | + native_stage_assembly.prove_native_dynamic_contract |
| 312 | + ) |
| 313 | + |
| 314 | + def recorded(**keywords: Any) -> DynamicColumnContractProof | None: |
| 315 | + proofs.append(native_proof(**keywords)) |
| 316 | + return proofs[-1] |
| 317 | + |
| 318 | + python_proof: Callable[..., DynamicColumnContractProof | None] = ( |
| 319 | + project_assembly.analyze_dynamic_column_contract |
| 320 | + ) |
| 321 | + |
| 322 | + def wheel_not_expected(**keywords: Any) -> DynamicColumnContractProof | None: |
| 323 | + assert not keywords["families"], "the Python wheel proved a pivot native should prove" |
| 324 | + return python_proof(**keywords) |
| 325 | + |
| 326 | + monkeypatch.setattr(native_stage_assembly, "prove_native_dynamic_contract", recorded) |
| 327 | + monkeypatch.setattr(project_assembly, "analyze_dynamic_column_contract", wheel_not_expected) |
| 328 | + return proofs |
| 329 | + |
| 330 | + |
259 | 331 | def generated_analysis_files(*, rng: random.Random, model_count: int) -> dict[str, str]: |
260 | 332 | """Sources (typed, inferred and untyped), a pivot and models drawing on every template.""" |
261 | 333 |
|
262 | 334 | files: dict[str, str] = { |
263 | 335 | "sqlbuild_project.toml": _PROJECT_TOML, |
264 | 336 | "sources/raw.yml": _SOURCES, |
265 | 337 | **_pivot_models(rng), |
| 338 | + **_CONTRACT_CTE_MODELS, |
266 | 339 | } |
267 | 340 | relations: list[_Relation] = list(_INITIAL_RELATIONS) |
268 | 341 | for index in range(model_count): |
@@ -302,6 +375,7 @@ class AnalysisParity: |
302 | 375 | pivot_proofs: int = 0 |
303 | 376 | standalone_proofs: int = 0 |
304 | 377 | proven_pivots: int = 0 |
| 378 | + native_enrichments: int = 0 |
305 | 379 |
|
306 | 380 |
|
307 | 381 | def compare_analyses( |
@@ -380,7 +454,16 @@ def shapes_by_both( |
380 | 454 | ) |
381 | 455 | return python |
382 | 456 |
|
| 457 | + native_lineage_facts: Callable[..., tuple[CompiledLineageColumnFact, ...]] = ( |
| 458 | + native_model_analysis.lineage_facts |
| 459 | + ) |
| 460 | + |
| 461 | + def counted_native_enrichment(rows: list[Any]) -> tuple[CompiledLineageColumnFact, ...]: |
| 462 | + parity.native_enrichments += 1 |
| 463 | + return native_lineage_facts(rows) |
| 464 | + |
383 | 465 | with monkeypatch.context() as patch: |
| 466 | + patch.setattr(native_model_analysis, "lineage_facts", counted_native_enrichment) |
384 | 467 | patch.setattr(project_assembly, "analyze_model_sql_by_engine", models_by_both) |
385 | 468 | patch.setattr(project_assembly, "expression_source_shapes_by_engine", shapes_by_both) |
386 | 469 | patch.setattr(project_assembly, "dynamic_column_contract_by_engine", proofs_by_both) |
|
0 commit comments