Skip to content

Commit 19087ad

Browse files
committed
refactor(native): read compiled models through one counted native accessor
`model_reads::native_model` returns the native view behind a model object and counts every other object, a Python-built record that is read by attribute instead, in the `compiled_view_fallbacks` stage counter. Every model read helper goes through it. An integration test shows compile and plan of a native project never fall back, and a unit test shows Python-built models are still read, and counted.
1 parent 1ecc1d0 commit 19087ad

7 files changed

Lines changed: 192 additions & 30 deletions

File tree

‎crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ pub(crate) const NATIVE_HEADER_COLUMNS: &str = "native_header_columns";
3434
pub(crate) const PARSED_HEADER_COLUMNS: &str = "parsed_header_columns";
3535
/// Config values converted into typed nodes for a key a reader asked for.
3636
pub(crate) const TYPED_CONFIG_READS: &str = "typed_config_reads";
37+
/// Reads of a model or project that is not a native compiled view, read by attribute instead.
38+
pub(crate) const COMPILED_VIEW_FALLBACKS: &str = "compiled_view_fallbacks";
3739

3840
/// Compile stage runs since the last reset, by stage.
3941
#[pyclass]

‎crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs‎

Lines changed: 42 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
33
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
44

5+
use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage};
56
use crate::bindings::_helpers::compiled_views::compiled_model_view::{
67
DeclaredRow, LineageInput, NativeCompiledModel,
78
};
@@ -27,10 +28,22 @@ const TRANSFORM_VALUES: [&str; 6] = [
2728
];
2829
const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"];
2930

31+
/// The native view behind a model object; any other object is a Python-built record, counted.
32+
#[inline]
33+
pub(crate) fn native_model<'a>(
34+
model: &'a Bound<'_, PyAny>,
35+
) -> PyResult<Option<&'a NativeCompiledModel>> {
36+
if let Ok(view) = model.cast::<NativeCompiledModel>() {
37+
return Ok(Some(view.get()));
38+
}
39+
count_stage(model.py(), COMPILED_VIEW_FALLBACKS, 1)?;
40+
Ok(None)
41+
}
42+
3043
/// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred.
3144
pub(crate) fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<InferredRow>>> {
32-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
33-
return view.get().inferred_rows(model.py());
45+
if let Some(view) = native_model(model)? {
46+
return view.inferred_rows(model.py());
3447
}
3548
let rows: Bound<'_, PyAny> = native_field(model, "inferred_rows")?;
3649
if rows.is_none() {
@@ -56,24 +69,23 @@ fn column_names(rows: Vec<InferredRow>) -> Vec<String> {
5669

5770
/// Each declared column of the attached schema as `(name, type)`, `None` without a schema.
5871
pub(crate) fn declared_rows(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<DeclaredRow>>> {
59-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
60-
return view.get().schema_column_rows(model.py());
72+
if let Some(view) = native_model(model)? {
73+
return view.schema_column_rows(model.py());
6174
}
6275
native_field(model, "schema_column_rows")?.extract()
6376
}
6477

6578
/// Whether the attached schema declares dynamic column families.
6679
pub(crate) fn dynamic_columns_declared(model: &Bound<'_, PyAny>) -> PyResult<bool> {
67-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
68-
return view.get().dynamic_columns_declared(model.py());
80+
if let Some(view) = native_model(model)? {
81+
return view.dynamic_columns_declared(model.py());
6982
}
7083
native_field(model, "has_dynamic_columns")?.is_truthy()
7184
}
7285

7386
/// `(has star, star resolved)` of the model's analysed query.
7487
pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> {
75-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
76-
let view: &NativeCompiledModel = view.get();
88+
if let Some(view) = native_model(model)? {
7789
return Ok((
7890
view.fast_lineage_has_star(model.py())?,
7991
view.fast_lineage_star_resolved(model.py())?,
@@ -87,8 +99,8 @@ pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> {
8799

88100
/// Output columns whose types semantic recovery forgot, in any order.
89101
pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult<Vec<String>> {
90-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
91-
return Ok(view.get().unchecked_columns().to_vec());
102+
if let Some(view) = native_model(model)? {
103+
return Ok(view.unchecked_columns().to_vec());
92104
}
93105
native_field(model, "unchecked_output_columns")?
94106
.try_iter()?
@@ -98,8 +110,8 @@ pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult<Vec<String
98110

99111
/// Each lineage output with the `(resource, column)` pairs it reads; empty without lineage.
100112
pub(crate) fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult<Vec<LineageInput>> {
101-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
102-
return Ok(view.get().lineage_inputs(model.py())?.unwrap_or_default());
113+
if let Some(view) = native_model(model)? {
114+
return Ok(view.lineage_inputs(model.py())?.unwrap_or_default());
103115
}
104116
let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?;
105117
if !columns.is_truthy()? {
@@ -123,25 +135,25 @@ pub(crate) fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult<Vec<LineageIn
123135

124136
/// Whether the native session validated the model's bindings.
125137
pub(crate) fn binding_validated(model: &Bound<'_, PyAny>) -> PyResult<bool> {
126-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
127-
return view.get().binding_validated(model.py());
138+
if let Some(view) = native_model(model)? {
139+
return view.binding_validated(model.py());
128140
}
129141
native_field(model, "binding_validated")?.is_truthy()
130142
}
131143

132144
/// Whether the model's dynamic pivot proof proved its output columns.
133145
pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult<bool> {
134-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
135-
return view.get().output_proven(model.py());
146+
if let Some(view) = native_model(model)? {
147+
return view.output_proven(model.py());
136148
}
137149
let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?;
138150
Ok(!proof.is_none() && native_field(&proof, "output_proven")?.is_truthy()?)
139151
}
140152

141153
/// Each lineage output with its transform kind and confidence values, `None` without lineage.
142154
pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<LineageFactRow>>> {
143-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
144-
let Some(rows) = view.get().lineage_rows(model.py())? else {
155+
if let Some(view) = native_model(model)? {
156+
let Some(rows) = view.lineage_rows(model.py())? else {
145157
return Ok(None);
146158
};
147159
let mut facts: Vec<LineageFactRow> = Vec::with_capacity(rows.len());
@@ -190,8 +202,8 @@ fn coded(values: &[&str], code: u8) -> PyResult<String> {
190202

191203
/// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`.
192204
pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<String>>> {
193-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
194-
return view.get().proven_inputs(model.py());
205+
if let Some(view) = native_model(model)? {
206+
return view.proven_inputs(model.py());
195207
}
196208
let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?;
197209
if proof.is_none() || !native_field(&proof, "output_proven")?.is_truthy()? {
@@ -202,40 +214,40 @@ pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<Str
202214

203215
/// The model's name.
204216
pub(crate) fn model_name(model: &Bound<'_, PyAny>) -> PyResult<String> {
205-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
206-
return view.get().name(model.py());
217+
if let Some(view) = native_model(model)? {
218+
return view.name(model.py());
207219
}
208220
native_field(model, "name")?.extract()
209221
}
210222

211223
/// The model's compiled query SQL.
212224
pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
213-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
214-
return Ok(view.get().query_sql(model.py())?.into_any());
225+
if let Some(view) = native_model(model)? {
226+
return Ok(view.query_sql(model.py())?.into_any());
215227
}
216228
native_field(model, "query_sql")
217229
}
218230

219231
/// The model's `CompiledObjectKey`.
220232
pub(crate) fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
221-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
222-
return Ok(view.get().key(model.py())?.into_bound(model.py()));
233+
if let Some(view) = native_model(model)? {
234+
return Ok(view.key(model.py())?.into_bound(model.py()));
223235
}
224236
native_field(model, "key")
225237
}
226238

227239
/// The model's destination `CompiledRelationLocation`.
228240
pub(crate) fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
229-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
230-
return Ok(view.get().destination(model.py())?.into_bound(model.py()));
241+
if let Some(view) = native_model(model)? {
242+
return Ok(view.destination(model.py())?.into_bound(model.py()));
231243
}
232244
native_field(model, "destination")
233245
}
234246

235247
/// The model's name as a Python string.
236248
pub(crate) fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
237-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
238-
let name: String = view.get().name(model.py())?;
249+
if let Some(view) = native_model(model)? {
250+
let name: String = view.name(model.py())?;
239251
return Ok(pyo3::types::PyString::new(model.py(), &name).into_any());
240252
}
241253
native_field(model, "name")

‎tests/integration/src/sqlbuild/cli/commands/main/_test_types.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1276,3 +1276,13 @@ class SetOperationStarRuleTestCase:
12761276
model_sql: str
12771277
expected_codes: tuple[str, ...]
12781278
expected_distinct_lines: int
1279+
1280+
1281+
@dataclass(frozen=True)
1282+
class CompiledViewFallbacksTestCase:
1283+
"""One command run over a natively compiled project."""
1284+
1285+
description: str
1286+
command: tuple[str, ...]
1287+
expected_exit_code: int
1288+
expected_fallbacks: int

‎tests/integration/src/sqlbuild/cli/commands/main/helpers.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1011,6 +1011,41 @@ def typed_config_reads(*, project_dir: Path, extra_header: str) -> int:
10111011
return native_module.compile_stage_work_info(reset=True).get("typed_config_reads", 0)
10121012

10131013

1014+
def compiled_view_fallbacks(*, project_dir: Path, command: tuple[str, ...]) -> tuple[int, int]:
1015+
"""The exit code of `command` over a native project and its reads that missed a native view."""
1016+
1017+
project_dir.mkdir(parents=True)
1018+
(project_dir / "sqlbuild_project.toml").write_text(
1019+
'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n'
1020+
'[connections.local]\ndatabase = ":memory:"\n\n[targets.dev]\nschema = "main"\n',
1021+
encoding="utf-8",
1022+
)
1023+
models_dir: Path = project_dir / "models"
1024+
models_dir.mkdir()
1025+
(models_dir / "orders.sql").write_text(
1026+
'MODEL (description "Orders.", materialized table, contract enforced, '
1027+
'columns (order_id (type "INTEGER"), loaded_at (type "INTEGER")));\n\n'
1028+
"SELECT CAST(1 AS INTEGER) AS order_id, CAST(1 AS INTEGER) AS loaded_at\n",
1029+
encoding="utf-8",
1030+
)
1031+
(models_dir / "daily_orders.sql").write_text(
1032+
'MODEL (description "Orders per day.", materialized table);\n\n'
1033+
'SELECT order_id, loaded_at FROM __ref("orders")\n',
1034+
encoding="utf-8",
1035+
)
1036+
(models_dir / "order_view.sql").write_text(
1037+
'MODEL (description "Order ids.", materialized view);\n\n'
1038+
'SELECT order_id FROM __ref("daily_orders")\n',
1039+
encoding="utf-8",
1040+
)
1041+
_ = native_module.compile_stage_work_info(reset=True)
1042+
exit_code: int = main(["--project-dir", str(project_dir), *command])
1043+
fallbacks: int = native_module.compile_stage_work_info(reset=True).get(
1044+
"compiled_view_fallbacks", 0
1045+
)
1046+
return exit_code, fallbacks
1047+
1048+
10141049
FORMAT_CACHE_UNFORMATTED_MODELS: dict[str, str] = {
10151050
"orders.sql": (
10161051
'MODEL (description "Orders placed by customers.", materialized table);\n'
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
"""Compile and plan read a native project's models through its native views."""
2+
3+
from __future__ import annotations
4+
5+
from pathlib import Path
6+
7+
import pytest
8+
9+
from tests.integration.src.sqlbuild.cli.commands.main._test_types import (
10+
CompiledViewFallbacksTestCase,
11+
)
12+
from tests.integration.src.sqlbuild.cli.commands.main.helpers import compiled_view_fallbacks
13+
14+
15+
@pytest.mark.parametrize(
16+
"test_case",
17+
[
18+
CompiledViewFallbacksTestCase(
19+
description="compiling reads every model natively",
20+
command=("compile", "--json", "--no-cache"),
21+
expected_exit_code=0,
22+
expected_fallbacks=0,
23+
),
24+
CompiledViewFallbacksTestCase(
25+
description="planning reads every model natively",
26+
command=("plan", "--json", "--no-cache"),
27+
expected_exit_code=0,
28+
expected_fallbacks=0,
29+
),
30+
],
31+
ids=lambda case: case.description,
32+
)
33+
def test_given_native_project_when_running_then_no_model_read_falls_back_to_python(
34+
test_case: CompiledViewFallbacksTestCase,
35+
tmp_path: Path,
36+
capsys: pytest.CaptureFixture[str],
37+
) -> None:
38+
exit_code, fallbacks = compiled_view_fallbacks(
39+
project_dir=tmp_path / "project", command=test_case.command
40+
)
41+
_ = capsys.readouterr()
42+
43+
assert (exit_code, fallbacks) == (test_case.expected_exit_code, test_case.expected_fallbacks)

‎tests/unit/src/sqlbuild/compiler/lineage/_test_types.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,10 @@ class RichLineageAnalysisScalingTestCase:
4343
class ExpectedCountTestCase:
4444
description: str
4545
expected_count: int
46+
47+
48+
@dataclass(frozen=True)
49+
class PythonRecordFallbackTestCase:
50+
description: str
51+
expected_upstream_resource: str
52+
expected_fallbacks_counted: bool
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
"""Python-built compiled models are read by attribute, and each such read is counted."""
2+
3+
from __future__ import annotations
4+
5+
import pytest
6+
7+
import sqlbuild._native as native
8+
from sqlbuild._native import ProjectColumnLineage
9+
from sqlbuild.compiler.compile.types import CompiledModelView
10+
from tests.unit.src.sqlbuild.compiler.lineage._test_types import PythonRecordFallbackTestCase
11+
from tests.unit.src.sqlbuild.compiler.lineage.helpers import (
12+
make_compiled_model,
13+
make_compiled_project,
14+
project_column_lineage,
15+
)
16+
17+
18+
@pytest.mark.parametrize(
19+
"test_case",
20+
[
21+
PythonRecordFallbackTestCase(
22+
description="lineage over Python-built models reads them by attribute",
23+
expected_upstream_resource="orders",
24+
expected_fallbacks_counted=True,
25+
),
26+
],
27+
ids=lambda case: case.description,
28+
)
29+
def test_given_python_built_models_when_building_lineage_then_reads_fall_back_and_are_counted(
30+
test_case: PythonRecordFallbackTestCase,
31+
) -> None:
32+
models: tuple[CompiledModelView, ...] = (
33+
make_compiled_model(
34+
name="orders", query_sql="SELECT 1 AS order_id", inferred_columns=("order_id",)
35+
),
36+
make_compiled_model(
37+
name="orders_out",
38+
query_sql='SELECT order_id FROM __ref("orders")',
39+
inferred_columns=("order_id",),
40+
),
41+
)
42+
_ = native.compile_stage_work_info(reset=True)
43+
lineage: ProjectColumnLineage | None = project_column_lineage(
44+
project=make_compiled_project(models=models)
45+
)
46+
fallbacks: int = native.compile_stage_work_info(reset=True).get("compiled_view_fallbacks", 0)
47+
48+
assert lineage is not None
49+
upstream: str = lineage.models["orders_out"].columns[0].upstream_columns[0].resource_name
50+
assert (upstream, fallbacks > 0) == (
51+
test_case.expected_upstream_resource,
52+
test_case.expected_fallbacks_counted,
53+
)

0 commit comments

Comments
 (0)