Skip to content

Commit 7945e66

Browse files
committed
fix: keep migration planning on native scope data
Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66
1 parent e643366 commit 7945e66

8 files changed

Lines changed: 123 additions & 43 deletions

File tree

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
MODEL (
2+
description "Ledger report whose branches share an upstream model during migration planning",
3+
tags [edge_case],
4+
columns (
5+
entry_id (type VARCHAR, nullable false, audits [not_null, unique]),
6+
channel (type VARCHAR),
7+
unit_amount (type NUMBER(38, 10)),
8+
),
9+
);
10+
11+
WITH ledger AS (
12+
SELECT entry_id
13+
FROM __ref("stg_edge_ledger_entries")
14+
),
15+
16+
amounts AS (
17+
SELECT
18+
entry_id,
19+
unit_amount
20+
FROM __ref("int_edge_line_amounts")
21+
),
22+
23+
channels AS (
24+
SELECT
25+
entry_id,
26+
channel
27+
FROM __ref("int_edge_orders_all_channels")
28+
),
29+
30+
final AS (
31+
SELECT
32+
l.entry_id,
33+
c.channel,
34+
a.unit_amount
35+
FROM ledger AS l
36+
INNER JOIN amounts AS a
37+
ON l.entry_id = a.entry_id
38+
INNER JOIN channels AS c
39+
ON l.entry_id = c.entry_id
40+
)
41+
42+
SELECT
43+
entry_id,
44+
channel,
45+
unit_amount
46+
FROM final
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
TEST (name "rpt_edge_planner_fan_in__keeps_only_entries_in_every_branch");
2+
3+
WITH __ref__stg_edge_ledger_entries AS (
4+
SELECT v.entry_id
5+
FROM VALUES ('LE-01'), ('LE-02'), ('LE-03') AS v(entry_id)
6+
),
7+
8+
__ref__int_edge_line_amounts AS (
9+
SELECT
10+
v.entry_id,
11+
v.unit_amount::DECIMAL(38, 10) AS unit_amount
12+
FROM VALUES ('LE-01', 2.50), ('LE-02', 7.25), ('LE-04', 9.50) AS v(entry_id, unit_amount)
13+
),
14+
15+
__ref__int_edge_orders_all_channels AS (
16+
SELECT
17+
v.entry_id,
18+
v.channel
19+
FROM VALUES ('LE-01', 'web'), ('LE-03', 'app'), ('LE-04', 'partner') AS v(entry_id, channel)
20+
),
21+
22+
__expected__rpt_edge_planner_fan_in AS (
23+
SELECT
24+
'LE-01' AS entry_id,
25+
'web' AS channel,
26+
2.50::DECIMAL(38, 10) AS unit_amount
27+
)
28+
29+
SELECT 1

‎crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,27 @@ use crate::bindings::_helpers::planner_migrations::models::{
88
ColumnCandidate, ColumnMigrationInputs, MigrationPhase,
99
};
1010
use crate::bindings::_helpers::planner_migrations::projections::parse_query_shape;
11+
use crate::bindings::_helpers::planner_scope::retained::scope_data;
1112
use crate::bindings::_helpers::records::fields::field as native_field;
1213
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
1314
use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple};
15+
use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE;
1416

1517
pub(super) fn candidates<'py>(
1618
(phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>),
1719
dialect: &Bound<'py, PyAny>,
1820
) -> PyResult<Vec<ColumnCandidate<'py>>> {
19-
let scope = phase.scope.clone();
2021
let snapshot = phase.snapshot.clone();
21-
let py = scope.py();
22+
let py = phase.scope.py();
23+
let scope = scope_data(&phase.scope)?;
2224
let mut result: Vec<ColumnCandidate<'py>> = Vec::new();
23-
for key in native_field(&scope, "execution_order")?.try_iter()? {
24-
let key = key?;
25-
if !native_field(&scope, "selected_keys")?.contains(&key)?
26-
|| !native_field(&key, "resource_type")?.eq("model")?
27-
{
25+
for key in &scope.order {
26+
if !scope.selected.contains(key) || key.0 != MODEL_RESOURCE {
2827
continue;
2928
}
30-
let model = native_field(&scope, "models_by_name")?
31-
.call_method1("get", (native_field(&key, "name")?,))?;
29+
let Some(model) = scope.model(py, &key.1) else {
30+
continue;
31+
};
3232
if model.is_none() {
3333
continue;
3434
}

‎crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,19 @@ use sqlbuild_host::entries::models::UserCodeHost;
44

55
use crate::bindings::_helpers::compiled_views::model_reads;
66
use crate::bindings::_helpers::compiled_views::project_reads;
7-
use crate::bindings::_helpers::graph::node_set::PyNodeSet;
7+
use crate::bindings::_helpers::graph::keys::native_key;
88
use crate::bindings::_helpers::native_host::host;
99
use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes;
1010
use crate::bindings::_helpers::planner_identity::metadata::build_model_version_identity_metadata_json;
1111
use crate::bindings::_helpers::planner_phases::objects::object;
1212
use crate::bindings::_helpers::planner_phases::runtime::runtime_field;
13+
use crate::bindings::_helpers::planner_scope::retained::scope_data;
1314
use crate::bindings::_helpers::records::fields::field as native_field;
1415
use crate::bindings::_helpers::records::fields::mapping_items;
1516
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
16-
use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods};
17+
use pyo3::types::{PyDict, PyDictMethods};
18+
use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys;
1719
use sqlbuild_analysis::planner_migrations::main::discovery_order::discovery_order;
18-
use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach;
1920
use std::collections::BTreeMap;
2021

2122
pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult<Vec<Bound<'py, PyAny>>> {
@@ -48,22 +49,13 @@ pub(super) fn triggered(
4849
snapshot: &Bound<'_, PyAny>,
4950
declared: &Bound<'_, PyDict>,
5051
) -> PyResult<bool> {
51-
let py = scope.py();
52-
let selected: Vec<Bound<'_, PyAny>> = native_field(scope, "selected_keys")?
53-
.try_iter()?
54-
.collect::<PyResult<_>>()?;
55-
let mut relevant = PyNodeSet(PySet::new(py, &selected)?);
56-
breadth_first_reach(selected, None, &mut relevant, |key| {
57-
native_field(scope, "upstream_deps")?
58-
.call_method1("get", (key, pyo3::types::PyTuple::empty(py)))?
59-
.try_iter()?
60-
.collect::<PyResult<Vec<Bound<'_, PyAny>>>>()
61-
})?;
62-
let relevant = relevant.0;
52+
let scope = scope_data(scope)?;
53+
let selected: Vec<_> = scope.selected.iter().cloned().collect();
54+
let relevant = reachable_keys(&selected, &scope.upstream);
6355
let fingerprints = native_field(&native_field(snapshot, "fingerprints")?, "models")?;
6456
for item in models {
6557
let name = model_reads::name_object(item)?;
66-
if relevant.contains(model_reads::model_key(item)?)?
58+
if relevant.contains(&native_key(&model_reads::model_key(item)?)?)
6759
&& !fingerprints.contains(&name)?
6860
&& !declared.contains(&name)?
6961
{

‎crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
33
use crate::bindings::_helpers::boundary::python_calls::call_with;
44
use crate::bindings::_helpers::compiled_views::model_reads;
5+
use crate::bindings::_helpers::graph::keys::native_key;
56
use crate::bindings::_helpers::planner_migrations::column_objects::config;
67
use crate::bindings::_helpers::planner_migrations::column_objects::native;
78
use crate::bindings::_helpers::planner_migrations::discovery_events::{
@@ -10,6 +11,7 @@ use crate::bindings::_helpers::planner_migrations::discovery_events::{
1011
use crate::bindings::_helpers::planner_migrations::models::{
1112
ModelMigrationDeclaration, ModelMigrationRequest,
1213
};
14+
use crate::bindings::_helpers::planner_scope::retained::scope_data;
1315
use crate::bindings::_helpers::records::construct::enum_member;
1416
use crate::bindings::_helpers::records::fields::field as native_field;
1517
use crate::bindings::_helpers::records::fields::mapping_get;
@@ -113,9 +115,13 @@ pub(super) fn automatic<'py>(
113115
matches: (&Bound<'py, PyAny>, &Bound<'py, PyDict>),
114116
) -> PyResult<Vec<ModelMigrationRequest<'py>>> {
115117
let py = runtime.py();
118+
let scope = scope_data(scope)?;
116119
let mut requests: Vec<ModelMigrationRequest<'py>> = Vec::new();
117120
for item in models {
118-
if !native_field(scope, "selected_keys")?.contains(model_reads::model_key(item)?)? {
121+
if !scope
122+
.selected
123+
.contains(&native_key(&model_reads::model_key(item)?)?)
124+
{
119125
continue;
120126
}
121127
let name = model_reads::name_object(item)?;
@@ -161,9 +167,13 @@ pub(super) fn warnings<'py>(
161167
ambiguous: &Bound<'py, PyAny>,
162168
) -> PyResult<Bound<'py, PyTuple>> {
163169
let py = scope.py();
170+
let scope = scope_data(scope)?;
164171
let selected = PySet::empty(py)?;
165172
for item in models {
166-
if native_field(scope, "selected_keys")?.contains(model_reads::model_key(item)?)? {
173+
if scope
174+
.selected
175+
.contains(&native_key(&model_reads::model_key(item)?)?)
176+
{
167177
selected.add(model_reads::name_object(item)?)?;
168178
}
169179
}

‎crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@ use crate::bindings::_helpers::planner_migrations::planning_origin::{
1919
};
2020
use crate::bindings::_helpers::planner_phases::objects::object;
2121
use crate::bindings::_helpers::planner_phases::runtime::runtime_field;
22+
use crate::bindings::_helpers::planner_scope::retained::scope_data;
2223
use crate::bindings::_helpers::records::construct::{enum_member, new_record};
2324
use crate::bindings::_helpers::records::fields::field as native_field;
2425
use crate::bindings::_helpers::records::fields::mapping_get_or;
2526
use crate::bindings::_helpers::records::ids::RecordId;
2627
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
2728
use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple};
29+
use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE;
2830
use sqlbuild_model_config::config_values::models::ConfigNode;
2931

3032
const MIGRATE_FROM_KEY: &str = "migrate_from";
@@ -55,16 +57,16 @@ fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult<Option<ModelMigrationRequ
5557
fn manual_migration_requests<'py>(
5658
scope: &Bound<'py, PyAny>,
5759
) -> PyResult<Vec<ModelMigrationRequest<'py>>> {
60+
let py = scope.py();
61+
let scope = scope_data(scope)?;
5862
let mut requests: Vec<ModelMigrationRequest<'py>> = Vec::new();
59-
for key in native_field(scope, "execution_order")?.try_iter()? {
60-
let key = key?;
61-
if !native_field(scope, "selected_keys")?.contains(&key)?
62-
|| !native_field(&key, "resource_type")?.eq("model")?
63-
{
63+
for key in &scope.order {
64+
if !scope.selected.contains(key) || key.0 != MODEL_RESOURCE {
6465
continue;
6566
}
66-
let item = native_field(scope, "models_by_name")?
67-
.call_method1("get", (native_field(&key, "name")?,))?;
67+
let Some(item) = scope.model(py, &key.1) else {
68+
continue;
69+
};
6870
if !item.is_none()
6971
&& let Some(request) = declared(&item)?
7072
{

‎crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@ use crate::bindings::_helpers::planner_migrations::old_names::old_name_claim_err
1313
use crate::bindings::_helpers::planner_migrations::old_names::old_name_schemas_to_read;
1414
use crate::bindings::_helpers::planner_phases::objects::object;
1515
use crate::bindings::_helpers::planner_phases::runtime::runtime_field;
16+
use crate::bindings::_helpers::planner_scope::retained::scope_data;
1617
use crate::bindings::_helpers::records::fields::field as native_field;
1718
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
1819
use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple};
20+
use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE;
1921

2022
fn schemas<'py>(
2123
(phase, migration_entries): (&MigrationPhase<'py>, &Bound<'py, PyAny>),
@@ -75,7 +77,7 @@ pub(crate) fn plan_old_name_views<'py>(
7577
) -> PyResult<OldNameViewPlanning<'py>> {
7678
let py = phase.runtime.py();
7779
let runtime = phase.runtime.clone();
78-
let scope = phase.scope.clone();
80+
let scope = scope_data(&phase.scope)?;
7981
let project = runtime_field(&runtime, |runtime| &runtime.project)?;
8082
let models: Vec<Bound<'py, PyAny>> = project_reads::models(&project)?
8183
.try_iter()?
@@ -85,11 +87,10 @@ pub(crate) fn plan_old_name_views<'py>(
8587
by_name.set_item(model_reads::name_object(item)?, item)?;
8688
}
8789
let mut selected: Vec<Bound<'py, PyAny>> = Vec::new();
88-
for key in native_field(&scope, "execution_order")?.try_iter()? {
89-
let key = key?;
90-
if native_field(&scope, "selected_keys")?.contains(&key)?
91-
&& native_field(&key, "resource_type")?.eq("model")?
92-
&& let Some(item) = by_name.get_item(native_field(&key, "name")?)?
90+
for key in &scope.order {
91+
if scope.selected.contains(key)
92+
&& key.0 == MODEL_RESOURCE
93+
&& let Some(item) = by_name.get_item(&key.1)?
9394
{
9495
selected.push(item);
9596
}

‎fensu.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2028,7 +2028,7 @@ pending_budgets = [
20282028
"crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs=4",
20292029
"crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs=6",
20302030
"crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=1",
2031-
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs=4",
2031+
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs=3",
20322032
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs=3",
20332033
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs=1",
20342034
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs=8",
@@ -2037,14 +2037,14 @@ pending_budgets = [
20372037
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs=1",
20382038
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs=1",
20392039
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs=6",
2040-
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=2",
2040+
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=1",
20412041
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs=2",
20422042
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=2",
20432043
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=1",
20442044
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=9",
20452045
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs=5",
20462046
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs=4",
2047-
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs=2",
2047+
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs=1",
20482048
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs=2",
20492049
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs=13",
20502050
"crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs=4",

0 commit comments

Comments
 (0)