Skip to content

Commit 2065d96

Browse files
authored
refactor: finish native compiler and planner ownership (#1279)
* refactor: retain native compiler and planner ownership * fix: keep migration planning on native scope data Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 * chore: keep orb bootstrap in private project settings Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66
1 parent def6424 commit 2065d96

101 files changed

Lines changed: 2830 additions & 1220 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.agents/setup‎

Lines changed: 0 additions & 52 deletions
This file was deleted.

‎CONTRIBUTING.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,28 @@ uv run fensu dupes --since origin/main
111111
- Use project-local config files for local overrides, and do not commit credentials or personal settings.
112112
- Follow existing code style and module organization before introducing new patterns.
113113

114+
### Native ownership and Python interfaces
115+
116+
Compiler and planner algorithms belong in the Rust crates. Keep parsed header metadata, config
117+
nodes, dependency graphs and selections native between their producers and consumers. Constructing
118+
a Python container or public record only to extract it again is not an interface requirement.
119+
120+
The supported Python interfaces have specific owners:
121+
122+
| Interface | Why Python remains | Ownership rule |
123+
| --- | --- | --- |
124+
| Adapters and warehouse execution | Projects subclass adapters; the executor runs warehouse operations | Call through the adapter/executor boundary, retaining their public inputs and results |
125+
| User functions, factories, hooks, rules and custom constructors | These execute project-authored Python | Preserve invocation timing, errors and opaque return values; do not execute them speculatively |
126+
| Public records and caller-authored protocol objects | Python callers access, replace, compare and serialize documented records | Project native values on public access; import caller-authored values at entry, not repeatedly inside native algorithms |
127+
| Config and sample values | Python can supply values without a lossless native representation | Retain opaque values and their Python semantics; keep ordinary builtin containers native |
128+
| Streams, progress, logging and interpreter shutdown | These integrate with Python's runtime and caller-owned resources | Keep resource lifetime and error propagation at the owning boundary |
129+
130+
`python-boundary.toml` lists the Python source that intentionally stays. The native crossing,
131+
storage and layer rules in `fensu.toml` prevent new accidental dependencies; pending budgets are
132+
an inventory to reduce, not permission to add callbacks. `native_host` owns genuine host edges,
133+
not a destination for movable compiler or planner algorithms. Remove an obsolete private
134+
interface with its last caller, while retaining supported public facades and stubs.
135+
114136
## Commits And Pull Requests
115137

116138
- Use concise conventional commit messages, such as `fix: ...`, `feat: ...`, `test: ...`, or `docs: ...`.
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
MODEL (
2+
description "Order contract metadata retained beside a templated destination alias",
3+
tags [edge_case],
4+
alias "rpt_edge_header_contracts_${reporting_currency}",
5+
contract enforced,
6+
columns (
7+
order_id (type INTEGER, nullable false, audits [not_null, unique]),
8+
status (type order_status, nullable false, audits [not_null]),
9+
amount (type DECIMAL(10, 2)),
10+
),
11+
);
12+
13+
WITH final AS (
14+
SELECT
15+
7::INTEGER AS order_id,
16+
@enum("order_status").PENDING::VARCHAR AS status,
17+
12.50::DECIMAL(10, 2) AS amount
18+
FROM __seed("currencies")
19+
WHERE currency_code = 'USD'
20+
)
21+
22+
SELECT order_id, status, amount
23+
FROM final
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
MODEL (
2+
description "Ledger report whose branches share an upstream model during migration planning",
3+
tags [edge_case],
4+
columns (
5+
entry_id (type VARCHAR, nullable false, audits [not_null, unique]),
6+
channel (type VARCHAR),
7+
unit_amount (type NUMBER(38, 10)),
8+
),
9+
);
10+
11+
WITH ledger AS (
12+
SELECT entry_id
13+
FROM __ref("stg_edge_ledger_entries")
14+
),
15+
16+
amounts AS (
17+
SELECT
18+
entry_id,
19+
unit_amount
20+
FROM __ref("int_edge_line_amounts")
21+
),
22+
23+
channels AS (
24+
SELECT
25+
entry_id,
26+
channel
27+
FROM __ref("int_edge_orders_all_channels")
28+
),
29+
30+
final AS (
31+
SELECT
32+
l.entry_id,
33+
c.channel,
34+
a.unit_amount
35+
FROM ledger AS l
36+
INNER JOIN amounts AS a
37+
ON l.entry_id = a.entry_id
38+
INNER JOIN channels AS c
39+
ON l.entry_id = c.entry_id
40+
)
41+
42+
SELECT
43+
entry_id,
44+
channel,
45+
unit_amount
46+
FROM final
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
TEST (name "rpt_edge_header_contracts__keeps_enum_contract_with_templated_alias");
2+
3+
WITH __seed__currencies AS (
4+
SELECT 'USD' AS currency_code
5+
UNION ALL
6+
SELECT 'EUR' AS currency_code
7+
),
8+
__expected__rpt_edge_header_contracts AS (
9+
SELECT
10+
7::INTEGER AS order_id,
11+
'PENDING'::VARCHAR AS status,
12+
12.50::DECIMAL(10, 2) AS amount
13+
)
14+
15+
SELECT 1
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
TEST (name "rpt_edge_planner_fan_in__keeps_only_entries_in_every_branch");
2+
3+
WITH __ref__stg_edge_ledger_entries AS (
4+
SELECT v.entry_id
5+
FROM VALUES ('LE-01'), ('LE-02'), ('LE-03') AS v(entry_id)
6+
),
7+
8+
__ref__int_edge_line_amounts AS (
9+
SELECT
10+
v.entry_id,
11+
v.unit_amount::DECIMAL(38, 10) AS unit_amount
12+
FROM VALUES ('LE-01', 2.50), ('LE-02', 7.25), ('LE-04', 9.50) AS v(entry_id, unit_amount)
13+
),
14+
15+
__ref__int_edge_orders_all_channels AS (
16+
SELECT
17+
v.entry_id,
18+
v.channel
19+
FROM VALUES ('LE-01', 'web'), ('LE-03', 'app'), ('LE-04', 'partner') AS v(entry_id, channel)
20+
),
21+
22+
__expected__rpt_edge_planner_fan_in AS (
23+
SELECT
24+
'LE-01' AS entry_id,
25+
'web' AS channel,
26+
2.50::DECIMAL(38, 10) AS unit_amount
27+
)
28+
29+
SELECT 1

‎crates/sqlbuild-analysis/src/graph/_helpers/indexes.rs‎

Lines changed: 11 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,14 @@ use std::collections::{HashMap, HashSet};
44

55
use crate::assembly::project::types::ObjectKey;
66
use crate::graph::constants::{MODEL_ROOT, MODEL_ROOT_PREFIX, SQL_TEST_RESOURCE};
7-
use crate::graph::models::{GraphIndexes, GraphResource, ProjectGraph};
7+
use crate::graph::main::downstream_edges::downstream_edges;
8+
use crate::graph::models::{GraphIndexes, GraphResource, OrderedEdges, ProjectGraph};
89

910
type Edges = Vec<(ObjectKey, Vec<ObjectKey>)>;
1011

11-
/// Insertion-ordered key lists, as Python's `dict.setdefault(key, []).append(...)` builds them.
12-
#[derive(Default)]
13-
struct OrderedEdges {
14-
entries: Edges,
15-
index: HashMap<ObjectKey, usize>,
16-
}
17-
1812
impl OrderedEdges {
1913
/// The list stored for `key`, appended empty in insertion order when the key is new.
20-
fn list(&mut self, key: &ObjectKey) -> &mut Vec<ObjectKey> {
14+
pub(crate) fn list(&mut self, key: &ObjectKey) -> &mut Vec<ObjectKey> {
2115
let position: usize = match self.index.get(key) {
2216
Some(&position) => position,
2317
None => {
@@ -28,6 +22,13 @@ impl OrderedEdges {
2822
};
2923
&mut self.entries[position].1
3024
}
25+
26+
/// The dependencies already stored for `key`, without inserting a missing target.
27+
pub(crate) fn get(&self, key: &ObjectKey) -> Option<&Vec<ObjectKey>> {
28+
self.index
29+
.get(key)
30+
.map(|&position| &self.entries[position].1)
31+
}
3132
}
3233

3334
/// Tag to keys in first-tagged order, each key once, as Python's tag sets hold them.
@@ -70,7 +71,7 @@ pub(crate) fn build(resources: &[GraphResource]) -> ProjectGraph {
7071
{
7172
*upstream.list(&resource.key) = lineage_deps(&resource.deps);
7273
}
73-
let downstream: Edges = downstream(&upstream.entries);
74+
let downstream: Edges = downstream_edges(&upstream.entries);
7475
let mut tags: OrderedTags = OrderedTags::default();
7576
let mut graph = ProjectGraph {
7677
downstream_index: positions(&downstream),
@@ -125,24 +126,6 @@ fn lineage_deps(deps: &[ObjectKey]) -> Vec<ObjectKey> {
125126
.collect()
126127
}
127128

128-
/// Inverted edges keyed in upstream order, each list sorted by `(resource type, name)`.
129-
fn downstream(upstream: &[(ObjectKey, Vec<ObjectKey>)]) -> Edges {
130-
let mut inverted: OrderedEdges = OrderedEdges::default();
131-
for (key, _) in upstream {
132-
let _ = inverted.list(key);
133-
}
134-
for (key, deps) in upstream {
135-
for dep in deps {
136-
inverted.list(dep).push(key.clone());
137-
}
138-
}
139-
let mut entries: Edges = inverted.entries;
140-
for (_, keys) in &mut entries {
141-
keys.sort();
142-
}
143-
entries
144-
}
145-
146129
fn positions(entries: &[(ObjectKey, Vec<ObjectKey>)]) -> HashMap<ObjectKey, usize> {
147130
entries
148131
.iter()
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
//! Reverse ordered execution or lineage edges without projecting Python graph maps.
2+
3+
use crate::assembly::project::types::ObjectKey;
4+
use crate::graph::models::OrderedEdges;
5+
6+
/// Inverted edges keyed in upstream order, each list sorted by `(resource type, name)`.
7+
pub fn downstream_edges(
8+
upstream: &[(ObjectKey, Vec<ObjectKey>)],
9+
) -> Vec<(ObjectKey, Vec<ObjectKey>)> {
10+
let mut inverted = OrderedEdges::default();
11+
for (key, _) in upstream {
12+
let _ = inverted.list(key);
13+
}
14+
for (key, deps) in upstream {
15+
for dep in deps {
16+
inverted.list(dep).push(key.clone());
17+
}
18+
}
19+
let mut entries = inverted.entries;
20+
for (_, keys) in &mut entries {
21+
keys.sort();
22+
}
23+
entries
24+
}

‎crates/sqlbuild-analysis/src/graph/main/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
pub mod build_project_graph;
22
pub mod build_resources;
3+
pub mod downstream_edges;
34
pub mod edge_keys;
45
pub mod graph_from_indexes;
56
pub mod match_selector;
67
pub mod model_layer_count;
78
pub mod parse_selector;
9+
pub mod reachable_graph_keys;
10+
pub mod reachable_keys;
811
pub mod resolve_selector_tokens;
912
pub mod resolve_selectors;
1013
pub mod selector_name_help;
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
//! Reachable keys over the project's existing edge indexes.
2+
3+
use std::collections::HashSet;
4+
use std::convert::Infallible;
5+
6+
use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach;
7+
8+
use crate::assembly::project::types::ObjectKey;
9+
use crate::graph::main::edge_keys::edge_keys;
10+
use crate::graph::models::{EdgeDirection, ProjectGraph};
11+
12+
/// The starts and every key reachable in the given direction.
13+
pub fn reachable_graph_keys(
14+
starts: &[ObjectKey],
15+
graph: &ProjectGraph,
16+
direction: EdgeDirection,
17+
) -> HashSet<ObjectKey> {
18+
let mut visited: HashSet<&ObjectKey> = starts.iter().collect();
19+
let Ok(()) = breadth_first_reach(starts, None, &mut visited, |current| {
20+
Ok::<_, Infallible>(edge_keys(graph, current, direction))
21+
});
22+
visited.into_iter().cloned().collect()
23+
}

0 commit comments

Comments
 (0)