Skip to content

Commit 2ff5a9e

Browse files
committed
refactor: move column lineage to its own change
Column lineage publication would push large projects past the publish body limit, so it lands after the publish request is compressed. This change keeps node detail, labels and the SQL preview fixes.
1 parent 2c59909 commit 2ff5a9e

22 files changed

Lines changed: 35 additions & 710 deletions

File tree

‎clients/typescript/src/schema.d.ts‎

Lines changed: 0 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -306,22 +306,6 @@ export interface paths {
306306
patch?: never;
307307
trace?: never;
308308
};
309-
"/api/control/locations/{location_id}/targets/{target}/column-lineage": {
310-
parameters: {
311-
query?: never;
312-
header?: never;
313-
path?: never;
314-
cookie?: never;
315-
};
316-
get: operations["readPersonColumnLineage"];
317-
put?: never;
318-
post?: never;
319-
delete?: never;
320-
options?: never;
321-
head?: never;
322-
patch?: never;
323-
trace?: never;
324-
};
325309
"/api/control/locations/{location_id}/targets/{target}/connection-tests": {
326310
parameters: {
327311
query?: never;
@@ -1122,21 +1106,6 @@ export interface components {
11221106
CliLoginStartRequest: {
11231107
client_hash: string;
11241108
};
1125-
/** @description One published column lineage edge; `transform` is the compiler's transform kind, such as `direct` or `cast`. */
1126-
ColumnLineageEdge: {
1127-
source_column: string;
1128-
source_key: string;
1129-
target_column: string;
1130-
target_key: string;
1131-
transform: string;
1132-
};
1133-
/** @description The column lineage reaching or leaving one node, from the current snapshot; empty before one publishes lineage. */
1134-
ColumnLineageRead: {
1135-
edges: components["schemas"]["ColumnLineageEdge"][];
1136-
nodes: components["schemas"]["LineageNodeColumns"][];
1137-
/** Format: uuid */
1138-
snapshot_id?: string | null;
1139-
};
11401109
/** @description A row whose key was already stored with different content; the first row is kept. */
11411110
ConflictEvidence: {
11421111
existing_sha256: string;
@@ -1267,16 +1236,6 @@ export interface components {
12671236
target: string;
12681237
window?: null | components["schemas"]["RunWindow"];
12691238
};
1270-
LineageColumn: {
1271-
data_type?: string | null;
1272-
name: string;
1273-
};
1274-
/** @description A node's columns as its compiled definition infers them; `data_type` is absent when inference could not tell. */
1275-
LineageNodeColumns: {
1276-
columns: components["schemas"]["LineageColumn"][];
1277-
kind: string;
1278-
node_key: string;
1279-
};
12801239
/** @description One location target the person can view, with the actions allowed on at least one of its nodes. */
12811240
LocationAccess: {
12821241
actions: components["schemas"]["Action"][];
@@ -2895,46 +2854,6 @@ export interface operations {
28952854
};
28962855
};
28972856
};
2898-
readPersonColumnLineage: {
2899-
parameters: {
2900-
query: {
2901-
/** @description The node whose upstream and downstream column lineage to return */
2902-
node_key: string;
2903-
};
2904-
header?: never;
2905-
path: {
2906-
location_id: string;
2907-
target: string;
2908-
};
2909-
cookie?: never;
2910-
};
2911-
requestBody?: never;
2912-
responses: {
2913-
200: {
2914-
headers: {
2915-
[name: string]: unknown;
2916-
};
2917-
content: {
2918-
"application/json": components["schemas"]["ColumnLineageRead"];
2919-
};
2920-
};
2921-
/** @description Session rejected */
2922-
401: {
2923-
headers: {
2924-
[name: string]: unknown;
2925-
};
2926-
content?: never;
2927-
};
2928-
403: {
2929-
headers: {
2930-
[name: string]: unknown;
2931-
};
2932-
content: {
2933-
"application/json": components["schemas"]["ApiErrorBody"];
2934-
};
2935-
};
2936-
};
2937-
};
29382857
requestConnectionTest: {
29392858
parameters: {
29402859
query?: never;

‎crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,10 +121,6 @@ pub(crate) fn person_routes() -> Router<ControlStore> {
121121
"/api/control/locations/{location_id}/targets/{target}/snapshots",
122122
get(crate::api::_helpers::resources::person_reads::snapshots),
123123
)
124-
.route(
125-
"/api/control/locations/{location_id}/targets/{target}/column-lineage",
126-
get(crate::api::_helpers::resources::person_operations::column_lineage),
127-
)
128124
.route(
129125
"/api/control/locations/{location_id}/targets/{target}/catalog",
130126
get(crate::api::_helpers::resources::person_operations::catalog),

‎crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs‎

Lines changed: 2 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,12 @@ use crate::runs::main::request_rerun::request_rerun;
1717
use crate::runs::models::{CancelRunRequest, JobRunRequest, RerunRequest, RunRecord};
1818
use crate::store::models::ControlStore;
1919
use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog;
20-
use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage;
2120
use crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint;
2221
use crate::warehouse_metadata::main::read_person_node_status::read_person_node_status;
2322
use crate::warehouse_metadata::main::read_person_statistics::read_person_statistics;
2423
use crate::warehouse_metadata::models::{
25-
CatalogPage, CatalogQuery, ColumnLineageRead, FingerprintObservation, FingerprintQuery,
26-
NodeStatus, NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention,
24+
CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, NodeStatus,
25+
NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention,
2726
};
2827

2928
#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/statistics", operation_id = "readPersonStatistics",
@@ -185,28 +184,6 @@ pub(crate) async fn fingerprint(
185184
))
186185
}
187186

188-
#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/column-lineage", operation_id = "readPersonColumnLineage",
189-
params(("location_id" = Uuid, Path), ("target" = String, Path),
190-
("node_key" = String, Query, description = "The node whose upstream and downstream column lineage to return")),
191-
responses((status = 200, body = ColumnLineageRead), (status = 401, description = "Session rejected"),
192-
(status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))]
193-
pub(crate) async fn column_lineage(
194-
(State(store), Extension(context)): (State<ControlStore>, Extension<AuditContext>),
195-
Extension(default_role): Extension<DefaultRole>,
196-
(Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query<NodeStatusQuery>),
197-
Extension(AuthenticatedPerson(principal, _)): Extension<AuthenticatedPerson>,
198-
) -> Result<Json<ColumnLineageRead>, ApiError> {
199-
Ok(Json(
200-
read_person_column_lineage(
201-
&store,
202-
(&context, principal),
203-
(location_id, &target, &options.node_key),
204-
default_role,
205-
)
206-
.await?,
207-
))
208-
}
209-
210187
#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/node-status", operation_id = "readPersonNodeStatus",
211188
params(("location_id" = Uuid, Path), ("target" = String, Path),
212189
("node_key" = String, Query, description = "One exact published node key")),

‎crates/sqlbuild-orchestration/src/api/models.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ use crate::api::_helpers::resources::{
3030
person_operations::statistics,
3131
person_operations::fingerprint,
3232
person_operations::node_status,
33-
person_operations::column_lineage,
3433
person_checks::list,
3534
person_checks::history,
3635
person_checks::sql,
Lines changed: 2 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,11 @@
1-
//! Catalog listing rows and column lineage from snapshot node definitions and published edges.
2-
3-
use std::collections::{BTreeMap, BTreeSet, HashMap};
1+
//! Catalog listing rows from snapshot node definitions and each node's latest build.
42
53
use serde_json::Value;
64
use uuid::Uuid;
75

86
use crate::sensors::constants::SOURCE_NODE_KIND;
97
use crate::snapshots::models::SnapshotNode;
10-
use crate::warehouse_metadata::models::{
11-
CatalogNodeBuild, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, LineageColumn,
12-
LineageNodeColumns,
13-
};
8+
use crate::warehouse_metadata::models::{CatalogNodeBuild, CatalogNodeRecord, CatalogRelation};
149

1510
/// A declared value from the definition itself, its schema entry or its config, in that order.
1611
fn declared<'a>(definition: &'a Value, key: &str) -> Option<&'a Value> {
@@ -83,120 +78,3 @@ pub(crate) fn catalog_node(
8378
snapshot_id,
8479
}
8580
}
86-
87-
/// Published edge rows whose both ends are current nodes, so the graph never points at a hidden node.
88-
pub(crate) fn published_edges(
89-
document: Option<&Value>,
90-
nodes: &BTreeMap<String, SnapshotNode>,
91-
) -> Vec<ColumnLineageEdge> {
92-
let rows = document
93-
.and_then(|document| document.get("edges"))
94-
.and_then(Value::as_array);
95-
rows.into_iter()
96-
.flatten()
97-
.filter_map(edge_row)
98-
.filter(|edge| nodes.contains_key(&edge.source_key) && nodes.contains_key(&edge.target_key))
99-
.collect()
100-
}
101-
102-
fn edge_row(row: &Value) -> Option<ColumnLineageEdge> {
103-
let [
104-
source_key,
105-
source_column,
106-
target_key,
107-
target_column,
108-
transform,
109-
] = row.as_array()?.as_slice()
110-
else {
111-
return None;
112-
};
113-
let text = |value: &Value| value.as_str().map(str::to_owned);
114-
Some(ColumnLineageEdge {
115-
source_key: text(source_key)?,
116-
source_column: text(source_column)?,
117-
target_key: text(target_key)?,
118-
target_column: text(target_column)?,
119-
transform: text(transform)?,
120-
})
121-
}
122-
123-
/// Edges reached upstream and downstream from any column of `node_key`, each once, in published order.
124-
pub(crate) fn neighbourhood(edges: &[ColumnLineageEdge], node_key: &str) -> Vec<ColumnLineageEdge> {
125-
let mut reached: BTreeSet<usize> = walk(edges, node_key, true);
126-
reached.extend(walk(edges, node_key, false));
127-
reached
128-
.into_iter()
129-
.map(|index| edges[index].clone())
130-
.collect()
131-
}
132-
133-
/// The edges followed from the node's columns towards sources (`upstream`) or consumers.
134-
fn walk(edges: &[ColumnLineageEdge], node_key: &str, upstream: bool) -> BTreeSet<usize> {
135-
let mut reached: BTreeSet<usize> = BTreeSet::new();
136-
let mut by_near: HashMap<(&str, &str), Vec<usize>> = HashMap::new();
137-
for (index, edge) in edges.iter().enumerate() {
138-
by_near.entry(end(edge, !upstream)).or_default().push(index);
139-
}
140-
let mut pending: Vec<(&str, &str)> = by_near
141-
.keys()
142-
.filter(|(key, _)| *key == node_key)
143-
.copied()
144-
.collect();
145-
let mut seen: BTreeSet<(&str, &str)> = pending.iter().copied().collect();
146-
while let Some(column) = pending.pop() {
147-
for &index in by_near.get(&column).into_iter().flatten() {
148-
reached.insert(index);
149-
let far = end(&edges[index], upstream);
150-
if seen.insert(far) {
151-
pending.push(far);
152-
}
153-
}
154-
}
155-
reached
156-
}
157-
158-
/// An edge's source column, or its target column when `source` is false.
159-
fn end(edge: &ColumnLineageEdge, source: bool) -> (&str, &str) {
160-
if source {
161-
(&edge.source_key, &edge.source_column)
162-
} else {
163-
(&edge.target_key, &edge.target_column)
164-
}
165-
}
166-
167-
/// The inferred columns of the focal node and of every node an edge touches, in node key order.
168-
pub(crate) fn node_columns(
169-
nodes: &BTreeMap<String, SnapshotNode>,
170-
node_key: &str,
171-
edges: &[ColumnLineageEdge],
172-
) -> Vec<LineageNodeColumns> {
173-
let mut keys: BTreeSet<&str> = BTreeSet::from([node_key]);
174-
for edge in edges {
175-
keys.extend([edge.source_key.as_str(), edge.target_key.as_str()]);
176-
}
177-
keys.into_iter()
178-
.filter_map(|key| nodes.get_key_value(key))
179-
.map(|(key, node)| LineageNodeColumns {
180-
node_key: key.clone(),
181-
kind: node.kind.clone(),
182-
columns: definition_columns(&node.definition),
183-
})
184-
.collect()
185-
}
186-
187-
fn definition_columns(definition: &Value) -> Vec<LineageColumn> {
188-
let columns = definition.get("columns").and_then(Value::as_array);
189-
columns
190-
.into_iter()
191-
.flatten()
192-
.filter_map(|column| {
193-
Some(LineageColumn {
194-
name: column.get("name")?.as_str()?.to_owned(),
195-
data_type: column
196-
.get("type")
197-
.and_then(Value::as_str)
198-
.map(str::to_owned),
199-
})
200-
})
201-
.collect()
202-
}

‎crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ pub mod ingest_statistics;
1010
pub mod list_person_catalog_nodes;
1111
pub(crate) mod prune_statistics;
1212
pub mod read_person_catalog;
13-
pub mod read_person_column_lineage;
1413
pub mod read_person_fingerprint;
1514
pub mod read_person_node_status;
1615
pub mod read_person_statistics;

‎crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

0 commit comments

Comments
 (0)