|
1 | | -//! Catalog listing rows and column lineage from snapshot node definitions and published edges. |
2 | | -
|
3 | | -use std::collections::{BTreeMap, BTreeSet, HashMap}; |
| 1 | +//! Catalog listing rows from snapshot node definitions and each node's latest build. |
4 | 2 |
|
5 | 3 | use serde_json::Value; |
6 | 4 | use uuid::Uuid; |
7 | 5 |
|
8 | 6 | use crate::sensors::constants::SOURCE_NODE_KIND; |
9 | 7 | use crate::snapshots::models::SnapshotNode; |
10 | | -use crate::warehouse_metadata::models::{ |
11 | | - CatalogNodeBuild, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, LineageColumn, |
12 | | - LineageNodeColumns, |
13 | | -}; |
| 8 | +use crate::warehouse_metadata::models::{CatalogNodeBuild, CatalogNodeRecord, CatalogRelation}; |
14 | 9 |
|
15 | 10 | /// A declared value from the definition itself, its schema entry or its config, in that order. |
16 | 11 | fn declared<'a>(definition: &'a Value, key: &str) -> Option<&'a Value> { |
@@ -83,120 +78,3 @@ pub(crate) fn catalog_node( |
83 | 78 | snapshot_id, |
84 | 79 | } |
85 | 80 | } |
86 | | - |
87 | | -/// Published edge rows whose both ends are current nodes, so the graph never points at a hidden node. |
88 | | -pub(crate) fn published_edges( |
89 | | - document: Option<&Value>, |
90 | | - nodes: &BTreeMap<String, SnapshotNode>, |
91 | | -) -> Vec<ColumnLineageEdge> { |
92 | | - let rows = document |
93 | | - .and_then(|document| document.get("edges")) |
94 | | - .and_then(Value::as_array); |
95 | | - rows.into_iter() |
96 | | - .flatten() |
97 | | - .filter_map(edge_row) |
98 | | - .filter(|edge| nodes.contains_key(&edge.source_key) && nodes.contains_key(&edge.target_key)) |
99 | | - .collect() |
100 | | -} |
101 | | - |
102 | | -fn edge_row(row: &Value) -> Option<ColumnLineageEdge> { |
103 | | - let [ |
104 | | - source_key, |
105 | | - source_column, |
106 | | - target_key, |
107 | | - target_column, |
108 | | - transform, |
109 | | - ] = row.as_array()?.as_slice() |
110 | | - else { |
111 | | - return None; |
112 | | - }; |
113 | | - let text = |value: &Value| value.as_str().map(str::to_owned); |
114 | | - Some(ColumnLineageEdge { |
115 | | - source_key: text(source_key)?, |
116 | | - source_column: text(source_column)?, |
117 | | - target_key: text(target_key)?, |
118 | | - target_column: text(target_column)?, |
119 | | - transform: text(transform)?, |
120 | | - }) |
121 | | -} |
122 | | - |
123 | | -/// Edges reached upstream and downstream from any column of `node_key`, each once, in published order. |
124 | | -pub(crate) fn neighbourhood(edges: &[ColumnLineageEdge], node_key: &str) -> Vec<ColumnLineageEdge> { |
125 | | - let mut reached: BTreeSet<usize> = walk(edges, node_key, true); |
126 | | - reached.extend(walk(edges, node_key, false)); |
127 | | - reached |
128 | | - .into_iter() |
129 | | - .map(|index| edges[index].clone()) |
130 | | - .collect() |
131 | | -} |
132 | | - |
133 | | -/// The edges followed from the node's columns towards sources (`upstream`) or consumers. |
134 | | -fn walk(edges: &[ColumnLineageEdge], node_key: &str, upstream: bool) -> BTreeSet<usize> { |
135 | | - let mut reached: BTreeSet<usize> = BTreeSet::new(); |
136 | | - let mut by_near: HashMap<(&str, &str), Vec<usize>> = HashMap::new(); |
137 | | - for (index, edge) in edges.iter().enumerate() { |
138 | | - by_near.entry(end(edge, !upstream)).or_default().push(index); |
139 | | - } |
140 | | - let mut pending: Vec<(&str, &str)> = by_near |
141 | | - .keys() |
142 | | - .filter(|(key, _)| *key == node_key) |
143 | | - .copied() |
144 | | - .collect(); |
145 | | - let mut seen: BTreeSet<(&str, &str)> = pending.iter().copied().collect(); |
146 | | - while let Some(column) = pending.pop() { |
147 | | - for &index in by_near.get(&column).into_iter().flatten() { |
148 | | - reached.insert(index); |
149 | | - let far = end(&edges[index], upstream); |
150 | | - if seen.insert(far) { |
151 | | - pending.push(far); |
152 | | - } |
153 | | - } |
154 | | - } |
155 | | - reached |
156 | | -} |
157 | | - |
158 | | -/// An edge's source column, or its target column when `source` is false. |
159 | | -fn end(edge: &ColumnLineageEdge, source: bool) -> (&str, &str) { |
160 | | - if source { |
161 | | - (&edge.source_key, &edge.source_column) |
162 | | - } else { |
163 | | - (&edge.target_key, &edge.target_column) |
164 | | - } |
165 | | -} |
166 | | - |
167 | | -/// The inferred columns of the focal node and of every node an edge touches, in node key order. |
168 | | -pub(crate) fn node_columns( |
169 | | - nodes: &BTreeMap<String, SnapshotNode>, |
170 | | - node_key: &str, |
171 | | - edges: &[ColumnLineageEdge], |
172 | | -) -> Vec<LineageNodeColumns> { |
173 | | - let mut keys: BTreeSet<&str> = BTreeSet::from([node_key]); |
174 | | - for edge in edges { |
175 | | - keys.extend([edge.source_key.as_str(), edge.target_key.as_str()]); |
176 | | - } |
177 | | - keys.into_iter() |
178 | | - .filter_map(|key| nodes.get_key_value(key)) |
179 | | - .map(|(key, node)| LineageNodeColumns { |
180 | | - node_key: key.clone(), |
181 | | - kind: node.kind.clone(), |
182 | | - columns: definition_columns(&node.definition), |
183 | | - }) |
184 | | - .collect() |
185 | | -} |
186 | | - |
187 | | -fn definition_columns(definition: &Value) -> Vec<LineageColumn> { |
188 | | - let columns = definition.get("columns").and_then(Value::as_array); |
189 | | - columns |
190 | | - .into_iter() |
191 | | - .flatten() |
192 | | - .filter_map(|column| { |
193 | | - Some(LineageColumn { |
194 | | - name: column.get("name")?.as_str()?.to_owned(), |
195 | | - data_type: column |
196 | | - .get("type") |
197 | | - .and_then(Value::as_str) |
198 | | - .map(str::to_owned), |
199 | | - }) |
200 | | - }) |
201 | | - .collect() |
202 | | -} |
0 commit comments