Skip to content

Commit a047997

Browse files
authored
feat(ui): checks in the catalog, build status by folder, and no owner field (#1451)
* feat(ui): checks per node in the catalog, build status by folder, and no owner field The catalog read now counts each node's checks by their newest results, so the catalog lists a Checks column on a live server. Overview's build status groups the nodes SQLBuild builds by top-level folder, and the catalog filters by the same folders. Models declare no owner, so the always-empty Owner fields, filters and lineage card field are gone. * fix(ui): show a short sqb run id on Maintenance * fix(ui): keep job names readable in the runs list
1 parent 3ef8ece commit a047997

28 files changed

Lines changed: 184 additions & 104 deletions

File tree

‎clients/typescript/src/schema.d.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1439,6 +1439,14 @@ export interface components {
14391439
run_id: string;
14401440
version_hash: string;
14411441
};
1442+
/** @description Checks without a result yet are not known to the server, so they are not counted. */
1443+
CatalogNodeChecks: {
1444+
/** @description Failed or errored. */
1445+
failing: number;
1446+
passing: number;
1447+
total: number;
1448+
warning: number;
1449+
};
14421450
/** @description `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler. */
14431451
CatalogNodeOutcome: {
14441452
/** Format: date-time */
@@ -1449,6 +1457,8 @@ export interface components {
14491457
};
14501458
/** @description A current snapshot node's declared metadata and latest build, without SQL, source or config. */
14511459
CatalogNodeRecord: {
1460+
/** @description The checks on the node by their newest result. */
1461+
checks: components["schemas"]["CatalogNodeChecks"];
14521462
column_count?: number | null;
14531463
/** @description The current definition differs from the one `last_build` recorded; false when either is unknown. */
14541464
definition_changed: boolean;

‎crates/sqlbuild-orchestration/src/warehouse_history/main/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
pub mod read_node_check_counts;
12
pub mod read_person_check;
23
pub mod read_person_check_evidence;
34
pub mod read_person_check_sql;
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
//! Each node's check counts from every check's newest result, keyed `kind:name` like the node keys.
2+
3+
use std::collections::HashMap;
4+
5+
use uuid::Uuid;
6+
7+
use crate::store::errors::ControlError;
8+
use crate::store::models::ControlTransaction;
9+
use crate::warehouse_history::_helpers::checks::latest_results;
10+
use crate::warehouse_metadata::models::CatalogNodeChecks;
11+
12+
pub(crate) async fn read_node_check_counts(
13+
transaction: ControlTransaction,
14+
(location_id, target): (Uuid, &str),
15+
) -> Result<(ControlTransaction, HashMap<String, CatalogNodeChecks>), ControlError> {
16+
let (transaction, latest) = latest_results(transaction, (location_id, target)).await?;
17+
let mut counts: HashMap<String, CatalogNodeChecks> = HashMap::new();
18+
for row in latest {
19+
let record = row.record;
20+
let (Some(kind), Some(name)) = (record.attached_target_kind, record.attached_target_name)
21+
else {
22+
continue;
23+
};
24+
let checks = counts.entry(format!("{kind}:{name}")).or_default();
25+
checks.total += 1;
26+
match record.outcome.as_str() {
27+
"pass" => checks.passing += 1,
28+
"warn" => checks.warning += 1,
29+
"fail" | "error" => checks.failing += 1,
30+
_ => {}
31+
}
32+
}
33+
Ok((transaction, counts))
34+
}

‎crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ use crate::warehouse_metadata::constants::{
1717
LINEAGE_DEFINITION_BATCH, OUTCOME_FAILURE_WINDOW, OUTCOME_RECENT_RUNS,
1818
};
1919
use crate::warehouse_metadata::models::{
20-
CatalogNodeBuild, CatalogNodeOutcome, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge,
21-
LineageColumn, LineageNodeColumns,
20+
CatalogNodeBuild, CatalogNodeChecks, CatalogNodeOutcome, CatalogNodeRecord, CatalogRelation,
21+
ColumnLineageEdge, LineageColumn, LineageNodeColumns,
2222
};
2323

2424
/// A declared value from the definition itself, its schema entry or its config, in that order.
@@ -70,6 +70,7 @@ pub(crate) fn catalog_node(
7070
Option<CatalogNodeOutcome>,
7171
bool,
7272
),
73+
checks: CatalogNodeChecks,
7374
) -> CatalogNodeRecord {
7475
let definition = &node.definition;
7576
CatalogNodeRecord {
@@ -95,6 +96,7 @@ pub(crate) fn catalog_node(
9596
last_build,
9697
last_outcome,
9798
definition_changed,
99+
checks,
98100
snapshot_id,
99101
}
100102
}

‎crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//! A person lists the current nodes with view on all of them; builds are batched indexed seeks, attempts one bounded read.
1+
//! A person lists the current nodes with view on all of them; builds are batched indexed seeks, attempts and check results one read each.
22
33
use uuid::Uuid;
44

@@ -9,6 +9,7 @@ use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes;
99
use crate::store::errors::ControlError;
1010
use crate::store::main::{begin::begin, commit::commit};
1111
use crate::store::models::{ControlStore, TransactionMode};
12+
use crate::warehouse_history::main::read_node_check_counts::read_node_check_counts;
1213
use crate::warehouse_metadata::_helpers::catalog_nodes::{
1314
catalog_node, last_outcome, latest_attempts,
1415
};
@@ -49,6 +50,8 @@ pub async fn list_person_catalog_nodes(
4950
let (transaction, found) =
5051
latest_fingerprints(transaction, (location_id, target, &queries)).await?;
5152
let (transaction, mut attempts) = latest_attempts(transaction, (location_id, target)).await?;
53+
let (transaction, mut checks) =
54+
read_node_check_counts(transaction, (location_id, target)).await?;
5255
let mut builds: Vec<Option<CatalogNodeBuild>> = vec![None; nodes.len()];
5356
let mut changed: Vec<bool> = vec![false; nodes.len()];
5457
for (position, build) in positions.iter().zip(found) {
@@ -67,7 +70,16 @@ pub async fn list_person_catalog_nodes(
6770
.zip(builds.into_iter().zip(changed))
6871
.map(|((key, node), (build, changed))| {
6972
let outcome = last_outcome(attempts.remove(key.as_str()), build.as_ref());
70-
catalog_node((key, node), snapshot.snapshot_id, (build, outcome, changed))
73+
let node_checks = checks
74+
.remove(key.as_str())
75+
.or_else(|| checks.remove(&format!("{}:{key}", node.kind)))
76+
.unwrap_or_default();
77+
catalog_node(
78+
(key, node),
79+
snapshot.snapshot_id,
80+
(build, outcome, changed),
81+
node_checks,
82+
)
7183
})
7284
.collect();
7385
commit(transaction).await?;

‎crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -882,9 +882,21 @@ pub struct CatalogNodeRecord {
882882
pub last_outcome: Option<CatalogNodeOutcome>,
883883
/// The current definition differs from the one `last_build` recorded; false when either is unknown.
884884
pub definition_changed: bool,
885+
/// The checks on the node by their newest result.
886+
pub checks: CatalogNodeChecks,
885887
pub snapshot_id: Uuid,
886888
}
887889

890+
/// Checks without a result yet are not known to the server, so they are not counted.
891+
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
892+
pub struct CatalogNodeChecks {
893+
pub total: usize,
894+
pub passing: usize,
895+
pub warning: usize,
896+
/// Failed or errored.
897+
pub failing: usize,
898+
}
899+
888900
/// `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler.
889901
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
890902
pub struct CatalogNodeOutcome {

‎crates/sqlbuild-orchestration/tests/api/test_checks.rs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,36 @@ fn given_checks_sharing_an_audit_name_when_people_read_them_then_binding_keys_an
101101
],
102102
"one entry per binding key, each its newest result"
103103
);
104+
let (status, nodes) = get(
105+
&client,
106+
(&api.url, &viewer),
107+
&format!("/api/control/locations/{location}/targets/development/nodes"),
108+
)
109+
.await;
110+
assert_eq!(status, StatusCode::OK);
111+
let checks_of = |key: &str| {
112+
nodes
113+
.as_array()
114+
.expect("nodes")
115+
.iter()
116+
.find(|node| node["node_key"] == key)
117+
.map(|node| node["checks"].clone())
118+
.unwrap_or_else(|| panic!("{key} listed"))
119+
};
120+
assert_eq!(
121+
checks_of("orders"),
122+
json!({"total": 1, "passing": 1, "warning": 0, "failing": 0}),
123+
"a node counts each check by its newest result"
124+
);
125+
assert_eq!(
126+
checks_of("customers"),
127+
json!({"total": 1, "passing": 0, "warning": 0, "failing": 1}),
128+
"an errored check is failing"
129+
);
130+
assert_eq!(
131+
checks_of("products"),
132+
json!({"total": 0, "passing": 0, "warning": 0, "failing": 0})
133+
);
104134
for disclosed in ["SELECT", "private@example.com"] {
105135
assert!(
106136
!list.to_string().contains(disclosed),

‎ui/src/lib/api/http-catalog.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ type W = components['schemas'];
77

88
function node(key: string, extra: Partial<W['CatalogNodeRecord']> = {}): W['CatalogNodeRecord'] {
99
const [kind, name] = key.split(':');
10-
return { node_key: key, kind, name, tags: [], dependencies: [], definition_changed: false, snapshot_id: '01a1b000-0000-7000-8000-000000000001', ...extra };
10+
return { node_key: key, kind, name, tags: [], dependencies: [], definition_changed: false, checks: { total: 0, passing: 0, warning: 0, failing: 0 }, snapshot_id: '01a1b000-0000-7000-8000-000000000001', ...extra };
1111
}
1212

1313
describe('catalog node mapping', () => {
@@ -52,7 +52,7 @@ describe('catalog node mapping', () => {
5252
]);
5353
expect(orders.lastBuild).toEqual({ fingerprintRunId: 'sqb-run-1', runId: null, jobKey: null, builtAt: '2026-11-07T06:00:00Z', fingerprint: 'v1' });
5454
expect([orders.description, orders.tags, orders.columnCount, orders.relation]).toEqual(['One row per order', ['marts'], 4, { database: null, schema: 'main', name: 'orders' }]);
55-
expect([orders.stale, orders.checks, orders.lastOutcome, orders.group, orders.owner]).toEqual([[], null, null, null, []]);
55+
expect([orders.stale, orders.lastOutcome, orders.group]).toEqual([[], null, null]);
5656
});
5757

5858
it('marks a changed definition, and a built table older than the data upstream of it, even through a view, as stale', () => {

‎ui/src/lib/api/http-catalog.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ function kindOf(wire: Wire['CatalogNodeRecord']): NodeKind | null {
1616

1717
/**
1818
* One node of the current snapshot. The server reports declared metadata, the newest fingerprint,
19-
* the newest decisive outcome and whether the definition changed since that fingerprint, not
20-
* check counts, owners or groups, so those stay null or empty and the pages hide them. The
19+
* the newest decisive outcome, whether the definition changed since that fingerprint, and its
20+
* checks by their newest results. It reports no groups, so pages group by folder instead. The
2121
* fingerprint's run is a `sqb` run, not a scheduler run. Staleness from upstream builds needs the
2222
* whole listing, so `toCatalogNodes` adds it.
2323
*/
@@ -31,7 +31,6 @@ export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | nu
3131
materialization: kind === 'source' || kind === 'seed' ? null : (wire.materialization ?? null),
3232
group: null,
3333
tags: wire.tags,
34-
owner: [],
3534
description: wire.description ?? '',
3635
path: wire.path ?? '',
3736
upstream: wire.dependencies,
@@ -46,7 +45,8 @@ export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | nu
4645
? [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: wire.last_build.observed_at }]
4746
: [],
4847
sourceFreshness: null,
49-
checks: null
48+
// A recording from an older server, such as the static demo's, has no counts: unknown, not zero.
49+
checks: wire.checks ?? null
5050
};
5151
}
5252

‎ui/src/lib/api/http-overview.test.ts‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ function run(id: string, status: Run['status'], queuedHoursAgo: number, endedHou
99
return { id, status, parentRunId, jobKey, narrowing, window: null, kind: jobKey ? 'job' : 'adhoc', statusReason: null, queuedAt: hoursAgo(queuedHoursAgo), startedAt: hoursAgo(queuedHoursAgo), endedAt: endedHoursAgo === null ? null : hoursAgo(endedHoursAgo) } as Run;
1010
}
1111

12-
const node = (key: string, kind: CatalogNode['kind'], stale: StaleReason['reason'][] = []) => ({ key, kind, stale: stale.map((reason) => ({ reason })) }) as CatalogNode;
12+
const node = (key: string, kind: CatalogNode['kind'], path: string, stale: StaleReason['reason'][] = [], extra: Partial<CatalogNode> = {}) =>
13+
({ key, kind, path, group: null, lastOutcome: null, lastBuild: null, stale: stale.map((reason) => ({ reason })), ...extra }) as CatalogNode;
14+
const built = { lastBuild: { builtAt: hoursAgo(1) } } as Partial<CatalogNode>;
1315
const check = (id: string, outcome: string | null) => ({ id, latest: outcome ? { outcome } : null }) as unknown as Check;
1416
const change = (publishedHoursAgo: number, nodes: [NodeChange['op'], string][]) =>
1517
({ snapshotId: `s-${publishedHoursAgo}`, publishedAt: hoursAgo(publishedHoursAgo), nodes: nodes.map(([op, key]) => ({ key, op })) }) as SnapshotChange;
@@ -39,13 +41,28 @@ describe('overview from live reads', () => {
3941
const overview = overviewOf({
4042
runs,
4143
checks: [check('a', 'pass'), check('b', 'fail'), check('c', 'error'), check('d', 'warn'), check('e', null)],
42-
nodes: [node('model:orders', 'table', ['changed', 'behind']), node('model:totals', 'view'), node('model:events', 'incremental', ['behind']), node('source:raw', 'source'), node('seed:regions', 'seed')],
44+
nodes: [
45+
node('model:orders', 'table', 'models/marts/core/orders.sql', ['changed', 'behind'], built),
46+
node('model:totals', 'view', 'models/marts/totals.sql', [], built),
47+
node('model:returns', 'table', 'models/marts/returns.sql', [], { ...built, lastOutcome: 'failed' }),
48+
node('model:events', 'incremental', 'models/staging/events.sql', ['behind'], built),
49+
node('source:raw', 'source', 'models/staging/sources.yml'),
50+
node('seed:regions', 'seed', 'seeds/regions.csv')
51+
],
4352
changes: [change(1, [['added', 'model:orders'], ['added', 'seed:regions'], ['removed', 'model:old']]), change(24 * 8, [['added', 'model:totals']])],
4453
events: [event('scheduler', true), event('cleared', false)],
4554
acknowledgements: new Map([['orders-new', { by: 'ada', at: hoursAgo(1) }]]),
4655
now
4756
});
4857

58+
it('counts the nodes it builds by top-level folder, leaving sources out', () => {
59+
expect(overview.groups).toEqual([
60+
{ group: 'marts', total: 3, current: 1, changed: 1, behind: 1, failed: 1 },
61+
{ group: 'seeds', total: 1, current: 0, changed: 0, behind: 0, failed: 0 },
62+
{ group: 'staging', total: 1, current: 0, changed: 0, behind: 1, failed: 0 }
63+
]);
64+
});
65+
4966
it('counts top-level runs queued in the last 24 hours', () => {
5067
expect(overview.metrics.runs24h).toEqual({ total: 13, success: 2, warn: 1, partial: 1, failed: 6, active: 2 });
5168
});
@@ -65,7 +82,7 @@ describe('overview from live reads', () => {
6582
});
6683

6784
it('counts models and this week’s model changes only', () => {
68-
expect(overview.metrics.models).toEqual({ total: 3, addedThisWeek: 1, removedThisWeek: 1 });
85+
expect(overview.metrics.models).toEqual({ total: 4, addedThisWeek: 1, removedThisWeek: 1 });
6986
});
7087

7188
it('counts check outcomes, with errors as failing', () => {
@@ -82,6 +99,8 @@ describe('overview from live reads', () => {
8299
});
83100

84101
it('leaves parts without a server read empty instead of zero', () => {
85-
expect([overview.metrics.cost, overview.groups]).toEqual([null, null]);
102+
expect(overview.metrics.cost).toBeNull();
103+
const unknown = overviewOf({ runs: [], checks: [], nodes: [{ ...node('model:a', 'table', 'models/a.sql'), stale: null }], changes: [], events: [], acknowledgements: new Map(), now });
104+
expect(unknown.groups).toBeNull();
86105
});
87106
});

0 commit comments

Comments
 (0)