Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1439,6 +1439,14 @@ export interface components {
run_id: string;
version_hash: string;
};
/** @description Checks without a result yet are not known to the server, so they are not counted. */
CatalogNodeChecks: {
/** @description Failed or errored. */
failing: number;
passing: number;
total: number;
warning: number;
};
/** @description `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler. */
CatalogNodeOutcome: {
/** Format: date-time */
Expand All @@ -1449,6 +1457,8 @@ export interface components {
};
/** @description A current snapshot node's declared metadata and latest build, without SQL, source or config. */
CatalogNodeRecord: {
/** @description The checks on the node by their newest result. */
checks: components["schemas"]["CatalogNodeChecks"];
column_count?: number | null;
/** @description The current definition differs from the one `last_build` recorded; false when either is unknown. */
definition_changed: boolean;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
pub mod read_node_check_counts;
pub mod read_person_check;
pub mod read_person_check_evidence;
pub mod read_person_check_sql;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
//! Each node's check counts from every check's newest result, keyed `kind:name` like the node keys.

use std::collections::HashMap;

use uuid::Uuid;

use crate::store::errors::ControlError;
use crate::store::models::ControlTransaction;
use crate::warehouse_history::_helpers::checks::latest_results;
use crate::warehouse_metadata::models::CatalogNodeChecks;

pub(crate) async fn read_node_check_counts(
transaction: ControlTransaction,
(location_id, target): (Uuid, &str),
) -> Result<(ControlTransaction, HashMap<String, CatalogNodeChecks>), ControlError> {
let (transaction, latest) = latest_results(transaction, (location_id, target)).await?;
let mut counts: HashMap<String, CatalogNodeChecks> = HashMap::new();
for row in latest {
let record = row.record;
let (Some(kind), Some(name)) = (record.attached_target_kind, record.attached_target_name)
else {
continue;
};
let checks = counts.entry(format!("{kind}:{name}")).or_default();
checks.total += 1;
match record.outcome.as_str() {
"pass" => checks.passing += 1,
"warn" => checks.warning += 1,
"fail" | "error" => checks.failing += 1,
_ => {}
}
}
Ok((transaction, counts))
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ use crate::warehouse_metadata::constants::{
LINEAGE_DEFINITION_BATCH, OUTCOME_FAILURE_WINDOW, OUTCOME_RECENT_RUNS,
};
use crate::warehouse_metadata::models::{
CatalogNodeBuild, CatalogNodeOutcome, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge,
LineageColumn, LineageNodeColumns,
CatalogNodeBuild, CatalogNodeChecks, CatalogNodeOutcome, CatalogNodeRecord, CatalogRelation,
ColumnLineageEdge, LineageColumn, LineageNodeColumns,
};

/// A declared value from the definition itself, its schema entry or its config, in that order.
Expand Down Expand Up @@ -70,6 +70,7 @@ pub(crate) fn catalog_node(
Option<CatalogNodeOutcome>,
bool,
),
checks: CatalogNodeChecks,
) -> CatalogNodeRecord {
let definition = &node.definition;
CatalogNodeRecord {
Expand All @@ -95,6 +96,7 @@ pub(crate) fn catalog_node(
last_build,
last_outcome,
definition_changed,
checks,
snapshot_id,
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! A person lists the current nodes with view on all of them; builds are batched indexed seeks, attempts one bounded read.
//! A person lists the current nodes with view on all of them; builds are batched indexed seeks, attempts and check results one read each.

use uuid::Uuid;

Expand All @@ -9,6 +9,7 @@ use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes;
use crate::store::errors::ControlError;
use crate::store::main::{begin::begin, commit::commit};
use crate::store::models::{ControlStore, TransactionMode};
use crate::warehouse_history::main::read_node_check_counts::read_node_check_counts;
use crate::warehouse_metadata::_helpers::catalog_nodes::{
catalog_node, last_outcome, latest_attempts,
};
Expand Down Expand Up @@ -49,6 +50,8 @@ pub async fn list_person_catalog_nodes(
let (transaction, found) =
latest_fingerprints(transaction, (location_id, target, &queries)).await?;
let (transaction, mut attempts) = latest_attempts(transaction, (location_id, target)).await?;
let (transaction, mut checks) =
read_node_check_counts(transaction, (location_id, target)).await?;
let mut builds: Vec<Option<CatalogNodeBuild>> = vec![None; nodes.len()];
let mut changed: Vec<bool> = vec![false; nodes.len()];
for (position, build) in positions.iter().zip(found) {
Expand All @@ -67,7 +70,16 @@ pub async fn list_person_catalog_nodes(
.zip(builds.into_iter().zip(changed))
.map(|((key, node), (build, changed))| {
let outcome = last_outcome(attempts.remove(key.as_str()), build.as_ref());
catalog_node((key, node), snapshot.snapshot_id, (build, outcome, changed))
let node_checks = checks
.remove(key.as_str())
.or_else(|| checks.remove(&format!("{}:{key}", node.kind)))
.unwrap_or_default();
catalog_node(
(key, node),
snapshot.snapshot_id,
(build, outcome, changed),
node_checks,
)
})
.collect();
commit(transaction).await?;
Expand Down
12 changes: 12 additions & 0 deletions crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -882,9 +882,21 @@ pub struct CatalogNodeRecord {
pub last_outcome: Option<CatalogNodeOutcome>,
/// The current definition differs from the one `last_build` recorded; false when either is unknown.
pub definition_changed: bool,
/// The checks on the node by their newest result.
pub checks: CatalogNodeChecks,
pub snapshot_id: Uuid,
}

/// Checks without a result yet are not known to the server, so they are not counted.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
pub struct CatalogNodeChecks {
pub total: usize,
pub passing: usize,
pub warning: usize,
/// Failed or errored.
pub failing: usize,
}

/// `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
pub struct CatalogNodeOutcome {
Expand Down
30 changes: 30 additions & 0 deletions crates/sqlbuild-orchestration/tests/api/test_checks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,36 @@ fn given_checks_sharing_an_audit_name_when_people_read_them_then_binding_keys_an
],
"one entry per binding key, each its newest result"
);
let (status, nodes) = get(
&client,
(&api.url, &viewer),
&format!("/api/control/locations/{location}/targets/development/nodes"),
)
.await;
assert_eq!(status, StatusCode::OK);
let checks_of = |key: &str| {
nodes
.as_array()
.expect("nodes")
.iter()
.find(|node| node["node_key"] == key)
.map(|node| node["checks"].clone())
.unwrap_or_else(|| panic!("{key} listed"))
};
assert_eq!(
checks_of("orders"),
json!({"total": 1, "passing": 1, "warning": 0, "failing": 0}),
"a node counts each check by its newest result"
);
assert_eq!(
checks_of("customers"),
json!({"total": 1, "passing": 0, "warning": 0, "failing": 1}),
"an errored check is failing"
);
assert_eq!(
checks_of("products"),
json!({"total": 0, "passing": 0, "warning": 0, "failing": 0})
);
for disclosed in ["SELECT", "private@example.com"] {
assert!(
!list.to_string().contains(disclosed),
Expand Down
4 changes: 2 additions & 2 deletions ui/src/lib/api/http-catalog.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ type W = components['schemas'];

function node(key: string, extra: Partial<W['CatalogNodeRecord']> = {}): W['CatalogNodeRecord'] {
const [kind, name] = key.split(':');
return { node_key: key, kind, name, tags: [], dependencies: [], definition_changed: false, snapshot_id: '01a1b000-0000-7000-8000-000000000001', ...extra };
return { node_key: key, kind, name, tags: [], dependencies: [], definition_changed: false, checks: { total: 0, passing: 0, warning: 0, failing: 0 }, snapshot_id: '01a1b000-0000-7000-8000-000000000001', ...extra };
}

describe('catalog node mapping', () => {
Expand Down Expand Up @@ -52,7 +52,7 @@ describe('catalog node mapping', () => {
]);
expect(orders.lastBuild).toEqual({ fingerprintRunId: 'sqb-run-1', runId: null, jobKey: null, builtAt: '2026-11-07T06:00:00Z', fingerprint: 'v1' });
expect([orders.description, orders.tags, orders.columnCount, orders.relation]).toEqual(['One row per order', ['marts'], 4, { database: null, schema: 'main', name: 'orders' }]);
expect([orders.stale, orders.checks, orders.lastOutcome, orders.group, orders.owner]).toEqual([[], null, null, null, []]);
expect([orders.stale, orders.lastOutcome, orders.group]).toEqual([[], null, null]);
});

it('marks a changed definition, and a built table older than the data upstream of it, even through a view, as stale', () => {
Expand Down
8 changes: 4 additions & 4 deletions ui/src/lib/api/http-catalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ function kindOf(wire: Wire['CatalogNodeRecord']): NodeKind | null {

/**
* One node of the current snapshot. The server reports declared metadata, the newest fingerprint,
* the newest decisive outcome and whether the definition changed since that fingerprint, not
* check counts, owners or groups, so those stay null or empty and the pages hide them. The
* the newest decisive outcome, whether the definition changed since that fingerprint, and its
* checks by their newest results. It reports no groups, so pages group by folder instead. The
* fingerprint's run is a `sqb` run, not a scheduler run. Staleness from upstream builds needs the
* whole listing, so `toCatalogNodes` adds it.
*/
Expand All @@ -31,7 +31,6 @@ export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | nu
materialization: kind === 'source' || kind === 'seed' ? null : (wire.materialization ?? null),
group: null,
tags: wire.tags,
owner: [],
description: wire.description ?? '',
path: wire.path ?? '',
upstream: wire.dependencies,
Expand All @@ -46,7 +45,8 @@ export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | nu
? [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: wire.last_build.observed_at }]
: [],
sourceFreshness: null,
checks: null
// A recording from an older server, such as the static demo's, has no counts: unknown, not zero.
checks: wire.checks ?? null
};
}

Expand Down
27 changes: 23 additions & 4 deletions ui/src/lib/api/http-overview.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ function run(id: string, status: Run['status'], queuedHoursAgo: number, endedHou
return { id, status, parentRunId, jobKey, narrowing, window: null, kind: jobKey ? 'job' : 'adhoc', statusReason: null, queuedAt: hoursAgo(queuedHoursAgo), startedAt: hoursAgo(queuedHoursAgo), endedAt: endedHoursAgo === null ? null : hoursAgo(endedHoursAgo) } as Run;
}

const node = (key: string, kind: CatalogNode['kind'], stale: StaleReason['reason'][] = []) => ({ key, kind, stale: stale.map((reason) => ({ reason })) }) as CatalogNode;
const node = (key: string, kind: CatalogNode['kind'], path: string, stale: StaleReason['reason'][] = [], extra: Partial<CatalogNode> = {}) =>
({ key, kind, path, group: null, lastOutcome: null, lastBuild: null, stale: stale.map((reason) => ({ reason })), ...extra }) as CatalogNode;
const built = { lastBuild: { builtAt: hoursAgo(1) } } as Partial<CatalogNode>;
const check = (id: string, outcome: string | null) => ({ id, latest: outcome ? { outcome } : null }) as unknown as Check;
const change = (publishedHoursAgo: number, nodes: [NodeChange['op'], string][]) =>
({ snapshotId: `s-${publishedHoursAgo}`, publishedAt: hoursAgo(publishedHoursAgo), nodes: nodes.map(([op, key]) => ({ key, op })) }) as SnapshotChange;
Expand Down Expand Up @@ -39,13 +41,28 @@ describe('overview from live reads', () => {
const overview = overviewOf({
runs,
checks: [check('a', 'pass'), check('b', 'fail'), check('c', 'error'), check('d', 'warn'), check('e', null)],
nodes: [node('model:orders', 'table', ['changed', 'behind']), node('model:totals', 'view'), node('model:events', 'incremental', ['behind']), node('source:raw', 'source'), node('seed:regions', 'seed')],
nodes: [
node('model:orders', 'table', 'models/marts/core/orders.sql', ['changed', 'behind'], built),
node('model:totals', 'view', 'models/marts/totals.sql', [], built),
node('model:returns', 'table', 'models/marts/returns.sql', [], { ...built, lastOutcome: 'failed' }),
node('model:events', 'incremental', 'models/staging/events.sql', ['behind'], built),
node('source:raw', 'source', 'models/staging/sources.yml'),
node('seed:regions', 'seed', 'seeds/regions.csv')
],
changes: [change(1, [['added', 'model:orders'], ['added', 'seed:regions'], ['removed', 'model:old']]), change(24 * 8, [['added', 'model:totals']])],
events: [event('scheduler', true), event('cleared', false)],
acknowledgements: new Map([['orders-new', { by: 'ada', at: hoursAgo(1) }]]),
now
});

it('counts the nodes it builds by top-level folder, leaving sources out', () => {
expect(overview.groups).toEqual([
{ group: 'marts', total: 3, current: 1, changed: 1, behind: 1, failed: 1 },
{ group: 'seeds', total: 1, current: 0, changed: 0, behind: 0, failed: 0 },
{ group: 'staging', total: 1, current: 0, changed: 0, behind: 1, failed: 0 }
]);
});

it('counts top-level runs queued in the last 24 hours', () => {
expect(overview.metrics.runs24h).toEqual({ total: 13, success: 2, warn: 1, partial: 1, failed: 6, active: 2 });
});
Expand All @@ -65,7 +82,7 @@ describe('overview from live reads', () => {
});

it('counts models and this week’s model changes only', () => {
expect(overview.metrics.models).toEqual({ total: 3, addedThisWeek: 1, removedThisWeek: 1 });
expect(overview.metrics.models).toEqual({ total: 4, addedThisWeek: 1, removedThisWeek: 1 });
});

it('counts check outcomes, with errors as failing', () => {
Expand All @@ -82,6 +99,8 @@ describe('overview from live reads', () => {
});

it('leaves parts without a server read empty instead of zero', () => {
expect([overview.metrics.cost, overview.groups]).toEqual([null, null]);
expect(overview.metrics.cost).toBeNull();
const unknown = overviewOf({ runs: [], checks: [], nodes: [{ ...node('model:a', 'table', 'models/a.sql'), stale: null }], changes: [], events: [], acknowledgements: new Map(), now });
expect(unknown.groups).toBeNull();
});
});
27 changes: 23 additions & 4 deletions ui/src/lib/api/http-overview.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { topFolder } from '$lib/lineage/catalog-graph';
import { checkCountsOf } from './http-catalog';
import type { Acknowledgement, AttentionItem, AttentionRun, CatalogNode, Check, Overview, PlatformEvent, Run, SnapshotChange } from './types';
import type { Acknowledgement, AttentionItem, AttentionRun, CatalogNode, Check, GroupStatus, Overview, PlatformEvent, Run, SnapshotChange } from './types';

const DAY = 24 * 60 * 60 * 1000;
const MODEL_KINDS = new Set(['view', 'table', 'incremental']);
Expand Down Expand Up @@ -41,10 +42,28 @@ export function problemRunsOf(topLevel: Run[], now: number): Run[] {

const attentionRun = ({ id, jobKey, kind, status, statusReason, nodeCounts, startedAt, endedAt }: Run): AttentionRun => ({ id, jobKey, kind, status, statusReason, nodeCounts, startedAt, endedAt });

/** Build state per top-level folder, for the nodes SQLBuild builds; a node can be both changed and behind. */
export function groupStatusOf(nodes: CatalogNode[]): GroupStatus[] {
const groups = new Map<string, GroupStatus>();
for (const n of nodes) {
if (n.kind === 'source') continue;
const group = topFolder(n);
const g = groups.get(group) ?? { group, total: 0, current: 0, changed: 0, behind: 0, failed: 0 };
const stale = n.stale ?? [];
g.total += 1;
if (n.lastOutcome === 'failed') g.failed += 1;
if (stale.some((s) => s.reason !== 'behind')) g.changed += 1;
if (stale.some((s) => s.reason === 'behind')) g.behind += 1;
if (!stale.length && n.lastOutcome !== 'failed' && n.lastBuild) g.current += 1;
groups.set(group, g);
}
return [...groups.values()].sort((a, b) => a.group.localeCompare(b.group));
}

/**
* The overview from reads the person already has. Staleness comes from the catalog listing, which
* does not check source freshness, so late sources stay unknown. The server has no cost or group
* read yet, so group status is null and cost is left out; the page hides them.
* does not check source freshness, so late sources stay unknown. Groups are top-level folders, and
* group status is null when the server reports no build state. There is no cost read, so the page hides cost.
*/
export function overviewOf({ runs, checks, nodes, changes, events, acknowledgements, now }: OverviewInputs): Overview {
const acknowledged = (r: Run) => acknowledgements.get(r.id) ?? null;
Expand Down Expand Up @@ -85,7 +104,7 @@ export function overviewOf({ runs, checks, nodes, changes, events, acknowledgeme
cost: null
},
attention,
groups: null,
groups: nodes.some((n) => n.stale !== null) ? groupStatusOf(nodes) : null,
recentChanges: changes.slice(0, RECENT_CHANGES),
latestRun: latest ? { id: latest.id, status: latest.status, endedAt: latest.endedAt } : null
};
Expand Down
Loading
Loading