diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs index 72e3e37879..d974cbc37c 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs @@ -39,12 +39,16 @@ pub(crate) async fn check_launch_claim( Ok(transaction) } +/// A retry matches on its stored `change`; derived fields such as resource columns vary by server version. pub(crate) fn validate_retry( previous: &Row, request: &RunChangeRequest, body: &serde_json::Value, ) -> Result<(), ControlError> { - if previous.get("payload") != Some(body) + let change = + |payload: Option<&serde_json::Value>| payload.and_then(|p| p.get("change")).cloned(); + if change(previous.get("payload")).is_none() + || change(previous.get("payload")) != change(Some(body)) || previous .get("invocation_id") .and_then(serde_json::Value::as_str) diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs index 9f76494612..bd03320cd7 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -1,6 +1,6 @@ //! Publish facts atomically; launch observations never project state or release physical ownership. -use serde_json::json; +use serde_json::{Value, json}; use crate::platform::models::PlatformEvent; use crate::runs::_helpers::lifecycle::events::append_run_event; @@ -13,6 +13,20 @@ use crate::store::main::append_control_event::append_control_event; use crate::store::main::query::query; use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; +/// A change's event payload; node and check events also name their node for Recall's resource columns. +fn change_payload(request: &RunChangeRequest) -> Value { + let mut payload = json!({"change": request}); + if let Some((kind, name)) = request + .change + .node_key() + .and_then(|key| key.split_once(':')) + { + payload["resource_kind"] = json!(kind); + payload["resource_name"] = json!(name); + } + payload +} + pub(crate) async fn publish_change( transaction: ControlTransaction, request: &RunChangeRequest, @@ -22,7 +36,7 @@ pub(crate) async fn publish_change( transaction, (request.run_id, request.event_id), request.change.event_type(), - json!({"change": request}), + change_payload(request), ) .await?; if let (RunChange::LaunchFailed { code }, Some(claim_id)) = (&request.change, request.claim_id) diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index a179c87e98..c8571e0401 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -650,6 +650,17 @@ impl RunChange { } } + /// The node or check the change is about, keyed `:`. + pub(crate) fn node_key(&self) -> Option<&str> { + match self { + Self::NodeStarted { node_key } + | Self::NodeFinished { node_key, .. } + | Self::CheckStarted { node_key } + | Self::CheckFinished { node_key, .. } => Some(node_key), + _ => None, + } + } + pub(crate) fn finished_node(&self) -> Option<(&str, NodeOutcome)> { match self { Self::NodeFinished { diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs index bff9055c08..fdebec8e48 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs @@ -22,6 +22,18 @@ use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; use crate::store::types::Row; +/// Events whose node may be named only in their payload: older node and check events, and audit results. +const NODE_EVENTS: &str = "'run_node_started','run_node_finished','run_check_started','run_check_finished','audit_completed'"; + +/// The payload fields that name the node of an event in `NODE_EVENTS`, as `(column, JSON path)`. +const NODE_FIELDS: [(&str, &str); 5] = [ + ("node_key", "payload->'change'->'change'->>'node_key'"), + ("attached_target_kind", "payload->>'attached_target_kind'"), + ("attached_target_name", "payload->>'attached_target_name'"), + ("attachment_kind", "payload->>'attachment_kind'"), + ("target_name", "payload->>'target_name'"), +]; + /// The Recall invocations of one run and the nodes it selected. pub(crate) struct RunRecallScope { namespaces: Vec, @@ -222,7 +234,7 @@ pub(crate) async fn run_events_in( format!("SELECT storage_id, event_id, invocation_id, occurred_at, event_type, severity, producer, resource_id, \ resource_kind, resource_name, resource_attempt_id, statement_id, error_type, error_code, duration_ms, outcome, \ row_count, affected_rows, CASE WHEN event_type IN ('{MEMBER_RETRYING_EVENT}','{MEMBER_RECOVERED_EVENT}') \ - THEN payload END AS member_payload FROM events WHERE {filter} AND storage_id>${next} ORDER BY storage_id LIMIT ${}", next + 1) + THEN payload END AS member_payload, {} FROM events WHERE {filter} AND storage_id>${next} ORDER BY storage_id LIMIT ${}", node_columns(), next + 1) }, vec![ Parameter::Integer(after), @@ -234,6 +246,14 @@ pub(crate) async fn run_events_in( if let Some(message) = member_message(&payload)? { row.insert("message".into(), Value::String(message)); } + let node = node_of(row); + for (column, _) in NODE_FIELDS { + row.remove(column); + } + if let Some((kind, name)) = node { + row.insert("resource_kind".into(), Value::String(kind)); + row.insert("resource_name".into(), Value::String(name)); + } Ok(()) }, ) @@ -261,6 +281,35 @@ async fn recall_records( Ok((transaction, records)) } +/// Selects only the node's identity from the payload of an event without resource columns. +fn node_columns() -> String { + NODE_FIELDS + .iter() + .map(|(column, path)| { + format!("CASE WHEN resource_name IS NULL AND event_type IN ({NODE_EVENTS}) THEN {path} END AS {column}") + }) + .collect::>() + .join(", ") +} + +/// The `(kind, name)` an event is about, from its `NODE_FIELDS`: a change's node key, else an audit's target. +pub(crate) fn node_of(row: &Row) -> Option<(String, String)> { + let text = |column: &str| { + row.get(column) + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + }; + if let Some(key) = text("node_key") { + let (kind, name) = key.split_once(':')?; + return Some((kind.into(), name.into())); + } + let (kind, name) = match (text("attached_target_kind"), text("attached_target_name")) { + (Some(kind), Some(name)) => (kind, name), + _ => (text("attachment_kind")?, text("target_name")?), + }; + Some((kind.into(), name.into())) +} + /// The scheduler's summary of a member retry or recovery on its group run, from its own payload. fn member_message(payload: &Value) -> Result, ControlError> { let payload: Value = match payload { diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs index fe5e12c686..470dbf89c0 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -38,6 +38,8 @@ mod test_lifecycle; mod test_monitor; #[path = "test_node_errors.rs"] mod test_node_errors; +#[path = "test_node_event_resources.rs"] +mod test_node_event_resources; #[path = "test_permissions.rs"] mod test_permissions; #[path = "test_project_limits.rs"] diff --git a/crates/sqlbuild-orchestration/tests/runs/test_node_event_resources.rs b/crates/sqlbuild-orchestration/tests/runs/test_node_event_resources.rs new file mode 100644 index 0000000000..8a89a526cd --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_node_event_resources.rs @@ -0,0 +1,282 @@ +//! Node and check events name their node, including events stored before they had resource columns. + +use std::collections::BTreeMap; + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::execution::{change_run::change_run, claim_next_run::claim_next_run}; +use crate::runs::models::{NodeOutcome, RunAuthority, RunChange, RunChangeRequest}; +use crate::runs::tests::helpers::{queue, request, snapshot, snapshot_node}; +use crate::runs::tests::test_types::{NodeEventResourcesTestCase, NodeOfTestCase}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::store::types::Row; +use crate::warehouse_history::_helpers::run_recall::node_of; +use crate::warehouse_history::main::read_person_run_events::read_person_run_events; + +const NODE_EVENTS: [&str; 4] = [ + "run_node_started", + "run_node_finished", + "run_check_started", + "run_check_finished", +]; + +#[test] +fn given_node_and_check_reports_when_stored_and_read_then_each_event_names_its_node() { + let test_cases = [ + NodeEventResourcesTestCase { + description: "SQLite node and check events", + backend: Backend::Sqlite, + expected_named: [ + ("run_node_started", "model", "payments"), + ("run_check_started", "check", "payments_positive"), + ("run_check_finished", "check", "payments_positive"), + ("run_node_finished", "model", "payments"), + ], + }, + NodeEventResourcesTestCase { + description: "PostgreSQL node and check events", + backend: Backend::Postgres, + expected_named: [ + ("run_node_started", "model", "payments"), + ("run_check_started", "check", "payments_positive"), + ("run_check_finished", "check", "payments_positive"), + ("run_node_finished", "model", "payments"), + ], + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + crate::store::main::migrate_store::migrate_store(store) + .await + .expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.nodes.insert( + "model:payments".into(), + snapshot_node("model", json!({"name": "payments"})), + ); + input.nodes.insert( + "check:payments_positive".into(), + snapshot_node("check", json!({"name": "payments_positive"})), + ); + let published = publish_snapshot(store, &input).await.expect("snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + queue( + store, + &request((location, published.snapshot_id), 1, 1, &["model:payments"]), + ) + .await; + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("available"); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let report = |change| RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }; + let node_started = report(RunChange::NodeStarted { + node_key: "model:payments".into(), + }); + for request in [ + report(RunChange::Started), + node_started.clone(), + report(RunChange::CheckStarted { + node_key: "check:payments_positive".into(), + }), + report(RunChange::CheckFinished { + node_key: "check:payments_positive".into(), + outcome: NodeOutcome::Success, + }), + report(RunChange::NodeFinished { + node_key: "model:payments".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(3), + error_code: None, + skipped_because_of: None, + error_message: None, + }), + ] { + assert!(change_run(store, worker, &request).await.expect("report")); + } + + let stored: Vec<(String, Option, Option)> = + read_recall_events(store, 0, 1000) + .await + .expect("events") + .iter() + .filter(|row| NODE_EVENTS.contains(&row["event_type"].as_str().unwrap_or(""))) + .map(|row| { + let text = |field: &str| row[field].as_str().map(str::to_owned); + ( + text("event_type").unwrap_or_default(), + text("resource_kind"), + text("resource_name"), + ) + }) + .collect(); + assert_eq!( + stored, + test_case + .expected_named + .iter() + .map(|(event, kind, name)| ( + (*event).to_owned(), + Some((*kind).to_owned()), + Some((*name).to_owned()), + )) + .collect::>(), + "{}: each node and check event stores its node in the resource columns", + test_case.description + ); + assert!( + !change_run(store, worker, &node_started) + .await + .expect("retried report"), + "{}: a retried report is the same report", + test_case.description + ); + + // An event stored before node events carried resource columns: the old payload, no columns. + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let (transaction, _) = query( + transaction, + "UPDATE recall_events SET resource_kind=NULL, resource_name=NULL, payload=$1 WHERE event_id=$2", + vec![ + Parameter::Json(json!({"change": node_started})), + Parameter::Text(node_started.event_id.to_string()), + ], + ) + .await + .expect("store the old shape"); + commit(transaction).await.expect("commit"); + assert!( + !change_run(store, worker, &node_started) + .await + .expect("retried report against the old shape"), + "{}: a report stored by an older server still matches its retry", + test_case.description + ); + + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("administrator"); + let events = read_person_run_events( + store, + (&context, admin), + ((location, "development", claim.request.run_id), (0, 1000)), + DefaultRole::None, + ) + .await + .expect("run events") + .expect("run"); + let read: Vec<(String, Option, Option)> = events + .iter() + .filter(|event| NODE_EVENTS.contains(&event.event_type.as_str())) + .map(|event| { + ( + event.event_type.clone(), + event.resource_kind.clone(), + event.resource_name.clone(), + ) + }) + .collect(); + assert_eq!( + read, stored, + "{}: the old event is named from its payload", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_node_fields_when_reading_their_node_then_the_node_key_or_audit_target_names_it() { + let test_cases = [ + NodeOfTestCase { + description: "a scheduler change keyed :", + fields: &[("node_key", "seed:region_codes")], + expected_node: Some(("seed", "region_codes")), + }, + NodeOfTestCase { + description: "a node key without a kind names no node", + fields: &[ + ("node_key", "orders"), + ("attachment_kind", "model"), + ("target_name", "orders"), + ], + expected_node: None, + }, + NodeOfTestCase { + description: "a column audit names the table it is attached to", + fields: &[ + ("attachment_kind", "column"), + ("target_name", "code"), + ("attached_target_kind", "seed"), + ("attached_target_name", "region_codes"), + ], + expected_node: Some(("seed", "region_codes")), + }, + NodeOfTestCase { + description: "an audit without an attached target names its own target", + fields: &[("attachment_kind", "model"), ("target_name", "orders")], + expected_node: Some(("model", "orders")), + }, + NodeOfTestCase { + description: "a run-scope audit has no node", + fields: &[("attachment_kind", "end")], + expected_node: None, + }, + NodeOfTestCase { + description: "no fields", + fields: &[], + expected_node: None, + }, + ]; + for test_case in test_cases { + let row: Row = test_case + .fields + .iter() + .map(|(column, value)| ((*column).to_owned(), json!(value))) + .collect(); + assert_eq!( + node_of(&row), + test_case + .expected_node + .map(|(kind, name)| (kind.to_owned(), name.to_owned())), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 625a97e45b..552984c3fe 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -517,3 +517,17 @@ pub(crate) struct GroupMemberSettingsTestCase { pub(crate) expected_group_status: RunStatus, pub(crate) expected_group_ignored: u32, } + +pub(crate) struct NodeEventResourcesTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// Each node and check event's type, resource kind and resource name, in the order they happened. + pub(crate) expected_named: [(&'static str, &'static str, &'static str); 4], +} + +pub(crate) struct NodeOfTestCase { + pub(crate) description: &'static str, + /// The node identity columns a run's events read selects from an event's payload. + pub(crate) fields: &'static [(&'static str, &'static str)], + pub(crate) expected_node: Option<(&'static str, &'static str)>, +} diff --git a/ui/PERF.md b/ui/PERF.md index 6b793bb4c1..eb077ca57a 100644 --- a/ui/PERF.md +++ b/ui/PERF.md @@ -9,6 +9,8 @@ fail when a page breaks these budgets; keep them passing rather than raising the - Opening a list page, first or repeat visit: settled within 2 s. - Graph pan, wheel zoom, trackpad momentum and pinch: 90% of frames ≤ 50 ms, none over 200 ms. - Typing in a search or selection field: suggestions update without a visible stall. +- A page open while a run goes on (a change about every second): under half the time busy, so it keeps + up; the run page's timeline scrolls at the graph budget above. - The plan page on a real-sized plan: switching section, or Show more, has its rows on screen within 1 s. A large section loads one page of rows when it is opened, never all of them. @@ -27,10 +29,15 @@ fail when a page breaks these budgets; keep them passing rather than raising the worked out once per card, and again only when its font loads, and shown by the class. Edge routes are worked out when the nodes change (a layout, a measure, a drop), never per frame. 4. **No DOM measuring in reactive paths** that run on scroll, pan or zoom. -5. **Fetch once, derive locally.** Per-snapshot data (node lists, selection vocabulary) is cached per +5. **Reload in proportion to what changed.** A running run sends a change about every second. A read + that fetches a whole listing (all checks, all of a run's events) reloads at most every few seconds + (`createResource(..., { every })`, `throttled`), and app-wide badges at most every 10 s; small reads + (a run's node states) reload after each burst. A reload never shares a request started before the + change it answers. +6. **Fetch once, derive locally.** Per-snapshot data (node lists, selection vocabulary) is cached per location and snapshot; debounce server evaluation of typed input (~150 ms). -6. **Measure on realistic data.** Mock and benchmark data must be sized like a real project (thousands +7. **Measure on realistic data.** Mock and benchmark data must be sized like a real project (thousands of nodes, long names, populated state). An optimisation proved only on small or empty data does not count. -7. **Prove a guard can fail.** When adding a performance check, break the thing it protects once and +8. **Prove a guard can fail.** When adding a performance check, break the thing it protects once and confirm the check fails, then revert. diff --git a/ui/e2e/compact.spec.ts b/ui/e2e/compact.spec.ts index d2af2e7c79..7c7071504b 100644 --- a/ui/e2e/compact.spec.ts +++ b/ui/e2e/compact.spec.ts @@ -3,8 +3,8 @@ import { zoomTo } from './zoom'; // Zoomed out, cards take their compact form: a one-line pill, a ring in the colour of the card's state // holding its kind's icon, then the part of its name that tells it apart from its siblings. The name -// never wraps, shows at 12 px or more on screen, shrinks to fit before it is cut, and is cut only at its -// end. Cards are drawn in full down to 0.82. +// never wraps, shows at 12 px or more on screen, shrinks to fit before it is cut, and is cut at its +// start, keeping the end that tells it apart. Cards are drawn in full down to 0.82. // With SHOTS set, each graph is also photographed at each zoom: SHOTS=/some/dir npx playwright test e2e/compact.spec.ts const shots = process.env.SHOTS; @@ -57,18 +57,20 @@ function pillProblems(): string[] { problems.push(`${full}: no name shown`); continue; } - visible.push(`${pill.dataset.parent ?? ''} / ${name.textContent ?? ''}`); + // Down to 0.5 what each shows tells siblings apart; further out, about four characters can't, so their tails must. + visible.push(`${pill.dataset.parent ?? ''} / ${zoom >= 0.49 ? (name.textContent ?? '') : (name.dataset.tail ?? '')}`); const px = parseFloat(getComputedStyle(name).fontSize) * zoom; if (px < 11.95) problems.push(`${full}: its name shows at ${px.toFixed(2)} px`); if (Math.abs(box.height - 30) > 1) problems.push(`${full}: its pill is ${box.height.toFixed(1)} px tall`); // One line: never wrapped, one line box. if (getComputedStyle(name).whiteSpace !== 'nowrap' || name.getClientRects().length !== 1 || name.getBoundingClientRect().height > px * 1.6) problems.push(`${full}: its name wraps`); - // Cut only once it can shrink no further, and then at its end. - if (name.scrollWidth > name.clientWidth + 0.5) { - if (px > 12.05) problems.push(`${full}: cut at ${px.toFixed(2)} px, before shrinking to 12`); - if (getComputedStyle(name).textOverflow !== 'ellipsis') problems.push(`${full}: cut without an ellipsis at its end`); - } - if (!full.endsWith(name.textContent ?? '\0')) problems.push(`${full}: shows "${name.textContent}", not an ending of its name`); + // It fits; cut only once it can shrink no further, and then in its middle, keeping both ends. + const text = name.textContent ?? ''; + const tail = name.dataset.tail ?? ''; + const [head, end = ''] = text.split('…'); + if (name.scrollWidth > name.clientWidth + 0.5) problems.push(`${full}: "${text}" overflows its pill`); + if (text.includes('…') && px > 12.05) problems.push(`${full}: cut at ${px.toFixed(2)} px, before shrinking to 12`); + if (!full.endsWith(tail) || !tail.startsWith(head) || !tail.endsWith(end)) problems.push(`${full}: shows "${text}", not its tail "${tail}" cut in the middle`); const ring = pill.querySelector('.ring')!; const want = expected(card); if (want && getComputedStyle(ring).borderTopColor !== want) problems.push(`${full}: its ring is ${getComputedStyle(ring).borderTopColor}, its state ${want}`); @@ -129,3 +131,20 @@ test('a job pill’s play triangle sits in the middle of its ring', async ({ pag } } }); + +test('a selected pill stands out from the rest', async ({ page }) => { + await page.goto('/lineage?mock'); + await page.locator('.graph-canvas:not(.unfitted) .svelte-flow__node').first().waitFor({ timeout: 20_000 }); + await page.waitForTimeout(800); + await zoomTo(page, 0.5); + const pills = page.locator('.cface .pill'); + const looks = (i: number) => pills.nth(i).evaluate((p) => ({ border: getComputedStyle(p).borderTopColor, ring: getComputedStyle(p).boxShadow, fill: getComputedStyle(p).backgroundColor })); + const before = await looks(1); + await pills.nth(1).click(); + const [selected, other] = [await looks(1), await looks(2)]; + const primary = await page.evaluate(() => { const i = document.body.appendChild(document.createElement('i')); i.style.color = getComputedStyle(document.documentElement).getPropertyValue('--primary'); const c = getComputedStyle(i).color; i.remove(); return c; }); + expect(selected.border).toBe(primary); + expect(selected.ring).not.toBe('none'); + expect(selected.fill).not.toBe(before.fill); + expect(other.border).not.toBe(primary); +}); diff --git a/ui/e2e/run-page.spec.ts b/ui/e2e/run-page.spec.ts new file mode 100644 index 0000000000..bfdfd703a7 --- /dev/null +++ b/ui/e2e/run-page.spec.ts @@ -0,0 +1,30 @@ +import { expect, test } from '@playwright/test'; + +// The run page while a run goes on: nodes that haven't started are waiting their turn, not left out. +test.use({ viewport: { width: 1600, height: 1000 } }); + +test('a running run’s nodes that haven’t started read as waiting', async ({ page }) => { + await page.goto('/runs/run-9d21?mock'); + const panel = page.getByLabel('Nodes by outcome'); + await expect(panel.locator('.group[data-group="waiting"]')).toContainText('Waiting', { timeout: 20_000 }); + await expect(panel.locator('.group[data-group="not_run"]')).toHaveCount(0); + await expect(page.locator('.gantt .note', { hasText: 'Waiting' }).first()).toBeVisible(); + await expect(page.locator('.gantt .note', { hasText: /Pending|Not run/ })).toHaveCount(0); +}); + +// A row is painted only within its own box, so a duration that would run past it reads on its bar's left. +for (const width of [760, 1000]) { + test(`no node's duration runs past its row at ${width}px`, async ({ page }) => { + await page.setViewportSize({ width, height: 1000 }); + for (const run of ['run-2b70', 'run-4f2a']) { + await page.goto(`/runs/${run}?mock`); + await expect(page.locator('.gantt .dur').first()).toBeVisible({ timeout: 20_000 }); + const overflows = await page.locator('.gantt .dur').evaluateAll((labels) => + labels + .filter((label) => label.getBoundingClientRect().right > label.closest('.row')!.getBoundingClientRect().right + 0.5) + .map((label) => label.textContent) + ); + expect(overflows, run).toEqual([]); + } + }); +} diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index a12b202709..f48e614550 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -1256,7 +1256,12 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, return data; } - implemented.changes.subscribe(() => kept.clear()); + // A change makes every listing read since then out of date: a reload it asks for reads afresh rather + // than sharing a request that started before it, which could leave a run's last state unshown. + implemented.changes.subscribe(() => { + kept.clear(); + inFlight.clear(); + }); return new Proxy(implemented, { get(target, property, receiver) { if (property in target) return Reflect.get(target, property, receiver); diff --git a/ui/src/lib/api/http-runs.test.ts b/ui/src/lib/api/http-runs.test.ts index 25a8559f4e..33f1c12863 100644 --- a/ui/src/lib/api/http-runs.test.ts +++ b/ui/src/lib/api/http-runs.test.ts @@ -251,6 +251,21 @@ describe('run Recall reads', () => { ]); }); + it('names the node a seed build event is about, as the run keys its nodes, from the resource kind and name', () => { + // As the server sends them now: an opaque resource ID, with the resource's kind and name. + const seed = { resource_id: '3b7370f99e385665996e8f73523b4fd5', resource_kind: 'seed', resource_name: 'seed__order_type__alpha' }; + const mapped = [event('resource_attempt_started', seed), event('resource_attempt_completed', { ...seed, outcome: 'success' }), event('statement_completed', { ...seed, statement_id: 's9' })].flatMap((e) => toRunEvent(e, 'r1') ?? []); + expect(mapped.map((e) => e.nodeKey)).toEqual(['seed:seed__order_type__alpha', 'seed:seed__order_type__alpha', 'seed:seed__order_type__alpha']); + const [statement] = toRunStatements([{ invocation_id: inv, statement_id: 's9', sql_available: true, ...seed }], 'r1'); + expect(statement.nodeKey).toBe('seed:seed__order_type__alpha'); + // A model's rows carry its materialization, which the run keys as `model:`. + const table = { resource_id: '9c1e', resource_kind: 'table', resource_name: 'orders' }; + const view = { resource_id: '9c1f', resource_kind: 'view', resource_name: 'customers' }; + expect([toRunEvent(event('resource_attempt_failed', table), 'r1')?.nodeKey, toRunStatements([{ invocation_id: inv, statement_id: 's10', sql_available: true, ...view }], 'r1')[0].nodeKey]).toEqual(['model:orders', 'model:customers']); + // A hash alone names no node; the scheduler's node events carry none yet. + expect(toRunEvent(event('resource_attempt_started', { resource_id: '3b7370f99e385665996e8f73523b4fd5' }), 'r1')?.nodeKey).toBeNull(); + }); + it('orders statements as sent and tells running from finished by the end time', () => { const statement = (id: string, extra: Partial): W['RunStatementRecord'] => ({ invocation_id: inv, statement_id: id, sql_available: true, ...extra }); const statements = toRunStatements( @@ -268,6 +283,30 @@ describe('run Recall reads', () => { ]); }); + it('shows printed output as the terminal left it: spinner redraws and colours gone, lines whole across chunks', () => { + const chunk = (sequence: number, text: string) => ({ invocation_id: inv, sequence, stream: 'stdout', occurred_at: `2026-11-08T06:00:0${sequence}Z`, text }); + const output = toRunOutput({ + lines: [ + chunk(1, 'Location snapshot compiled.\n\x1b[?25l\r\x1b[K⠋ Compiling project...\r\x1b[K⠙ Compil'), + chunk(2, 'ing project...\r\x1b[K\x1b[?25hCompiled project. (18.07s)\n\n\n\x1b[1mPlan ready\x1b[0m 93 selected\n'), + chunk(3, '├── seed__order_type__alpha (current)\n'), + chunk(4, 'Loaded orders.\r\n50%\r100%\r\n⠙ Waiting...\r\x1b[K\nDone.\n') + ], + errors: [] + } as never); + expect(output.map((l) => [l.at.slice(-3, -1), l.text])).toEqual([ + ['01', 'Location snapshot compiled.'], + ['01', 'Compiled project. (18.07s)'], + ['02', ''], + ['02', 'Plan ready 93 selected'], + ['03', '├── seed__order_type__alpha (current)'], + ['04', 'Loaded orders.'], + ['04', '100%'], + ['04', ''], + ['04', 'Done.'] + ]); + }); + it('merges error text into the output as stderr in time order', () => { const output = toRunOutput({ lines: [ diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index a8eb203cea..45cb21f668 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -393,11 +393,20 @@ const KNOWN_EVENTS: ReadonlySet = new Set(RECALL_EVENT_TYPES); /** Scheduler facts are stored as `run_` events in the scheduler invocation. */ const SCHEDULER_EVENTS: ReadonlySet = new Set(SCHEDULER_FACTS.map((f) => `run_${f}`)); -/** The node key a Recall resource ID names: `[invocation, key]`, or null for run-level rows. */ -function recallNodeKey(resourceId: string | null | undefined): string | null { - if (!resourceId) return null; +/** Builds record a model under its materialization, while the run keys every model as `model:`. */ +const MODEL_RESOURCE_KINDS: ReadonlySet = new Set(['table', 'view', 'custom', 'snapshot']); + +/** + * The node a Recall row is about, as a run's nodes are keyed (`seed:countries`): its resource's kind and + * name. Older servers sent no kind and name, and a `["", ""]` resource ID instead; + * any other resource ID is an opaque hash. + */ +export function recallNodeKey(row: { resource_id?: string | null; resource_kind?: string | null; resource_name?: string | null }): string | null { + if (row.resource_kind && row.resource_name) return `${MODEL_RESOURCE_KINDS.has(row.resource_kind) ? 'model' : row.resource_kind}:${row.resource_name}`; + const id = row.resource_id; + if (!id?.startsWith('[')) return null; try { - const parsed: unknown = JSON.parse(resourceId); + const parsed: unknown = JSON.parse(id); return Array.isArray(parsed) && typeof parsed[1] === 'string' ? parsed[1] : null; } catch { return null; @@ -437,7 +446,7 @@ export function toRunEvent(wire: Wire['RunEventRecord'], runId: string): RunEven occurredAt: wire.occurred_at, level: levelOf(wire.severity), invocationId: wire.invocation_id, - nodeKey: recallNodeKey(wire.resource_id), + nodeKey: recallNodeKey(wire), statementId: wire.statement_id ?? null, attempt: null, message @@ -466,7 +475,7 @@ export function toRunStatements(wire: Wire['RunStatementRecord'][], runId: strin runId, invocationId: s.invocation_id, sequence: i + 1, - nodeKey: recallNodeKey(s.resource_id) ?? s.resource_name ?? '', + nodeKey: recallNodeKey(s) ?? s.resource_name ?? '', kind: s.statement_kind ?? s.purpose ?? 'statement', status: s.outcome === 'failed' ? 'failed' : s.outcome === 'cancelled' ? 'cancelled' : s.ended_at || s.outcome ? 'completed' : 'running', startedAt: s.started_at ?? '', @@ -480,8 +489,42 @@ export function toRunStatements(wire: Wire['RunStatementRecord'][], runId: strin } /** Captured lines and error text in time order; error text reads as stderr of its invocation. */ +/** ANSI escape sequences: colours, cursor moves and the like. */ +const ANSI = /\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b[@-Z\\-_]/g; + +/** + * Printed output as a terminal would leave it on screen. `sqb` redraws spinner and progress lines with + * a carriage return, across chunks, so each stream's chunks are joined, split into lines, and each line + * drops the carriage return of a CRLF ending, then keeps only what follows its last carriage return, + * without escape codes; a line erased by `\r\x1b[K` stays blank. A line is timed by the chunk + * it started in; runs of blank lines read as one. + */ +export function terminalLines(chunks: { invocationId: string; stream: 'stdout' | 'stderr'; at: string; text: string }[]): RunOutputLine[] { + const out: RunOutputLine[] = []; + for (const stream of Map.groupBy(chunks, (c) => `${c.invocationId}\u0000${c.stream}`).values()) { + let line = ''; + let at = stream[0].at; + const finish = () => { + const shown = line.replace(/\r+$/, '').replace(ANSI, '').split('\r').at(-1)!.trimEnd(); + if (shown || out.at(-1)?.text) out.push({ invocationId: stream[0].invocationId, stream: stream[0].stream, at, text: shown, error: false }); + line = ''; + }; + for (const chunk of stream) { + const parts = chunk.text.split('\n'); + parts.forEach((part, i) => { + if (!line) at = chunk.at; + line += part; + if (i < parts.length - 1) finish(); + }); + } + if (line) finish(); + } + return out; +} + export function toRunOutput(wire: Wire['RunOutput']): RunOutputLine[] { - const lines: RunOutputLine[] = wire.lines.map((l) => ({ invocationId: l.invocation_id, stream: l.stream === 'stderr' ? 'stderr' : 'stdout', at: l.occurred_at, text: l.text, error: false })); + const ordered = [...wire.lines].sort((a, b) => a.occurred_at.localeCompare(b.occurred_at) || (a.sequence ?? 0) - (b.sequence ?? 0)); + const lines = terminalLines(ordered.map((l) => ({ invocationId: l.invocation_id, stream: l.stream === 'stderr' ? 'stderr' : 'stdout', at: l.occurred_at, text: l.text }))); const errors: RunOutputLine[] = wire.errors.map((e) => ({ invocationId: e.invocation_id, stream: 'stderr', at: e.occurred_at ?? '', text: e.message, error: true })); return [...lines, ...errors].sort((a, b) => a.at.localeCompare(b.at)); } diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index 6346f6636c..43a32d1a54 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -52,6 +52,8 @@ export type SourceFreshness = { export type CheckCounts = { total: number; passing: number; warning: number; failing: number }; export type CatalogNode = { + /** Set by a page following an active run: the node's state in that run, which outranks its last build. */ + liveStatus?: 'run' | 'ok' | 'warn' | 'fail'; key: string; kind: NodeKind; /** e.g. `table`, `view`, `incremental · merge`, `incremental · microbatch`; null for sources and tasks. */ diff --git a/ui/src/lib/components/graph/compact-face.svelte b/ui/src/lib/components/graph/compact-face.svelte index 893ad44ae7..77ec59971f 100644 --- a/ui/src/lib/components/graph/compact-face.svelte +++ b/ui/src/lib/components/graph/compact-face.svelte @@ -5,7 +5,7 @@ import SquareSquareIcon from '@lucide/svelte/icons/square-square'; import { COMPACT_LEVELS } from '$lib/lineage/fit'; import { fonts } from './fonts.svelte'; - import { measureCompact, pillFont, pillRoom } from './compact-title'; + import { measureCompact, pillFont, pillRoom, pillText } from './compact-title'; import { COMPACT_TAILS, type CompactTails } from './compact-tails'; // A card's compact form, shown by the graph (the `compact` class on its canvas) when zoomed out too far @@ -21,7 +21,14 @@ * The name's size on screen at each level, fitted at the level's lowest zoom, where the card is * narrowest; measured again once the name's font has loaded. */ - const sizes = $derived((void fonts.ready, void fonts.version, COMPACT_LEVELS.map((zoom) => pillFont(tail, pillRoom(zoom), measureCompact)))); + const sizes = $derived( + (void fonts.ready, + void fonts.version, + COMPACT_LEVELS.map((zoom) => { + const px = pillFont(tail, pillRoom(zoom), measureCompact); + return { px, text: pillText(tail, px, pillRoom(zoom), measureCompact) }; + })) + ); /** The icon's size on screen. */ const ICON = 10; /** The badge's size on screen. */ @@ -33,7 +40,7 @@ {#if kind === 'job'}{:else if kind === 'group'}{:else}{/if} - {#each sizes as px, level (level)}{tail}{/each} + {#each sizes as fit, level (level)}{fit.text}{/each} {#if badge != null}{badge}{/if} @@ -80,6 +87,12 @@ border-color: color-mix(in srgb, var(--chart-3) 45%, transparent); background: color-mix(in srgb, var(--chart-3) 10%, var(--card)); } + /* Selected: unmistakable at any zoom, a primary edge, a ring around it and a tinted fill. */ + :global(.svelte-flow__node.selected) .cface .pill { + border-color: var(--primary); + background: color-mix(in srgb, var(--primary) 16%, var(--card)); + box-shadow: 0 0 0 calc(3px * var(--u)) color-mix(in srgb, var(--primary) 40%, transparent); + } .ring { display: grid; box-sizing: border-box; @@ -108,7 +121,6 @@ font-family: var(--font-node-compact); font-weight: 500; line-height: 1.2; - text-overflow: ellipsis; white-space: nowrap; } :global(.graph-canvas[data-compact='1']) .nm[data-level='1'], diff --git a/ui/src/lib/components/graph/compact-tails.test.ts b/ui/src/lib/components/graph/compact-tails.test.ts index c10f0c8f71..7e0957292b 100644 --- a/ui/src/lib/components/graph/compact-tails.test.ts +++ b/ui/src/lib/components/graph/compact-tails.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { compactTails, compactTitleOf, siblingTails } from './compact-tails'; +import { pillText, type Measure } from './compact-title'; describe('compact pill names', () => { it('show the shortest ending of `__` parts no other card shares', () => { @@ -39,3 +40,58 @@ describe('compact pill names', () => { expect(compactTitleOf({})).toBeNull(); }); }); + +describe('a pill name too long to fit', () => { + // Seeds of one job, named as a real project names them: shared endings across `__` layers, and long + // single-part names that differ only at their end. + const names = [ + 'seed__order_type__alpha', + 'seed__order_type__beta', + 'seed__order_type__gamma', + 'seed__order_type__delta', + 'seed__layout__alpha', + 'seed__layout__beta', + 'seed__layout__gamma', + 'seed__region_usability_country', + 'seed__region_usability_check', + 'seed__region_usability_profile', + 'seed__canonical_course_mapping__north', + 'seed__canonical_course_mapping__south' + ]; + const measure: Measure = (text, font) => text.length * font * 0.6; + + it('takes the shortest unique ending of `__` parts among its siblings, for every name alike', () => { + const tails = compactTails(names); + expect(names.map((n) => tails.get(n))).toEqual([ + 'order_type__alpha', + 'order_type__beta', + 'order_type__gamma', + 'delta', + 'layout__alpha', + 'layout__beta', + 'layout__gamma', + 'region_usability_country', + 'region_usability_check', + 'region_usability_profile', + 'north', + 'south' + ]); + }); + + it('loses its middle, keeping both ends, so siblings still read differently', () => { + const tails = compactTails(names); + const shown = names.map((n) => pillText(tails.get(n)!, 12, 100, measure)); + // Thirteen characters fit: four from the start, an ellipsis, eight from the end. + expect(shown[0]).toBe('orde…e__alpha'); + expect(shown[7]).toBe('regi…_country'); + expect(shown[3]).toBe('delta'); + for (const [i, s] of shown.entries()) { + const [head, end = ''] = s.split('…'); + expect(tails.get(names[i])!.startsWith(head) && tails.get(names[i])!.endsWith(end)).toBe(true); + } + // Names that differ only at their start keep it. + expect(['stg_customers_daily', 'dim_customers_daily'].map((n) => pillText(n, 12, 80, measure))).toEqual(['stg…s_daily', 'dim…s_daily']); + expect(new Set(shown).size).toBe(names.length); + for (const s of shown) expect(measure(s, 12)).toBeLessThanOrEqual(100); + }); +}); diff --git a/ui/src/lib/components/graph/compact-title.ts b/ui/src/lib/components/graph/compact-title.ts index 82d0ae3853..70b5642fc1 100644 --- a/ui/src/lib/components/graph/compact-title.ts +++ b/ui/src/lib/components/graph/compact-title.ts @@ -1,6 +1,8 @@ // Text measured to fit a card: the status line's parts, and a zoomed-out pill's name. A pill's name is -// sized on screen, as large as fits between `PILL.minFont` and `PILL.maxFont`, then cut with an -// ellipsis at its end. Worked out once per compact level, never per zoom step. +// sized on screen, as large as fits between `PILL.minFont` and `PILL.maxFont`; a name too long even +// then loses its middle, keeping its start and, more of, its end: siblings differ at either end +// (`stg_orders`, `dim_orders`; `region_check`, `region_country`). Worked out once per compact level, +// never per zoom step. import { CARD_WIDTH } from '$lib/lineage/fit'; @@ -26,6 +28,24 @@ export function pillFont(text: string, room: number, measure: Measure): number { return font; } +/** `text` without its middle: `kept` characters, a third from its start and the rest from its end. */ +function middleCut(text: string, kept: number): string { + const head = Math.floor(kept / 3); + return `${text.slice(0, head)}…${kept - head ? text.slice(-(kept - head)) : ''}`; +} + +/** `text` at `font` as it fits `room`: whole, or with its middle cut to the most characters that fit. */ +export function pillText(text: string, font: number, room: number, measure: Measure): string { + if (measure(text, font, PILL.weight) <= room) return text; + let [lo, hi] = [0, text.length - 1]; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (measure(middleCut(text, mid), font, PILL.weight) <= room) lo = mid; + else hi = mid - 1; + } + return middleCut(text, lo); +} + let context: CanvasRenderingContext2D | null | undefined; const loading = new Set(); /** A measurer in `family` (or the family `family()` names when it is measured), estimating generously until the font has loaded. */ diff --git a/ui/src/lib/components/lineage/asset-node.svelte b/ui/src/lib/components/lineage/asset-node.svelte index 3534b6e52a..1cf17e1527 100644 --- a/ui/src/lib/components/lineage/asset-node.svelte +++ b/ui/src/lib/components/lineage/asset-node.svelte @@ -33,6 +33,8 @@ status: 'ok' | 'fail' | 'warn' | 'run' | 'src' | 'none'; /** The main stale reason, if any. */ stale?: 'changed' | 'compiled' | 'upstream' | 'behind'; + /** The node is in an active run, so its status is that run's, not its last build's. */ + live?: boolean; checks?: string; checksColor?: string; dimmed?: boolean; @@ -93,7 +95,7 @@ // Zoomed out, the card's state is its colour: a failure or a run first, then staleness, then the outcome. const faceTone = $derived( - d.status === 'fail' || d.status === 'run' ? tone : d.stale === 'behind' ? 'var(--sb-behind)' : d.stale ? 'var(--sb-stale)' : tone + d.live || d.status === 'fail' || d.status === 'run' ? tone : d.stale === 'behind' ? 'var(--sb-behind)' : d.stale ? 'var(--sb-stale)' : tone ); // The status line stays one line: worked out once per card, the drift chip and then the status word // shorten when its parts don't fit; the time never does. diff --git a/ui/src/lib/lineage/catalog-graph.test.ts b/ui/src/lib/lineage/catalog-graph.test.ts index a8231aac9c..ed9fb0b97e 100644 --- a/ui/src/lib/lineage/catalog-graph.test.ts +++ b/ui/src/lib/lineage/catalog-graph.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { CatalogNode } from '$lib/api'; -import { folderOf } from './catalog-graph'; +import { cardData, folderOf } from './catalog-graph'; function node(key: string, path: string, upstream: string[] = [], extra: Partial = {}): CatalogNode { const kind = key.startsWith('source:') ? 'source' : key.startsWith('seed:') ? 'seed' : 'table'; @@ -18,3 +18,12 @@ describe('lineage folders', () => { ]).toEqual(['marts/core', 'sources/shop', 'seeds', 'models', 'shop']); }); }); + +describe('a card while its job runs', () => { + it('shows the node’s state in the active run, which outranks its last build and whether it was stale', () => { + const stale = node('seed:order_types', 'seeds/order_types.csv', [], { lastOutcome: 'success', lastBuild: { builtAt: '2026-11-01T00:00:00Z' } as CatalogNode['lastBuild'], stale: [{ reason: 'behind' } as never] }); + expect(cardData(stale)).toMatchObject({ status: 'ok', stale: 'behind', live: false }); + expect(cardData({ ...stale, liveStatus: 'run' })).toMatchObject({ status: 'run', stale: undefined, live: true }); + expect(cardData({ ...stale, liveStatus: 'fail' })).toMatchObject({ status: 'fail', live: true }); + }); +}); diff --git a/ui/src/lib/lineage/catalog-graph.ts b/ui/src/lib/lineage/catalog-graph.ts index de3912be9b..3f9b653d2c 100644 --- a/ui/src/lib/lineage/catalog-graph.ts +++ b/ui/src/lib/lineage/catalog-graph.ts @@ -39,8 +39,10 @@ export function cardData(node: CatalogNode): Record { kind: kindLine(node), group: node.group, nodeKind: node.kind, - status: cardStatus(node), - stale: cardStale(node.stale ?? []), + status: node.liveStatus ?? cardStatus(node), + // A node the active run is building, or has just built, shows that, not whether it was stale. + stale: node.liveStatus ? undefined : cardStale(node.stale ?? []), + live: !!node.liveStatus, icon: nodeIconKey(node), lastBuiltAt: node.kind === 'source' ? (node.sourceFreshness?.loadedAt ?? undefined) : node.lastBuild?.builtAt, description: node.description || undefined, diff --git a/ui/src/lib/operate/labels.ts b/ui/src/lib/operate/labels.ts index f6822d2230..03869bb5e4 100644 --- a/ui/src/lib/operate/labels.ts +++ b/ui/src/lib/operate/labels.ts @@ -297,3 +297,8 @@ export function clockTime(iso: string): string { export function narrowingText(narrowing: { select: string[]; exclude: string[] }): string { return [narrowing.select.length ? `--select ${narrowing.select.join(' ')}` : '', narrowing.exclude.length ? `--exclude ${narrowing.exclude.join(' ')}` : ''].filter(Boolean).join(' '); } + +/** A node key as a run shows it: `seed:countries` reads `countries`; the kind shows elsewhere, and the full key on hover. */ +export function nodeLabel(key: string): string { + return key.replace(/^[a-z_]+:(?=.)/, ''); +} diff --git a/ui/src/lib/state/app.svelte.ts b/ui/src/lib/state/app.svelte.ts index c0d635057d..2388747f74 100644 --- a/ui/src/lib/state/app.svelte.ts +++ b/ui/src/lib/state/app.svelte.ts @@ -1,3 +1,4 @@ +import { throttled } from './throttle'; import { browser } from '$app/environment'; import { logout } from '$lib/api/http-client'; import { MockApiClient } from '$lib/api/mock/mock-client'; @@ -23,6 +24,8 @@ import { const SCOPE_KEY = 'sb-scope'; /** How often the shell rereads live problems and warehouse sync; locations report every 30 seconds. */ const POLL_MS = 30_000; +/** The least time between reloads of the sidebar's counts while changes stream in. */ +const NAV_COUNTS_MS = 10_000; /** * App-wide state: the bootstrap, the selected location, and the @@ -85,7 +88,7 @@ class AppState { if (channelMatches(signal.channel, ['platform_events'])) void this.#loadPlatformEvents(); if (channelMatches(signal.channel, ['warehouse_sync'])) void this.#loadWarehouseSync(); if (channelMatches(signal.channel, ['snapshot'])) void this.#loadSnapshot(); - if (channelMatches(signal.channel, ['runs', 'triggers', 'checks'])) void this.#loadNavCounts(); + if (channelMatches(signal.channel, ['runs', 'triggers', 'checks'])) this.#navCounts.request(); }); try { this.bootstrap = await this.client.bootstrap(); @@ -142,6 +145,12 @@ class AppState { this.locationId = location.id; } + /** + * The sidebar's counts read whole listings (every check's result, every run), so a running run's + * stream of changes reloads them at most every `NAV_COUNTS_MS`, and once more after the last change. + */ + #navCounts = throttled(() => this.#loadNavCounts(), NAV_COUNTS_MS); + #loadScoped() { void this.#loadNavCounts(); void this.#loadWarehouseSync(); diff --git a/ui/src/lib/state/resource.svelte.ts b/ui/src/lib/state/resource.svelte.ts index e79faabe04..365d5322f0 100644 --- a/ui/src/lib/state/resource.svelte.ts +++ b/ui/src/lib/state/resource.svelte.ts @@ -42,7 +42,9 @@ export type Resource = { */ export function createResource( load: (client: ApiClient, scope: Scope) => Promise, - topics: readonly string[] = [] + topics: readonly string[] = [], + /** `every`: the least time between reloads a stream of change signals starts, for a large read. */ + options: { every?: number } = {} ): Resource { let data = $state(undefined); let error = $state(null); @@ -52,9 +54,11 @@ export function createResource( let settled = 0; let reloadAgain = false; let pause: ReturnType | null = null; + let started = -Infinity; function run(scope: Scope) { const request = ++latest; + started = Date.now(); // The page's path and query tell apart reads of different jobs, runs and filters made by the same code. const key = `${app.user?.name ?? ''}|${scope.locationId}|${browser ? window.location.pathname + window.location.search : ''}|${load}`; if (key !== shownKey && remembered.has(key)) data = remembered.get(key) as T; @@ -92,11 +96,12 @@ export function createResource( function signalled() { if (pause) return; + const wait = Math.max(SIGNAL_PAUSE_MS, started + (options.every ?? 0) - Date.now()); pause = setTimeout(() => { pause = null; if (settled === latest) refresh(); else reloadAgain = true; - }, SIGNAL_PAUSE_MS); + }, wait); } function refresh() { diff --git a/ui/src/lib/state/throttle.test.ts b/ui/src/lib/state/throttle.test.ts new file mode 100644 index 0000000000..66d1b29460 --- /dev/null +++ b/ui/src/lib/state/throttle.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; +import { throttled, type Timers } from './throttle'; + +/** Lets a finished load's handlers run. */ +const settle = () => new Promise((done) => setImmediate(done)); + +/** Timers driven by hand: `advance` moves the clock and fires what is due. */ +function fakeTimers() { + let now = 0; + let queue: { at: number; run: () => void; id: number }[] = []; + let next = 0; + const timers: Timers = { + now: () => now, + set: (run, ms) => (queue.push({ at: now + ms, run, id: ++next }), next), + clear: (id) => (queue = queue.filter((t) => t.id !== id)) + }; + const advance = async (ms: number) => { + const until = now + ms; + for (;;) { + queue.sort((a, b) => a.at - b.at); + const due = queue[0]; + if (!due || due.at > until) break; + queue.shift(); + now = due.at; + due.run(); + await settle(); + } + now = until; + }; + return { timers, advance }; +} + +describe('a throttled reload', () => { + it('starts at most once per interval under a stream of signals, and once more after the last', async () => { + const { timers, advance } = fakeTimers(); + const starts: number[] = []; + const reload = throttled(async () => void starts.push(timers.now()), 10_000, timers); + // A signal every second for 25 seconds. + for (let s = 0; s < 25; s++) { + reload.request(); + await advance(1000); + } + await advance(20_000); + expect(starts).toEqual([0, 10_000, 20_000, 30_000]); + }); + + it('never runs two loads at once; signals during a load ask for one more', async () => { + const { timers, advance } = fakeTimers(); + let running = 0; + let most = 0; + let finish: () => void = () => {}; + const reload = throttled( + () => { + running++; + most = Math.max(most, running); + return new Promise((done) => (finish = () => (running--, done()))); + }, + 0, + timers + ); + reload.request(); + reload.request(); + reload.request(); + expect(running).toBe(1); + finish(); + await settle(); + expect(running).toBe(1); + finish(); + await settle(); + expect(running).toBe(0); + expect(most).toBe(1); + }); +}); diff --git a/ui/src/lib/state/throttle.ts b/ui/src/lib/state/throttle.ts new file mode 100644 index 0000000000..c7787e60b6 --- /dev/null +++ b/ui/src/lib/state/throttle.ts @@ -0,0 +1,49 @@ +// A reload that a stream of change signals can ask for as often as it likes, but that starts at most +// once every `minMs`, never twice at a time, and always once more after the last signal. Pure apart +// from the timers it is given. + +export type Timers = { now: () => number; set: (run: () => void, ms: number) => unknown; clear: (handle: unknown) => void }; +const realTimers: Timers = { now: () => Date.now(), set: (run, ms) => setTimeout(run, ms), clear: (h) => clearTimeout(h as ReturnType) }; + +export type Throttled = { request: () => void; cancel: () => void }; + +/** + * `load` run on `request()`: at once when idle and at least `minMs` after the last start, else once + * later. Requests while a load runs, or while one waits, ask for just one more. + */ +export function throttled(load: () => Promise, minMs: number, timers: Timers = realTimers): Throttled { + let lastStart = -Infinity; + let running = false; + let again = false; + let waiting: unknown = null; + const start = () => { + waiting = null; + running = true; + again = false; + lastStart = timers.now(); + void load() + .catch(() => {}) + .finally(() => { + running = false; + if (again) request(); + }); + }; + function request() { + if (running) { + again = true; + return; + } + if (waiting !== null) return; + const wait = lastStart + minMs - timers.now(); + if (wait <= 0) start(); + else waiting = timers.set(start, wait); + } + return { + request, + cancel() { + if (waiting !== null) timers.clear(waiting); + waiting = null; + again = false; + } + }; +} diff --git a/ui/src/routes/jobs/[name]/job-view.svelte b/ui/src/routes/jobs/[name]/job-view.svelte index bfea164275..1a3a9ee017 100644 --- a/ui/src/routes/jobs/[name]/job-view.svelte +++ b/ui/src/routes/jobs/[name]/job-view.svelte @@ -12,7 +12,7 @@ import { createDetailsPanel } from '$lib/state/details-panel.svelte'; import { openRow, to } from '$lib/paths'; import { goto } from '$app/navigation'; - import type { Trigger } from '$lib/api'; + import type { CatalogNode, RunNode, Trigger } from '$lib/api'; import AppTopbar from '$lib/components/app-topbar.svelte'; import CodePreview from '$lib/components/code/code-preview.svelte'; import GatedButton from '$lib/components/operate/gated-button.svelte'; @@ -20,7 +20,7 @@ import { createResource } from '$lib/state/resource.svelte'; import { useCatalogNodes } from '$lib/state/catalog.svelte'; import { formatAgo, formatDuration, formatIn, formatStamp } from '$lib/format/time'; - import { formatNodeCounts, nodeOutcomeLabel, nodeOutcomeTone, runOriginLabel, runStatusLabel, runStatusTone } from '$lib/status'; + import { formatNodeCounts, isActiveRun, nodeOutcomeLabel, nodeOutcomeTone, runOriginLabel, runStatusLabel, runStatusTone } from '$lib/status'; import { kindLine, shortName } from '$lib/build-area/labels'; import { planHref } from '$lib/run/command'; import { CONCURRENCY_DOCS, concurrencySource, formatCost, narrowingText, onFailureText, overlapHelp, overlapWhileRunning, poolText, shortRunId } from '$lib/operate/labels'; @@ -83,6 +83,15 @@ const pickedNodes = $derived(picked.map((k) => byKey.get(k)).filter((n) => !!n)); const canRunPicked = $derived(app.client.supports('launchAdhocRun')); const latest = $derived(runs[0] ?? null); + // While the job's latest run is active, its nodes show that run's progress: running, succeeded, failed. + const liveRunId = $derived(latest && isActiveRun(latest.status) ? latest.id : null); + const liveRun = createResource((client, scope) => (liveRunId ? client.getRunDetail(scope, liveRunId) : Promise.resolve(null)), ['runs']); + const LIVE: Partial>> = { running: 'run', success: 'ok', warn: 'warn', failed: 'fail' }; + const liveCatalog = $derived.by(() => { + const states = liveRunId && liveRun.data?.run.id === liveRunId ? new Map(liveRun.data.nodes.flatMap((n) => (LIVE[n.status] ? [[n.nodeKey, LIVE[n.status]!] as const] : []))) : null; + if (!states?.size || !catalog.data) return catalog.data; + return catalog.data.map((n) => (states.has(n.key) ? { ...n, liveStatus: states.get(n.key) } : n)); + }); // Recent finished runs, oldest first, for the sidebar's strip and typical duration. const finished = $derived(runs.filter((r) => r.endedAt).slice(0, 20).reverse()); const succeeded = $derived(finished.filter((r) => r.status === 'success').length); @@ -190,7 +199,7 @@ {:else if catalog.data} - (picked = keys)} /> + (picked = keys)} /> {:else} {#if catalog.error}

{catalog.error}

{:else}
{/if} {/if} diff --git a/ui/src/routes/runs/[id]/log-tab.svelte b/ui/src/routes/runs/[id]/log-tab.svelte index 8f253e95f6..68dd70a8e9 100644 --- a/ui/src/routes/runs/[id]/log-tab.svelte +++ b/ui/src/routes/runs/[id]/log-tab.svelte @@ -10,7 +10,7 @@ import SearchIcon from '@lucide/svelte/icons/search'; import type { RunEventLevel } from '$lib/api'; import { formatStamp } from '$lib/format/time'; - import { clockTime, shortRunId } from '$lib/operate/labels'; + import { clockTime, nodeLabel, shortRunId } from '$lib/operate/labels'; import type { EventCategory } from './types'; import { EVENT_CATEGORIES, type LogEntry } from './utils'; import SegmentedFilter from '$lib/components/list/segmented-filter.svelte'; @@ -111,7 +111,7 @@ {clockTime(e.at)} {#if grouped}{shortRunId(e.runId)}{/if} - {e.nodeKey ?? e.stream ?? 'run'} + {e.nodeKey ? nodeLabel(e.nodeKey) : (e.stream ?? 'run')} {#if e.stream} {@const lines = e.text.split('\n')} diff --git a/ui/src/routes/runs/[id]/node-panel.svelte b/ui/src/routes/runs/[id]/node-panel.svelte index 1f465834e3..c44e27ba3c 100644 --- a/ui/src/routes/runs/[id]/node-panel.svelte +++ b/ui/src/routes/runs/[id]/node-panel.svelte @@ -3,22 +3,26 @@ import Button from '$lib/components/ui/button/button.svelte'; import { to } from '$lib/paths'; import type { RunNode } from '$lib/api'; - import { shortRunId } from '$lib/operate/labels'; + import { nodeLabel, shortRunId } from '$lib/operate/labels'; import type { RunDetailState } from './state.svelte'; import type { NodeGroupKey } from './types'; - import { formatSpan, groupNodes, notRunReason } from './utils'; + import { formatSpan, notRunReason } from './utils'; let { view }: { view: RunDetailState } = $props(); - const groups = $derived(groupNodes(view.nodes)); - // Small runs show every node; large ones start with the successes folded away. + /** The most nodes a run may have for this panel to list them all. */ + const LIST_MAX = 40; + + const groups = $derived(view.nodeGroups); + /** + * A small run lists every node here. A large one's timeline already lists them, so here it is a summary: + * each outcome's count, which narrows the timeline to it, with only the failed and skipped nodes listed, + * as each says why. + */ + const summary = $derived(view.nodes.length > LIST_MAX); + const LISTED_IN_SUMMARY: NodeGroupKey[] = ['failed', 'skipped']; let closed = $state>(new Set()); - let folded = false; - $effect(() => { - if (folded || !view.nodes.length) return; - folded = true; - if (view.nodes.length > 40) closed = new Set(['succeeded']); - }); + const open = (key: NodeGroupKey) => (summary ? LISTED_IN_SUMMARY.includes(key) : !closed.has(key)); const took = (n: RunNode) => (n.startedAt && n.endedAt ? formatSpan((Date.parse(n.endedAt) - Date.parse(n.startedAt)) / 1000) : null); function toggle(key: NodeGroupKey) { @@ -38,17 +42,24 @@
{#each groups as g (g.key)}
- - - {#if !closed.has(g.key)} + {#if summary} + + + {:else} + + + {/if} + {#if open(g.key)}
    {#each g.nodes as n (`${n.runId}/${n.nodeKey}`)}
  • @@ -63,8 +74,8 @@ {/if} failed{#if why.runId !== n.runId} in {why.jobKey ?? shortRunId(why.runId)}{/if}
- {:else if notRunReason(n)} -
{notRunReason(n)}
+ {:else if n.status !== 'pending' && notRunReason(n, view.active)} +
{notRunReason(n, view.active)}
{/if} {/each} @@ -104,6 +115,10 @@ font-weight: 500; text-align: left; } + .group.picked { + background: color-mix(in srgb, var(--primary) 12%, var(--background)); + box-shadow: inset 2px 0 0 var(--primary); + } .group i { width: 8px; height: 8px; diff --git a/ui/src/routes/runs/[id]/run-gantt.svelte b/ui/src/routes/runs/[id]/run-gantt.svelte index 54f6122ff8..8a423ef12a 100644 --- a/ui/src/routes/runs/[id]/run-gantt.svelte +++ b/ui/src/routes/runs/[id]/run-gantt.svelte @@ -7,7 +7,7 @@ import ChevronRightIcon from '@lucide/svelte/icons/chevron-right'; import { onMount } from 'svelte'; import type { RunNode } from '$lib/api'; - import { nodeStatusTone } from '$lib/operate/labels'; + import { nodeLabel, nodeStatusTone } from '$lib/operate/labels'; import { axisTicks as ticksFor, formatSpan, formatTick, tickStep as stepFor } from './utils'; export type GanttGroup = { runId: string; label: string; tone: string; href: string }; @@ -18,6 +18,7 @@ now, selected = [], groups = [], + active = false, onselect }: { nodes: RunNode[]; @@ -27,6 +28,8 @@ selected?: string[]; /** Child runs of a group, to section the rows. */ groups?: GanttGroup[]; + /** The run is still going, so a node that hasn't started is waiting, not left out. */ + active?: boolean; /** `add` is true for Ctrl-, Cmd- or Shift-click. */ onselect: (nodeKey: string, add: boolean) => void; } = $props(); @@ -52,6 +55,19 @@ const trackW = $derived(trackBaseW * zoom); const pxPerSec = $derived(trackW / (total * 1.08)); const xOf = (sec: number) => sec * pxPerSec; + /** A bar's duration reads on its right unless it would run past the row, which ends 24px past the + * track and clips what overflows (`content-visibility`); then it reads on the bar's left if there is + * more room there. */ + let durCtx: CanvasRenderingContext2D | null = null; + function durFlips(text: string, start: number, end: number): boolean { + durCtx ??= document.createElement('canvas').getContext('2d'); + if (!durCtx) return false; + durCtx.font = '11.5px "Inter Variable", sans-serif'; + // Labels set tabular figures, which a canvas can't, so zeros stand in for the digits. + const width = Math.ceil(durCtx.measureText(text.replace(/[0-9]/g, '0')).width) + 1; + const right = trackW + 24 - xOf(end) - 8; + return width > right && xOf(start) - 8 > right; + } const tickStep = $derived(stepFor(total, Math.max(4, Math.floor(trackW / 110)))); const axisTicks = $derived(ticksFor(total, tickStep)); @@ -59,7 +75,7 @@ const ctx = document.createElement('canvas').getContext('2d'); if (!ctx) return 250; ctx.font = '12px "IBM Plex Mono", ui-monospace, monospace'; - const widest = Math.max(0, ...nodes.map((n) => ctx.measureText(n.nodeKey).width)); + const widest = Math.max(0, ...nodes.map((n) => ctx.measureText(nodeLabel(n.nodeKey)).width)); return Math.min(360, Math.max(GUTTER_MIN, Math.ceil(widest) + 60)); } @@ -107,7 +123,7 @@ const across = n.skippedBecauseOf.runId !== n.runId; return `skipped: ${n.skippedBecauseOf.nodeKey} failed${across ? ` in ${n.skippedBecauseOf.jobKey ?? n.skippedBecauseOf.runId}` : ''}`; } - if (n.status === 'pending') return 'Pending'; + if (n.status === 'pending') return active ? 'Waiting' : 'Not run'; if (n.status === 'not_run') return 'Not run'; if (n.status === 'cancelled') return 'Cancelled'; return n.status; @@ -128,7 +144,8 @@
(scrolledX = e.currentTarget.scrollLeft > 1)}> -
+ +
{#each axisTicks as tk (tk)}{/each}
@@ -185,14 +202,15 @@ title={n.nodeKey} > {#if selected.includes(n.nodeKey)}{/if} - {n.nodeKey}{#if n.critical}critical{/if} + {nodeLabel(n.nodeKey)}{#if n.critical}critical{/if} - + {#if s == null || e == null} {skipText(n)} {:else} - - trackW - 56}>{formatSpan(e - s)}{n.status === 'running' ? ' so far' : ''} + + {@const dur = `${formatSpan(e - s)}${n.status === 'running' ? ' so far' : ''}`} + {dur} {/if} @@ -336,11 +354,15 @@ background: var(--primary); content: ''; } + /* Rows out of view are neither laid out nor painted, so scrolling a long run, or zooming it out, only + draws the rows on screen; each row's sticky name cell would otherwise be its own compositor layer. */ .row { position: relative; display: flex; width: 100%; height: 32px; + content-visibility: auto; + contain-intrinsic-size: auto 32px; align-items: center; border-bottom: 1px solid var(--border-subtle); text-align: left; @@ -421,24 +443,40 @@ } .track { position: relative; + width: var(--track-w); height: 100%; } .bar { position: absolute; top: 50%; + left: calc(var(--s) * var(--pps) * 1px); + width: max(3px, calc((var(--e) - var(--s)) * var(--pps) * 1px)); height: 16px; transform: translateY(-50%); border-radius: 2px; box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.18); } + /* A running bar's stripes move by transform, which the compositor animates without repainting. */ .bar.live { - background-image: linear-gradient(90deg, transparent 0 60%, rgba(255, 255, 255, 0.28) 60% 80%, transparent 80%) !important; + overflow: hidden; + } + .bar.live::after { + position: absolute; + inset: 0 0 0 -24px; + background-image: linear-gradient(90deg, transparent 0 60%, rgba(255, 255, 255, 0.28) 60% 80%, transparent 80%); background-size: 24px 100%; animation: slide 1.2s linear infinite; + content: ''; + will-change: transform; } @keyframes slide { to { - background-position: 24px 0; + transform: translateX(24px); + } + } + @media (prefers-reduced-motion: reduce) { + .bar.live::after { + animation: none; } } .dur, @@ -450,7 +488,11 @@ font-size: 11.5px; white-space: nowrap; } + .dur { + left: calc(var(--e) * var(--pps) * 1px + 8px); + } .dur.flip { + left: calc(var(--s) * var(--pps) * 1px - 8px); transform: translate(-100%, -50%); } .note { diff --git a/ui/src/routes/runs/[id]/run-header.svelte b/ui/src/routes/runs/[id]/run-header.svelte index 985466d740..aecf964d93 100644 --- a/ui/src/routes/runs/[id]/run-header.svelte +++ b/ui/src/routes/runs/[id]/run-header.svelte @@ -1,4 +1,5 @@