From 2cbd4e5cc134f0d09545a73c069c5f49263425a5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:06:20 +0000 Subject: [PATCH 1/2] fix(ui): runs launched in the mock carry their location and target --- ui/src/lib/api/mock/operate-store.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index aacde0be9f..0ce23ab7da 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -564,10 +564,10 @@ export class MockOperate { if (job?.kind === 'group') { const members = this.#children(source).filter((c) => mode === 'all' || affected(c).length); if (!members.length) throw new ApiError('Nothing to rerun: no node failed or was skipped', 409); - const parent = this.#create({ id, origin: 'rerun', jobKey: job.key, rerunOf: runId, by: user, select: [], exclude: [], flags: {}, nodes: [], priority: job.priority }); + const parent = this.#create(scope, { id, origin: 'rerun', jobKey: job.key, rerunOf: runId, by: user, select: [], exclude: [], flags: {}, nodes: [], priority: job.priority }); members.forEach((c, i) => { const nodes = mode === 'all' ? c.script.nodes : affected(c); - this.#create({ id: `${id}-${i + 1}`, origin: 'rerun', jobKey: c.run.jobKey, parent: id, rerunOf: c.run.id, by: user, select: mode === 'all' ? c.script.select : nodes, exclude: mode === 'all' ? c.script.exclude : [], flags: c.script.flags, nodes, priority: job.priority }); + this.#create(scope, { id: `${id}-${i + 1}`, origin: 'rerun', jobKey: c.run.jobKey, parent: id, rerunOf: c.run.id, by: user, select: mode === 'all' ? c.script.select : nodes, exclude: mode === 'all' ? c.script.exclude : [], flags: c.script.flags, nodes, priority: job.priority }); }); parent.run.childRunCount = members.length; this.#ctx.changes.emit('runs'); @@ -575,7 +575,7 @@ export class MockOperate { } const nodes = mode === 'all' ? source.script.nodes : affected(source); if (!nodes.length) throw new ApiError('Nothing to rerun: no node failed or was skipped', 409); - const rec = this.#create({ + const rec = this.#create(scope, { id, origin: 'rerun', jobKey: source.run.jobKey, @@ -609,10 +609,10 @@ export class MockOperate { const id = this.#newId(); if (job.kind === 'group') { if (options.select?.length || options.exclude?.length) throw new ApiError('invalid control-plane data: a job group cannot be narrowed; narrow one of its jobs', 400); - const parent = this.#create({ id, origin: 'manual', jobKey: key, by: user, select: [], exclude: [], flags, nodes: [], priority: job.priority }); + const parent = this.#create(scope, { id, origin: 'manual', jobKey: key, by: user, select: [], exclude: [], flags, nodes: [], priority: job.priority }); job.members.forEach((m, i) => { const member = jobDef(m)!; - this.#create({ id: `${id}-${i + 1}`, origin: 'manual', jobKey: m, parent: id, by: user, select: selectorTerms(member), exclude: member.exclude, flags: { ...flags, ...Object.fromEntries(Object.entries(member.settings).map(([k, v]) => [k, v === true ? true : String(v)])) }, nodes: jobNodes(m), priority: job.priority }); + this.#create(scope, { id: `${id}-${i + 1}`, origin: 'manual', jobKey: m, parent: id, by: user, select: selectorTerms(member), exclude: member.exclude, flags: { ...flags, ...Object.fromEntries(Object.entries(member.settings).map(([k, v]) => [k, v === true ? true : String(v)])) }, nodes: jobNodes(m), priority: job.priority }); }); parent.run.childRunCount = job.members.length; this.#ctx.changes.emit('runs'); @@ -624,7 +624,7 @@ export class MockOperate { const nodes = jobNodes(key).filter((n) => (!options.select?.length || named(options.select, n)) && !named(options.exclude, n)); if (narrowed && !nodes.length) throw new ApiError("invalid control-plane data: the selectors leave none of the job's nodes", 400); const narrowing = narrowed ? { select: options.select ?? [], exclude: options.exclude ?? [] } : null; - const rec = this.#create({ id, origin: 'manual', jobKey: key, narrowing, by: user, select: selectorTerms(job), exclude: job.exclude, flags, nodes, priority: job.priority }); + const rec = this.#create(scope, { id, origin: 'manual', jobKey: key, narrowing, by: user, select: selectorTerms(job), exclude: job.exclude, flags, nodes, priority: job.priority }); this.#ctx.changes.emit('runs'); return this.#ctx.reply(this.#view(rec, scope)); } @@ -639,7 +639,7 @@ export class MockOperate { const denied = check.permissions.find((p) => !p.allowed); if (denied) throw new ApiError(denied.reason ?? `missing ${denied.action} permission for this target`, 403); const parts = parseCommand(request.command.join(' ')); - const rec = this.#create({ + const rec = this.#create(scope, { id: this.#newId(), origin: 'manual', jobKey: null, @@ -1016,7 +1016,7 @@ export class MockOperate { return id; } - #create(spec: { + #create(scope: Scope, spec: { id: string; origin: Run['origin']; jobKey: string | null; @@ -1055,8 +1055,8 @@ export class MockOperate { requestedBy: spec.by, priority: spec.priority, rewriteHistory: Object.keys(spec.flags).some((f) => flagSpec(spec.verb ?? 'build', f)?.requires === 'run.rewrite_history'), - locationId: '', - target: '', + locationId: scope.locationId, + target: this.#ctx.target(scope), snapshotId: 'snap-7d2e41a', gitRevision: '7d2e41a', queuedAt: now, From c8cfe98a9083b505a6de80962caa7bc5b190fd04 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:23:27 +0000 Subject: [PATCH 2/2] test(server): guard retained scans with an interleaved page ratio instead of a five-second wall clock --- .../reconciliation/helpers.rs | 88 +++++++++++++------ 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs index ac0f60fce2..2e3458d622 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs @@ -7,6 +7,13 @@ use uuid::Uuid; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, TransactionMode}; +/// Records a scan page carries, and the size of the control namespace scanned beside the retained one. +const SCAN_PAGE: usize = 100; +/// The median retained page may cost at most this many times the control page sent right after it. +const MAX_RETAINED_SCAN_RATIO: f64 = 1.8; +/// Only a hang trips this; per-commit disk syncs make wall time vary widely between runners. +const SCAN_HANG_BOUND: std::time::Duration = std::time::Duration::from_secs(60); + pub(crate) async fn reconcile_history( store: &ControlStore, (client, url, bearer): (&Client, &str, &str), @@ -145,21 +152,25 @@ async fn retained_history( (session_id, fixture, record, (kind, timestamp)): (Uuid, &Value, &Value, (&str, &str)), ) { let count = fixture["retained_count"].as_u64().expect("retained count"); - let records: Vec<_> = (0..count) - .map(|index| { - let mut record = record.clone(); - record["event_id"] = json!(format!("retained-{index}")); - record["state_database"] = json!("analytics"); - record["state_schema"] = json!("reconcile_retained"); - record[timestamp] = json!(format!( - "2026-01-01T10:{:02}:{:02}Z", - index / 120, - (index / 2) % 60 - )); - record - }) - .collect(); - for records in records.chunks(100) { + let namespace_records = |schema: &str, count: u64| -> Vec { + (0..count) + .map(|index| { + let mut record = record.clone(); + record["event_id"] = json!(format!("{schema}-{index}")); + record["state_database"] = json!("analytics"); + record["state_schema"] = json!(schema); + record[timestamp] = json!(format!( + "2026-01-01T10:{:02}:{:02}Z", + index / 120, + (index / 2) % 60 + )); + record + }) + .collect() + }; + let records = namespace_records("reconcile_retained", count); + let control = namespace_records("reconcile_control", SCAN_PAGE as u64); + for records in records.chunks(SCAN_PAGE).chain([control.as_slice()]) { post( (client, url, bearer), &json!({"kind":kind,"session_id":session_id,"generation":1,"records":records}), @@ -167,22 +178,47 @@ async fn retained_history( ) .await; } - let mut page = json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{ - "scan_id":Uuid::now_v7(),"location":{"database":"analytics","schema":"reconcile_retained"},"complete":false}}); - let pages = records.len().div_ceil(100); - tokio::time::timeout(std::time::Duration::from_secs(5), async { - for (index, records) in records.chunks(100).enumerate() { - page["records"] = json!(records); - page["scan"]["complete"] = json!(index + 1 == pages); - let reply = post((client, url, bearer), &page, StatusCode::OK).await; + let scan_page = |schema: &str| { + json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{ + "scan_id":Uuid::now_v7(),"location":{"database":"analytics","schema":schema},"complete":false}}) + }; + let (mut retained_page, mut control_page) = ( + scan_page("reconcile_retained"), + scan_page("reconcile_control"), + ); + control_page["records"] = json!(control); + let pages = records.len().div_ceil(SCAN_PAGE); + let mut ratios: Vec = tokio::time::timeout(SCAN_HANG_BOUND, async { + let mut ratios = Vec::with_capacity(pages); + for (index, records) in records.chunks(SCAN_PAGE).enumerate() { + retained_page["records"] = json!(records); + for page in [&mut retained_page, &mut control_page] { + page["scan"]["complete"] = json!(index + 1 == pages); + } + control_page["scan"]["scan_id"] = json!(Uuid::now_v7()); + let started = std::time::Instant::now(); + let retained = post((client, url, bearer), &retained_page, StatusCode::OK).await; + let retained_time = started.elapsed(); + let started = std::time::Instant::now(); + let unchanged = post((client, url, bearer), &control_page, StatusCode::OK).await; + ratios.push(retained_time.as_secs_f64() / started.elapsed().as_secs_f64()); assert_eq!( - reply["applied"], false, + [&retained["applied"], &unchanged["applied"]], + [false, false], "retained observations never change" ); } + ratios }) .await - .expect("a retained-heavy scan has a strict total deadline"); + .expect("a retained-heavy scan finishes"); + ratios.sort_by(f64::total_cmp); + let median = (ratios[(pages - 1) / 2] + ratios[pages / 2]) / 2.0; + assert!( + median < MAX_RETAINED_SCAN_RATIO, + "a scan page's cost must not grow with the namespace's history: over {count} retained \ + rows a page took {median:.2} times a page over {SCAN_PAGE}" + ); let (transaction, rows) = query( begin(store, TransactionMode::Read).await.expect("read retained"), &format!("SELECT record_json ->> 'event_id' AS source_event_id FROM {table} WHERE state_database='analytics' AND state_schema='reconcile_retained' ORDER BY source_event_id"), @@ -190,7 +226,7 @@ async fn retained_history( ).await.expect("retained identities"); commit(transaction).await.expect("close retained read"); let mut expected: Vec<_> = (0..count) - .map(|index| format!("retained-{index}")) + .map(|index| format!("reconcile_retained-{index}")) .collect(); expected.sort(); assert_eq!(