Skip to content

Commit d932622

Browse files
authored
test: interleaved ratio guard for retained metadata scans; mock runs keep their target (#1485)
* fix(ui): runs launched in the mock carry their location and target * test(server): guard retained scans with an interleaved page ratio instead of a five-second wall clock
1 parent b286141 commit d932622

2 files changed

Lines changed: 72 additions & 36 deletions

File tree

‎crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs‎

Lines changed: 62 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@ use uuid::Uuid;
77
use crate::store::main::{begin::begin, commit::commit, query::query};
88
use crate::store::models::{ControlStore, TransactionMode};
99

10+
/// Records a scan page carries, and the size of the control namespace scanned beside the retained one.
11+
const SCAN_PAGE: usize = 100;
12+
/// The median retained page may cost at most this many times the control page sent right after it.
13+
const MAX_RETAINED_SCAN_RATIO: f64 = 1.8;
14+
/// Only a hang trips this; per-commit disk syncs make wall time vary widely between runners.
15+
const SCAN_HANG_BOUND: std::time::Duration = std::time::Duration::from_secs(60);
16+
1017
pub(crate) async fn reconcile_history(
1118
store: &ControlStore,
1219
(client, url, bearer): (&Client, &str, &str),
@@ -145,52 +152,81 @@ async fn retained_history(
145152
(session_id, fixture, record, (kind, timestamp)): (Uuid, &Value, &Value, (&str, &str)),
146153
) {
147154
let count = fixture["retained_count"].as_u64().expect("retained count");
148-
let records: Vec<_> = (0..count)
149-
.map(|index| {
150-
let mut record = record.clone();
151-
record["event_id"] = json!(format!("retained-{index}"));
152-
record["state_database"] = json!("analytics");
153-
record["state_schema"] = json!("reconcile_retained");
154-
record[timestamp] = json!(format!(
155-
"2026-01-01T10:{:02}:{:02}Z",
156-
index / 120,
157-
(index / 2) % 60
158-
));
159-
record
160-
})
161-
.collect();
162-
for records in records.chunks(100) {
155+
let namespace_records = |schema: &str, count: u64| -> Vec<Value> {
156+
(0..count)
157+
.map(|index| {
158+
let mut record = record.clone();
159+
record["event_id"] = json!(format!("{schema}-{index}"));
160+
record["state_database"] = json!("analytics");
161+
record["state_schema"] = json!(schema);
162+
record[timestamp] = json!(format!(
163+
"2026-01-01T10:{:02}:{:02}Z",
164+
index / 120,
165+
(index / 2) % 60
166+
));
167+
record
168+
})
169+
.collect()
170+
};
171+
let records = namespace_records("reconcile_retained", count);
172+
let control = namespace_records("reconcile_control", SCAN_PAGE as u64);
173+
for records in records.chunks(SCAN_PAGE).chain([control.as_slice()]) {
163174
post(
164175
(client, url, bearer),
165176
&json!({"kind":kind,"session_id":session_id,"generation":1,"records":records}),
166177
StatusCode::OK,
167178
)
168179
.await;
169180
}
170-
let mut page = json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{
171-
"scan_id":Uuid::now_v7(),"location":{"database":"analytics","schema":"reconcile_retained"},"complete":false}});
172-
let pages = records.len().div_ceil(100);
173-
tokio::time::timeout(std::time::Duration::from_secs(5), async {
174-
for (index, records) in records.chunks(100).enumerate() {
175-
page["records"] = json!(records);
176-
page["scan"]["complete"] = json!(index + 1 == pages);
177-
let reply = post((client, url, bearer), &page, StatusCode::OK).await;
181+
let scan_page = |schema: &str| {
182+
json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{
183+
"scan_id":Uuid::now_v7(),"location":{"database":"analytics","schema":schema},"complete":false}})
184+
};
185+
let (mut retained_page, mut control_page) = (
186+
scan_page("reconcile_retained"),
187+
scan_page("reconcile_control"),
188+
);
189+
control_page["records"] = json!(control);
190+
let pages = records.len().div_ceil(SCAN_PAGE);
191+
let mut ratios: Vec<f64> = tokio::time::timeout(SCAN_HANG_BOUND, async {
192+
let mut ratios = Vec::with_capacity(pages);
193+
for (index, records) in records.chunks(SCAN_PAGE).enumerate() {
194+
retained_page["records"] = json!(records);
195+
for page in [&mut retained_page, &mut control_page] {
196+
page["scan"]["complete"] = json!(index + 1 == pages);
197+
}
198+
control_page["scan"]["scan_id"] = json!(Uuid::now_v7());
199+
let started = std::time::Instant::now();
200+
let retained = post((client, url, bearer), &retained_page, StatusCode::OK).await;
201+
let retained_time = started.elapsed();
202+
let started = std::time::Instant::now();
203+
let unchanged = post((client, url, bearer), &control_page, StatusCode::OK).await;
204+
ratios.push(retained_time.as_secs_f64() / started.elapsed().as_secs_f64());
178205
assert_eq!(
179-
reply["applied"], false,
206+
[&retained["applied"], &unchanged["applied"]],
207+
[false, false],
180208
"retained observations never change"
181209
);
182210
}
211+
ratios
183212
})
184213
.await
185-
.expect("a retained-heavy scan has a strict total deadline");
214+
.expect("a retained-heavy scan finishes");
215+
ratios.sort_by(f64::total_cmp);
216+
let median = (ratios[(pages - 1) / 2] + ratios[pages / 2]) / 2.0;
217+
assert!(
218+
median < MAX_RETAINED_SCAN_RATIO,
219+
"a scan page's cost must not grow with the namespace's history: over {count} retained \
220+
rows a page took {median:.2} times a page over {SCAN_PAGE}"
221+
);
186222
let (transaction, rows) = query(
187223
begin(store, TransactionMode::Read).await.expect("read retained"),
188224
&format!("SELECT record_json ->> 'event_id' AS source_event_id FROM {table} WHERE state_database='analytics' AND state_schema='reconcile_retained' ORDER BY source_event_id"),
189225
vec![],
190226
).await.expect("retained identities");
191227
commit(transaction).await.expect("close retained read");
192228
let mut expected: Vec<_> = (0..count)
193-
.map(|index| format!("retained-{index}"))
229+
.map(|index| format!("reconcile_retained-{index}"))
194230
.collect();
195231
expected.sort();
196232
assert_eq!(

‎ui/src/lib/api/mock/operate-store.ts‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -564,18 +564,18 @@ export class MockOperate {
564564
if (job?.kind === 'group') {
565565
const members = this.#children(source).filter((c) => mode === 'all' || affected(c).length);
566566
if (!members.length) throw new ApiError('Nothing to rerun: no node failed or was skipped', 409);
567-
const parent = this.#create({ id, origin: 'rerun', jobKey: job.key, rerunOf: runId, by: user, select: [], exclude: [], flags: {}, nodes: [], priority: job.priority });
567+
const parent = this.#create(scope, { id, origin: 'rerun', jobKey: job.key, rerunOf: runId, by: user, select: [], exclude: [], flags: {}, nodes: [], priority: job.priority });
568568
members.forEach((c, i) => {
569569
const nodes = mode === 'all' ? c.script.nodes : affected(c);
570-
this.#create({ id: `${id}-${i + 1}`, origin: 'rerun', jobKey: c.run.jobKey, parent: id, rerunOf: c.run.id, by: user, select: mode === 'all' ? c.script.select : nodes, exclude: mode === 'all' ? c.script.exclude : [], flags: c.script.flags, nodes, priority: job.priority });
570+
this.#create(scope, { id: `${id}-${i + 1}`, origin: 'rerun', jobKey: c.run.jobKey, parent: id, rerunOf: c.run.id, by: user, select: mode === 'all' ? c.script.select : nodes, exclude: mode === 'all' ? c.script.exclude : [], flags: c.script.flags, nodes, priority: job.priority });
571571
});
572572
parent.run.childRunCount = members.length;
573573
this.#ctx.changes.emit('runs');
574574
return this.#ctx.reply(this.#view(parent, scope));
575575
}
576576
const nodes = mode === 'all' ? source.script.nodes : affected(source);
577577
if (!nodes.length) throw new ApiError('Nothing to rerun: no node failed or was skipped', 409);
578-
const rec = this.#create({
578+
const rec = this.#create(scope, {
579579
id,
580580
origin: 'rerun',
581581
jobKey: source.run.jobKey,
@@ -609,10 +609,10 @@ export class MockOperate {
609609
const id = this.#newId();
610610
if (job.kind === 'group') {
611611
if (options.select?.length || options.exclude?.length) throw new ApiError('invalid control-plane data: a job group cannot be narrowed; narrow one of its jobs', 400);
612-
const parent = this.#create({ id, origin: 'manual', jobKey: key, by: user, select: [], exclude: [], flags, nodes: [], priority: job.priority });
612+
const parent = this.#create(scope, { id, origin: 'manual', jobKey: key, by: user, select: [], exclude: [], flags, nodes: [], priority: job.priority });
613613
job.members.forEach((m, i) => {
614614
const member = jobDef(m)!;
615-
this.#create({ id: `${id}-${i + 1}`, origin: 'manual', jobKey: m, parent: id, by: user, select: selectorTerms(member), exclude: member.exclude, flags: { ...flags, ...Object.fromEntries(Object.entries(member.settings).map(([k, v]) => [k, v === true ? true : String(v)])) }, nodes: jobNodes(m), priority: job.priority });
615+
this.#create(scope, { id: `${id}-${i + 1}`, origin: 'manual', jobKey: m, parent: id, by: user, select: selectorTerms(member), exclude: member.exclude, flags: { ...flags, ...Object.fromEntries(Object.entries(member.settings).map(([k, v]) => [k, v === true ? true : String(v)])) }, nodes: jobNodes(m), priority: job.priority });
616616
});
617617
parent.run.childRunCount = job.members.length;
618618
this.#ctx.changes.emit('runs');
@@ -624,7 +624,7 @@ export class MockOperate {
624624
const nodes = jobNodes(key).filter((n) => (!options.select?.length || named(options.select, n)) && !named(options.exclude, n));
625625
if (narrowed && !nodes.length) throw new ApiError("invalid control-plane data: the selectors leave none of the job's nodes", 400);
626626
const narrowing = narrowed ? { select: options.select ?? [], exclude: options.exclude ?? [] } : null;
627-
const rec = this.#create({ id, origin: 'manual', jobKey: key, narrowing, by: user, select: selectorTerms(job), exclude: job.exclude, flags, nodes, priority: job.priority });
627+
const rec = this.#create(scope, { id, origin: 'manual', jobKey: key, narrowing, by: user, select: selectorTerms(job), exclude: job.exclude, flags, nodes, priority: job.priority });
628628
this.#ctx.changes.emit('runs');
629629
return this.#ctx.reply(this.#view(rec, scope));
630630
}
@@ -639,7 +639,7 @@ export class MockOperate {
639639
const denied = check.permissions.find((p) => !p.allowed);
640640
if (denied) throw new ApiError(denied.reason ?? `missing ${denied.action} permission for this target`, 403);
641641
const parts = parseCommand(request.command.join(' '));
642-
const rec = this.#create({
642+
const rec = this.#create(scope, {
643643
id: this.#newId(),
644644
origin: 'manual',
645645
jobKey: null,
@@ -1016,7 +1016,7 @@ export class MockOperate {
10161016
return id;
10171017
}
10181018

1019-
#create(spec: {
1019+
#create(scope: Scope, spec: {
10201020
id: string;
10211021
origin: Run['origin'];
10221022
jobKey: string | null;
@@ -1055,8 +1055,8 @@ export class MockOperate {
10551055
requestedBy: spec.by,
10561056
priority: spec.priority,
10571057
rewriteHistory: Object.keys(spec.flags).some((f) => flagSpec(spec.verb ?? 'build', f)?.requires === 'run.rewrite_history'),
1058-
locationId: '',
1059-
target: '',
1058+
locationId: scope.locationId,
1059+
target: this.#ctx.target(scope),
10601060
snapshotId: 'snap-7d2e41a',
10611061
gitRevision: '7d2e41a',
10621062
queuedAt: now,

0 commit comments

Comments
 (0)