Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@ use uuid::Uuid;
use crate::store::main::{begin::begin, commit::commit, query::query};
use crate::store::models::{ControlStore, TransactionMode};

/// Records a scan page carries, and the size of the control namespace scanned beside the retained one.
const SCAN_PAGE: usize = 100;
/// The median retained page may cost at most this many times the control page sent right after it.
const MAX_RETAINED_SCAN_RATIO: f64 = 1.8;
/// Only a hang trips this; per-commit disk syncs make wall time vary widely between runners.
const SCAN_HANG_BOUND: std::time::Duration = std::time::Duration::from_secs(60);

pub(crate) async fn reconcile_history(
store: &ControlStore,
(client, url, bearer): (&Client, &str, &str),
Expand Down Expand Up @@ -145,52 +152,81 @@ async fn retained_history(
(session_id, fixture, record, (kind, timestamp)): (Uuid, &Value, &Value, (&str, &str)),
) {
let count = fixture["retained_count"].as_u64().expect("retained count");
let records: Vec<_> = (0..count)
.map(|index| {
let mut record = record.clone();
record["event_id"] = json!(format!("retained-{index}"));
record["state_database"] = json!("analytics");
record["state_schema"] = json!("reconcile_retained");
record[timestamp] = json!(format!(
"2026-01-01T10:{:02}:{:02}Z",
index / 120,
(index / 2) % 60
));
record
})
.collect();
for records in records.chunks(100) {
let namespace_records = |schema: &str, count: u64| -> Vec<Value> {
(0..count)
.map(|index| {
let mut record = record.clone();
record["event_id"] = json!(format!("{schema}-{index}"));
record["state_database"] = json!("analytics");
record["state_schema"] = json!(schema);
record[timestamp] = json!(format!(
"2026-01-01T10:{:02}:{:02}Z",
index / 120,
(index / 2) % 60
));
record
})
.collect()
};
let records = namespace_records("reconcile_retained", count);
let control = namespace_records("reconcile_control", SCAN_PAGE as u64);
for records in records.chunks(SCAN_PAGE).chain([control.as_slice()]) {
post(
(client, url, bearer),
&json!({"kind":kind,"session_id":session_id,"generation":1,"records":records}),
StatusCode::OK,
)
.await;
}
let mut page = json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{
"scan_id":Uuid::now_v7(),"location":{"database":"analytics","schema":"reconcile_retained"},"complete":false}});
let pages = records.len().div_ceil(100);
tokio::time::timeout(std::time::Duration::from_secs(5), async {
for (index, records) in records.chunks(100).enumerate() {
page["records"] = json!(records);
page["scan"]["complete"] = json!(index + 1 == pages);
let reply = post((client, url, bearer), &page, StatusCode::OK).await;
let scan_page = |schema: &str| {
json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{
"scan_id":Uuid::now_v7(),"location":{"database":"analytics","schema":schema},"complete":false}})
};
let (mut retained_page, mut control_page) = (
scan_page("reconcile_retained"),
scan_page("reconcile_control"),
);
control_page["records"] = json!(control);
let pages = records.len().div_ceil(SCAN_PAGE);
let mut ratios: Vec<f64> = tokio::time::timeout(SCAN_HANG_BOUND, async {
let mut ratios = Vec::with_capacity(pages);
for (index, records) in records.chunks(SCAN_PAGE).enumerate() {
retained_page["records"] = json!(records);
for page in [&mut retained_page, &mut control_page] {
page["scan"]["complete"] = json!(index + 1 == pages);
}
control_page["scan"]["scan_id"] = json!(Uuid::now_v7());
let started = std::time::Instant::now();
let retained = post((client, url, bearer), &retained_page, StatusCode::OK).await;
let retained_time = started.elapsed();
let started = std::time::Instant::now();
let unchanged = post((client, url, bearer), &control_page, StatusCode::OK).await;
ratios.push(retained_time.as_secs_f64() / started.elapsed().as_secs_f64());
assert_eq!(
reply["applied"], false,
[&retained["applied"], &unchanged["applied"]],
[false, false],
"retained observations never change"
);
}
ratios
})
.await
.expect("a retained-heavy scan has a strict total deadline");
.expect("a retained-heavy scan finishes");
ratios.sort_by(f64::total_cmp);
let median = (ratios[(pages - 1) / 2] + ratios[pages / 2]) / 2.0;
assert!(
median < MAX_RETAINED_SCAN_RATIO,
"a scan page's cost must not grow with the namespace's history: over {count} retained \
rows a page took {median:.2} times a page over {SCAN_PAGE}"
);
let (transaction, rows) = query(
begin(store, TransactionMode::Read).await.expect("read retained"),
&format!("SELECT record_json ->> 'event_id' AS source_event_id FROM {table} WHERE state_database='analytics' AND state_schema='reconcile_retained' ORDER BY source_event_id"),
vec![],
).await.expect("retained identities");
commit(transaction).await.expect("close retained read");
let mut expected: Vec<_> = (0..count)
.map(|index| format!("retained-{index}"))
.map(|index| format!("reconcile_retained-{index}"))
.collect();
expected.sort();
assert_eq!(
Expand Down
20 changes: 10 additions & 10 deletions ui/src/lib/api/mock/operate-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -564,18 +564,18 @@ export class MockOperate {
if (job?.kind === 'group') {
const members = this.#children(source).filter((c) => mode === 'all' || affected(c).length);
if (!members.length) throw new ApiError('Nothing to rerun: no node failed or was skipped', 409);
const parent = this.#create({ id, origin: 'rerun', jobKey: job.key, rerunOf: runId, by: user, select: [], exclude: [], flags: {}, nodes: [], priority: job.priority });
const parent = this.#create(scope, { id, origin: 'rerun', jobKey: job.key, rerunOf: runId, by: user, select: [], exclude: [], flags: {}, nodes: [], priority: job.priority });
members.forEach((c, i) => {
const nodes = mode === 'all' ? c.script.nodes : affected(c);
this.#create({ id: `${id}-${i + 1}`, origin: 'rerun', jobKey: c.run.jobKey, parent: id, rerunOf: c.run.id, by: user, select: mode === 'all' ? c.script.select : nodes, exclude: mode === 'all' ? c.script.exclude : [], flags: c.script.flags, nodes, priority: job.priority });
this.#create(scope, { id: `${id}-${i + 1}`, origin: 'rerun', jobKey: c.run.jobKey, parent: id, rerunOf: c.run.id, by: user, select: mode === 'all' ? c.script.select : nodes, exclude: mode === 'all' ? c.script.exclude : [], flags: c.script.flags, nodes, priority: job.priority });
});
parent.run.childRunCount = members.length;
this.#ctx.changes.emit('runs');
return this.#ctx.reply(this.#view(parent, scope));
}
const nodes = mode === 'all' ? source.script.nodes : affected(source);
if (!nodes.length) throw new ApiError('Nothing to rerun: no node failed or was skipped', 409);
const rec = this.#create({
const rec = this.#create(scope, {
id,
origin: 'rerun',
jobKey: source.run.jobKey,
Expand Down Expand Up @@ -609,10 +609,10 @@ export class MockOperate {
const id = this.#newId();
if (job.kind === 'group') {
if (options.select?.length || options.exclude?.length) throw new ApiError('invalid control-plane data: a job group cannot be narrowed; narrow one of its jobs', 400);
const parent = this.#create({ id, origin: 'manual', jobKey: key, by: user, select: [], exclude: [], flags, nodes: [], priority: job.priority });
const parent = this.#create(scope, { id, origin: 'manual', jobKey: key, by: user, select: [], exclude: [], flags, nodes: [], priority: job.priority });
job.members.forEach((m, i) => {
const member = jobDef(m)!;
this.#create({ id: `${id}-${i + 1}`, origin: 'manual', jobKey: m, parent: id, by: user, select: selectorTerms(member), exclude: member.exclude, flags: { ...flags, ...Object.fromEntries(Object.entries(member.settings).map(([k, v]) => [k, v === true ? true : String(v)])) }, nodes: jobNodes(m), priority: job.priority });
this.#create(scope, { id: `${id}-${i + 1}`, origin: 'manual', jobKey: m, parent: id, by: user, select: selectorTerms(member), exclude: member.exclude, flags: { ...flags, ...Object.fromEntries(Object.entries(member.settings).map(([k, v]) => [k, v === true ? true : String(v)])) }, nodes: jobNodes(m), priority: job.priority });
});
parent.run.childRunCount = job.members.length;
this.#ctx.changes.emit('runs');
Expand All @@ -624,7 +624,7 @@ export class MockOperate {
const nodes = jobNodes(key).filter((n) => (!options.select?.length || named(options.select, n)) && !named(options.exclude, n));
if (narrowed && !nodes.length) throw new ApiError("invalid control-plane data: the selectors leave none of the job's nodes", 400);
const narrowing = narrowed ? { select: options.select ?? [], exclude: options.exclude ?? [] } : null;
const rec = this.#create({ id, origin: 'manual', jobKey: key, narrowing, by: user, select: selectorTerms(job), exclude: job.exclude, flags, nodes, priority: job.priority });
const rec = this.#create(scope, { id, origin: 'manual', jobKey: key, narrowing, by: user, select: selectorTerms(job), exclude: job.exclude, flags, nodes, priority: job.priority });
this.#ctx.changes.emit('runs');
return this.#ctx.reply(this.#view(rec, scope));
}
Expand All @@ -639,7 +639,7 @@ export class MockOperate {
const denied = check.permissions.find((p) => !p.allowed);
if (denied) throw new ApiError(denied.reason ?? `missing ${denied.action} permission for this target`, 403);
const parts = parseCommand(request.command.join(' '));
const rec = this.#create({
const rec = this.#create(scope, {
id: this.#newId(),
origin: 'manual',
jobKey: null,
Expand Down Expand Up @@ -1016,7 +1016,7 @@ export class MockOperate {
return id;
}

#create(spec: {
#create(scope: Scope, spec: {
id: string;
origin: Run['origin'];
jobKey: string | null;
Expand Down Expand Up @@ -1055,8 +1055,8 @@ export class MockOperate {
requestedBy: spec.by,
priority: spec.priority,
rewriteHistory: Object.keys(spec.flags).some((f) => flagSpec(spec.verb ?? 'build', f)?.requires === 'run.rewrite_history'),
locationId: '',
target: '',
locationId: scope.locationId,
target: this.#ctx.target(scope),
snapshotId: 'snap-7d2e41a',
gitRevision: '7d2e41a',
queuedAt: now,
Expand Down
Loading