Skip to content

Commit c8cfe98

Browse files
committed
test(server): guard retained scans with an interleaved page ratio instead of a five-second wall clock
1 parent 2cbd4e5 commit c8cfe98

1 file changed

Lines changed: 62 additions & 26 deletions

File tree

  • crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation

‎crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs‎

Lines changed: 62 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@ use uuid::Uuid;
77
use crate::store::main::{begin::begin, commit::commit, query::query};
88
use crate::store::models::{ControlStore, TransactionMode};
99

10+
/// Records a scan page carries, and the size of the control namespace scanned beside the retained one.
11+
const SCAN_PAGE: usize = 100;
12+
/// The median retained page may cost at most this many times the control page sent right after it.
13+
const MAX_RETAINED_SCAN_RATIO: f64 = 1.8;
14+
/// Only a hang trips this; per-commit disk syncs make wall time vary widely between runners.
15+
const SCAN_HANG_BOUND: std::time::Duration = std::time::Duration::from_secs(60);
16+
1017
pub(crate) async fn reconcile_history(
1118
store: &ControlStore,
1219
(client, url, bearer): (&Client, &str, &str),
@@ -145,52 +152,81 @@ async fn retained_history(
145152
(session_id, fixture, record, (kind, timestamp)): (Uuid, &Value, &Value, (&str, &str)),
146153
) {
147154
let count = fixture["retained_count"].as_u64().expect("retained count");
148-
let records: Vec<_> = (0..count)
149-
.map(|index| {
150-
let mut record = record.clone();
151-
record["event_id"] = json!(format!("retained-{index}"));
152-
record["state_database"] = json!("analytics");
153-
record["state_schema"] = json!("reconcile_retained");
154-
record[timestamp] = json!(format!(
155-
"2026-01-01T10:{:02}:{:02}Z",
156-
index / 120,
157-
(index / 2) % 60
158-
));
159-
record
160-
})
161-
.collect();
162-
for records in records.chunks(100) {
155+
let namespace_records = |schema: &str, count: u64| -> Vec<Value> {
156+
(0..count)
157+
.map(|index| {
158+
let mut record = record.clone();
159+
record["event_id"] = json!(format!("{schema}-{index}"));
160+
record["state_database"] = json!("analytics");
161+
record["state_schema"] = json!(schema);
162+
record[timestamp] = json!(format!(
163+
"2026-01-01T10:{:02}:{:02}Z",
164+
index / 120,
165+
(index / 2) % 60
166+
));
167+
record
168+
})
169+
.collect()
170+
};
171+
let records = namespace_records("reconcile_retained", count);
172+
let control = namespace_records("reconcile_control", SCAN_PAGE as u64);
173+
for records in records.chunks(SCAN_PAGE).chain([control.as_slice()]) {
163174
post(
164175
(client, url, bearer),
165176
&json!({"kind":kind,"session_id":session_id,"generation":1,"records":records}),
166177
StatusCode::OK,
167178
)
168179
.await;
169180
}
170-
let mut page = json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{
171-
"scan_id":Uuid::now_v7(),"location":{"database":"analytics","schema":"reconcile_retained"},"complete":false}});
172-
let pages = records.len().div_ceil(100);
173-
tokio::time::timeout(std::time::Duration::from_secs(5), async {
174-
for (index, records) in records.chunks(100).enumerate() {
175-
page["records"] = json!(records);
176-
page["scan"]["complete"] = json!(index + 1 == pages);
177-
let reply = post((client, url, bearer), &page, StatusCode::OK).await;
181+
let scan_page = |schema: &str| {
182+
json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{
183+
"scan_id":Uuid::now_v7(),"location":{"database":"analytics","schema":schema},"complete":false}})
184+
};
185+
let (mut retained_page, mut control_page) = (
186+
scan_page("reconcile_retained"),
187+
scan_page("reconcile_control"),
188+
);
189+
control_page["records"] = json!(control);
190+
let pages = records.len().div_ceil(SCAN_PAGE);
191+
let mut ratios: Vec<f64> = tokio::time::timeout(SCAN_HANG_BOUND, async {
192+
let mut ratios = Vec::with_capacity(pages);
193+
for (index, records) in records.chunks(SCAN_PAGE).enumerate() {
194+
retained_page["records"] = json!(records);
195+
for page in [&mut retained_page, &mut control_page] {
196+
page["scan"]["complete"] = json!(index + 1 == pages);
197+
}
198+
control_page["scan"]["scan_id"] = json!(Uuid::now_v7());
199+
let started = std::time::Instant::now();
200+
let retained = post((client, url, bearer), &retained_page, StatusCode::OK).await;
201+
let retained_time = started.elapsed();
202+
let started = std::time::Instant::now();
203+
let unchanged = post((client, url, bearer), &control_page, StatusCode::OK).await;
204+
ratios.push(retained_time.as_secs_f64() / started.elapsed().as_secs_f64());
178205
assert_eq!(
179-
reply["applied"], false,
206+
[&retained["applied"], &unchanged["applied"]],
207+
[false, false],
180208
"retained observations never change"
181209
);
182210
}
211+
ratios
183212
})
184213
.await
185-
.expect("a retained-heavy scan has a strict total deadline");
214+
.expect("a retained-heavy scan finishes");
215+
ratios.sort_by(f64::total_cmp);
216+
let median = (ratios[(pages - 1) / 2] + ratios[pages / 2]) / 2.0;
217+
assert!(
218+
median < MAX_RETAINED_SCAN_RATIO,
219+
"a scan page's cost must not grow with the namespace's history: over {count} retained \
220+
rows a page took {median:.2} times a page over {SCAN_PAGE}"
221+
);
186222
let (transaction, rows) = query(
187223
begin(store, TransactionMode::Read).await.expect("read retained"),
188224
&format!("SELECT record_json ->> 'event_id' AS source_event_id FROM {table} WHERE state_database='analytics' AND state_schema='reconcile_retained' ORDER BY source_event_id"),
189225
vec![],
190226
).await.expect("retained identities");
191227
commit(transaction).await.expect("close retained read");
192228
let mut expected: Vec<_> = (0..count)
193-
.map(|index| format!("retained-{index}"))
229+
.map(|index| format!("reconcile_retained-{index}"))
194230
.collect();
195231
expected.sort();
196232
assert_eq!(

0 commit comments

Comments
 (0)