From b236d2d047654703b1c97575088ccd2d1b6b1dc6 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Sun, 4 Oct 2026 19:53:05 +0900 Subject: [PATCH] Make warm-up inbox bench test deterministic The "reports server metrics scoped past the warm-up" test ran a closed-loop load for 400ms of real time with a 120ms warm-up. Server metrics are only reported once the runner takes its baseline snapshot, which happens lazily on the first send scheduled past the warm-up. If both closed-loop workers were still busy with warm-up deliveries when the deadline passed (a cold target, CI scheduling), no measured send was ever dispatched, the baseline was never taken, and server was null. The success-rate assertion could not catch this, because an empty measured window reports a success rate of 1. The test now uses constant open-loop arrivals at 10/s over 400ms under a fake clock, which schedules exactly four sends: two inside the warm-up and two measured. Open-loop scheduling does not depend on completions, so the measured sends, and with them the baseline, always happen however slow the deliveries are. It also asserts that exactly two measured requests were counted client-side and that all four deliveries reached the inbox listener, so a vacuous pass is no longer possible. The fake clock used by the multi-recipient tests is now a shared createFakeClock() helper. Fixes https://github.com/fedify-dev/fedify/issues/1219 Assisted-by: Claude Code:claude-opus-5-5 Assisted-by: Codex:gpt-6-astra --- .../cli/src/bench/scenarios/inbox.test.ts | 132 ++++++++++-------- 1 file changed, 77 insertions(+), 55 deletions(-) diff --git a/packages/cli/src/bench/scenarios/inbox.test.ts b/packages/cli/src/bench/scenarios/inbox.test.ts index ac10a8912..574f4fd98 100644 --- a/packages/cli/src/bench/scenarios/inbox.test.ts +++ b/packages/cli/src/bench/scenarios/inbox.test.ts @@ -85,6 +85,19 @@ async function spawnBenchmarkTarget(usernames: string[] = ["alice"]) { }; } +// A clock that jumps straight to each scheduled arrival, so open-loop runs +// make a fixed number of attempts no matter how slow the deliveries are. +function createFakeClock(): Clock { + let now = 0; + return { + now: () => now, + sleepUntil: (timeMs) => { + now = Math.max(now, timeMs); + return Promise.resolve(); + }, + }; +} + test("inboxRunner - signed deliveries verify against a benchmarkMode target", async () => { const target = await spawnBenchmarkTarget(); let fleet: Awaited> | undefined; @@ -144,55 +157,72 @@ test("inboxRunner - signed deliveries verify against a benchmarkMode target", as } }); -test("inboxRunner - reports server metrics scoped past the warm-up", async () => { - const target = await spawnBenchmarkTarget(); - let fleet: Awaited> | undefined; - try { - fleet = await spawnSyntheticServer( - await buildFleet([{ - count: 1, - signatureStandards: ["draft-cavage-http-signatures-12"], - }]), - ); - const suite: Suite = { - version: 1, - target: target.url.href, - scenarios: [{ - name: "inbox-warmup", - type: "inbox", - recipient: new URL("/users/alice", target.url).href, - inbox: "shared", - load: { concurrency: 2 }, - // A non-zero warm-up exercises the measured-window baseline snapshot. - warmup: "120ms", - duration: "400ms", - }], - }; - const scenario = normalizeSuite(suite).scenarios[0]; - const measurement = await inboxRunner.run({ - scenario, - target: target.url, - documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), - contextLoader: await getContextLoader({ allowPrivateAddress: true }), - allowPrivateAddress: true, - fleet, - }); - - assert.strictEqual(measurement.requests.successRate, 1); - // The measured window verified signatures, so server metrics survive the - // baseline diff rather than being cancelled out by warm-up traffic. - assert.ok( - measurement.server?.signatureVerificationMs != null, - "expected windowed server signature-verification metrics", - ); - } finally { +test( + "inboxRunner - reports server metrics scoped past the warm-up", + { timeout: 60_000 }, + async () => { + const target = await spawnBenchmarkTarget(); + let fleet: Awaited> | undefined; try { - await fleet?.close(); + fleet = await spawnSyntheticServer( + await buildFleet([{ + count: 1, + signatureStandards: ["draft-cavage-http-signatures-12"], + }]), + ); + const suite: Suite = { + version: 1, + target: target.url.href, + scenarios: [{ + name: "inbox-warmup", + type: "inbox", + recipient: new URL("/users/alice", target.url).href, + inbox: "shared", + // 10/s over 400ms schedules exactly four arrivals: two inside the + // warm-up (0 and 100ms) and two measured (200 and 300ms), so the + // measured-window baseline snapshot is always taken. + load: { rate: 10, arrival: "constant", maxInFlight: 2 }, + warmup: "120ms", + duration: "400ms", + }], + }; + const scenario = normalizeSuite(suite).scenarios[0]; + const measurement = await inboxRunner.run({ + scenario, + target: target.url, + documentLoader: await getDocumentLoader({ allowPrivateAddress: true }), + contextLoader: await getContextLoader({ allowPrivateAddress: true }), + allowPrivateAddress: true, + fleet, + clock: createFakeClock(), + }); + + // Only the two measured deliveries are counted client-side, although all + // four, warm-up included, reached the inbox listener. + assert.strictEqual(measurement.requests.total, 2); + assert.strictEqual( + measurement.requests.successRate, + 1, + `expected all deliveries to succeed; errors: ${ + JSON.stringify(measurement.errors) + }`, + ); + assert.strictEqual(target.receivedCount(), 4); + // The measured window verified signatures, so server metrics survive the + // baseline diff rather than being cancelled out by warm-up traffic. + assert.ok( + measurement.server?.signatureVerificationMs != null, + "expected windowed server signature-verification metrics", + ); } finally { - await target.close(); + try { + await fleet?.close(); + } finally { + await target.close(); + } } - } -}); + }, +); // Recipients are allocated round-robin by signing index, so the run makes a // fixed number of attempts rather than however many fit in a real-time window: @@ -230,14 +260,6 @@ async function assertAllocatesAcrossRecipients( }], }; const scenario = normalizeSuite(suite).scenarios[0]; - let now = 0; - const clock: Clock = { - now: () => now, - sleepUntil: (timeMs) => { - now = Math.max(now, timeMs); - return Promise.resolve(); - }, - }; const measurement = await inboxRunner.run({ scenario, target: target.url, @@ -245,7 +267,7 @@ async function assertAllocatesAcrossRecipients( contextLoader: await getContextLoader({ allowPrivateAddress: true }), allowPrivateAddress: true, fleet, - clock, + clock: createFakeClock(), }); assert.strictEqual(measurement.requests.total, 4);