From adf23d36f43a94a089d68d633bf8fae8961e8f7e Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 9 Nov 2026 12:04:53 +0000 Subject: [PATCH] c0mpute edge: Fibonacci retries, serve the last good copy, 503 page instead of 502 When every gateway failed a request the edge answered 502 at once, so one gateway restart (dev2's gateway worker died today and cloud.killer went dark) took every Moshpit name on c0mpute down with it. - retry the gateway round 4 more times on a Fibonacci schedule (0.5/0.5/1/1.5 s) - keep the last good 200 of each page (LRU, 4 MiB/page, 128 MiB total) and serve it marked x-c0mpute-stale: 1 when every gateway is still down - otherwise a self-reloading 503 page with Retry-After: 30 - failing gateways sit out on a Fibonacci schedule (5 s .. 5 min) Co-Authored-By: Claude Opus 5.5 --- README.md | 6 + lib/c0mpute-edge.ts | 224 +++++++++++++++++++++++++++++-------- tests/c0mpute-edge.test.ts | 57 +++++++++- 3 files changed, 234 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 84c9670..28dac2d 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,12 @@ browser --TLS (Moshpit CA leaf)--> edge --HTTP--> any healthy c0mpute gateway -- that announce a gateway URL), re-checked every minute; the last good list is kept when discovery fails. No c0mpute server is on the path: any gateway can serve any site. +- When every gateway fails, the edge retries the round on a Fibonacci schedule + (0.5 s, 0.5 s, 1 s, 1.5 s), long enough to ride out a gateway restart or + self-update. If they are still down it serves the last good copy of the page + (kept in memory, 4 MiB per page, 128 MiB in all, marked `x-c0mpute-stale: 1`), + and only then a 503 page with `Retry-After: 30` that reloads itself. A failing + gateway sits out 5 s, 5 s, 10 s, 15 s, 25 s ... up to 5 minutes. - Plain `http://` serves the same sites rather than redirecting, for the reason `nginx/moshpit-origin.conf` gives: a visitor without the Moshpit root gets a page instead of a certificate error. diff --git a/lib/c0mpute-edge.ts b/lib/c0mpute-edge.ts index 61bac04..71479a7 100644 --- a/lib/c0mpute-edge.ts +++ b/lib/c0mpute-edge.ts @@ -21,6 +21,12 @@ // site, so there is no c0mpute server on the path to lose. Gateways are // discovered from seeds and from the public node list (nodes that announce a // gateway URL), and the last good list is kept when discovery fails. +// +// When every gateway fails a request the edge does not give up at once: it +// retries the round a few times on a Fibonacci schedule (a gateway restarting +// or re-executing for an update is back within seconds), then serves the last +// good copy of that page it saw (marked stale), and only then a 503 page with +// Retry-After. A failing gateway also sits out on a Fibonacci schedule. import { execFile } from "node:child_process"; import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; @@ -50,6 +56,27 @@ export function normalizeHost(input: string | undefined | null): string | null { return NAME_RE.test(host) ? host : null; } +/* -------------------------------------------------------------- backoff */ + +/** + * The first `n` Fibonacci multiples of `baseMs` (1, 1, 2, 3, 5, 8 ...), each + * capped at `capMs`. The retry schedule everywhere in the edge. + */ +export function fibonacciDelays(n: number, baseMs: number, capMs = Number.MAX_SAFE_INTEGER): number[] { + const out: number[] = []; + let a = 1, b = 1; + for (let i = 0; i < n; i++) { + out.push(Math.min(capMs, a * baseMs)); + [a, b] = [b, a + b]; + } + return out; +} + +/** The `k`th (1-based) Fibonacci multiple of `baseMs`, capped. */ +export function fibonacciDelay(k: number, baseMs: number, capMs = Number.MAX_SAFE_INTEGER): number { + return fibonacciDelays(Math.max(1, Math.min(k, 60)), baseMs, capMs).at(-1)!; +} + /* ------------------------------------------------------------- gateways */ /** An http(s) origin with no path, or null. */ @@ -162,8 +189,8 @@ export class GatewayPool { if (!s) return; s.failures += 1; s.healthy = false; - // 10 s, 20 s, 40 s ... capped at 5 minutes. - s.downUntil = this.now() + Math.min(300_000, 10_000 * 2 ** (s.failures - 1)); + // 5 s, 5 s, 10 s, 15 s, 25 s, 40 s ... capped at 5 minutes. + s.downUntil = this.now() + fibonacciDelay(s.failures, 5_000, 300_000); } /** @@ -367,9 +394,71 @@ export interface EdgeHandlerOptions { fetchImpl?: typeof fetch; timeoutMs?: number; attempts?: number; + /** Extra rounds over the gateways after the first fails, Fibonacci spaced. Default 4. */ + retries?: number; + /** The first retry's wait; later ones are Fibonacci multiples. Default 500 ms. */ + retryBaseMs?: number; + /** Last good copies to serve when every gateway fails; null disables. */ + lastGood?: LastGood | null; + sleep?: (ms: number) => Promise; log?: (line: string) => void; } +/** One page as a gateway last served it. */ +type Kept = { status: number; headers: Record; body: Buffer; at: number }; + +/** + * The last good copy of each page, so a visitor gets the site (marked stale) + * instead of an error while every gateway is down. In memory, least recently + * used out first, bounded per page and in total. + */ +export class LastGood { + private pages = new Map(); + private bytes = 0; + readonly maxEntryBytes: number; + readonly maxBytes: number; + private now: () => number; + + constructor(opts: { maxEntryBytes?: number; maxBytes?: number; now?: () => number } = {}) { + this.maxEntryBytes = opts.maxEntryBytes ?? 4 * 1024 * 1024; + this.maxBytes = opts.maxBytes ?? 128 * 1024 * 1024; + this.now = opts.now ?? Date.now; + } + + static key(site: string, path: string): string { return `${site}${path}`; } + + get size(): number { return this.pages.size; } + + get(key: string): Kept | null { + const k = this.pages.get(key); + if (!k) return null; + this.pages.delete(key); + this.pages.set(key, k); + return k; + } + + put(key: string, status: number, headers: Record, body: Buffer): void { + if (body.length > this.maxEntryBytes) return; + const old = this.pages.get(key); + if (old) { this.bytes -= old.body.length; this.pages.delete(key); } + this.pages.set(key, { status, headers, body, at: this.now() }); + this.bytes += body.length; + while (this.bytes > this.maxBytes && this.pages.size) { + const [k, v] = this.pages.entries().next().value as [string, Kept]; + this.pages.delete(k); + this.bytes -= v.body.length; + } + } +} + +const FAIL_PAGE = ` +Back in a moment + +

back in a moment

+

no c0mpute gateway could serve this site just now; the network is healing itself.

+

this page retries on its own every 30 seconds.

+`; + /** The gateway URL for a request path on a site. */ export function upstreamUrl(gateway: string, site: string, path: string): string { const rel = path.startsWith("/") ? path : `/${path}`; @@ -381,6 +470,10 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) { const fetchImpl = opts.fetchImpl ?? fetch; const timeoutMs = opts.timeoutMs ?? 10_000; const attempts = opts.attempts ?? 3; + const retries = opts.retries ?? 4; + const delays = fibonacciDelays(retries, opts.retryBaseMs ?? 500, 10_000); + const sleep = opts.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const lastGood = opts.lastGood === undefined ? new LastGood() : opts.lastGood; const log = opts.log ?? (() => {}); return async function handle(req: IncomingMessage, res: ServerResponse): Promise { @@ -409,56 +502,91 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) { if (!site) return send(421, `${host} does not point at a c0mpute site`); const path = (req.url || "/").startsWith("/") ? (req.url || "/") : "/"; - for (const gw of opts.gateways.pick(attempts)) { - let upstream: Response; - try { - upstream = await fetchImpl(upstreamUrl(gw, site, path), { - method: req.method, - headers: { - ...(req.headers["if-none-match"] ? { "if-none-match": String(req.headers["if-none-match"]) } : {}), - ...(req.headers["accept-encoding"] ? { "accept-encoding": String(req.headers["accept-encoding"]) } : {}), - }, - redirect: "manual", - signal: AbortSignal.timeout(timeoutMs), - }); - } catch (e) { - opts.gateways.failed(gw); - log(`[edge] ${host} via ${gw}: ${(e as Error).message}`); - continue; - } - if (upstream.status >= 500) { - await upstream.body?.cancel(); - opts.gateways.failed(gw); - log(`[edge] ${host} via ${gw}: ${upstream.status}`); - continue; - } - opts.gateways.succeeded(gw); - const headers: Record = { "x-c0mpute-gateway": new URL(gw).host }; - for (const h of PASS_HEADERS) { - const v = upstream.headers.get(h); - if (v !== null) headers[h] = v; - } - res.writeHead(upstream.status, headers); - if (req.method === "HEAD" || !upstream.body) { - await upstream.body?.cancel(); - res.end(); - return; - } - const reader = upstream.body.getReader(); - try { - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - if (!res.write(value)) await new Promise((r) => res.once("drain", r)); + const key = LastGood.key(site, path); + const conditional = Boolean(req.headers["if-none-match"]); + + // One pass over the gateways; true when a response was sent. + const tryRound = async (): Promise => { + for (const gw of opts.gateways.pick(attempts)) { + let upstream: Response; + try { + upstream = await fetchImpl(upstreamUrl(gw, site, path), { + method: req.method, + headers: { + ...(conditional ? { "if-none-match": String(req.headers["if-none-match"]) } : {}), + ...(req.headers["accept-encoding"] ? { "accept-encoding": String(req.headers["accept-encoding"]) } : {}), + }, + redirect: "manual", + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (e) { + opts.gateways.failed(gw); + log(`[edge] ${host} via ${gw}: ${(e as Error).message}`); + continue; + } + if (upstream.status >= 500) { + await upstream.body?.cancel(); + opts.gateways.failed(gw); + log(`[edge] ${host} via ${gw}: ${upstream.status}`); + continue; + } + opts.gateways.succeeded(gw); + const headers: Record = { "x-c0mpute-gateway": new URL(gw).host }; + for (const h of PASS_HEADERS) { + const v = upstream.headers.get(h); + if (v !== null) headers[h] = v; + } + res.writeHead(upstream.status, headers); + if (req.method === "HEAD" || !upstream.body) { + await upstream.body?.cancel(); + res.end(); + return true; + } + // Keep a copy of a full 200 for the day every gateway is down. + const keep = lastGood && upstream.status === 200 && !headers["content-encoding"] ? [] as Buffer[] : null; + let kept = 0; + const reader = upstream.body.getReader(); + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (keep) { + kept += value.length; + if (kept <= lastGood!.maxEntryBytes) keep.push(Buffer.from(value)); + } + if (!res.write(value)) await new Promise((r) => res.once("drain", r)); + } + } catch { + res.destroy(); + return true; } - } catch { - res.destroy(); - return; + res.end(); + if (keep && kept <= lastGood!.maxEntryBytes) lastGood!.put(key, 200, headers, Buffer.concat(keep)); + return true; } - res.end(); + return false; + }; + + if (await tryRound()) return; + for (const ms of delays) { + if (res.destroyed || req.socket.destroyed) return; + await sleep(ms); + if (await tryRound()) return; + } + + const stale = lastGood?.get(key); + if (stale) { + log(`[edge] ${host}${path}: every gateway failed ${delays.length + 1} rounds; serving the last good copy`); + const headers: Record = { ...stale.headers, "x-c0mpute-stale": "1", + "cache-control": "no-store", "content-length": String(stale.body.length) }; + delete headers["content-encoding"]; + res.writeHead(stale.status, headers); + res.end(req.method === "HEAD" ? undefined : stale.body); return; } - send(502, "no c0mpute gateway answered for this site; try again shortly"); + log(`[edge] ${host}${path}: every gateway failed ${delays.length + 1} rounds; no copy kept`); + res.writeHead(503, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store", "retry-after": "30" }); + res.end(req.method === "HEAD" ? undefined : FAIL_PAGE); }; } diff --git a/tests/c0mpute-edge.test.ts b/tests/c0mpute-edge.test.ts index 29d514f..4f7d767 100644 --- a/tests/c0mpute-edge.test.ts +++ b/tests/c0mpute-edge.test.ts @@ -19,6 +19,9 @@ import { GatewayPool, SiteLookup, createEdgeHandler, + fibonacciDelay, + fibonacciDelays, + LastGood, gatewaysFromNodes, normalizeGateway, normalizeHost, @@ -394,14 +397,16 @@ test("https:// is served from a healthy gateway with a registry-signed lea }); }); -test("plain http serves the same sites, GET and HEAD only, 502 when no gateway answers", async (t) => { +test("plain http serves the same sites, GET and HEAD only, stale then 503 when no gateway answers", async (t) => { const sites = { site: async (name: string) => (name === "blue.eggs" ? SITE : null) }; let failures = 0; const gateways = { pick: () => ["http://g1:7777", "http://g2:7777"], succeeded: () => {}, failed: () => { failures++; } }; let up = true; + const slept: number[] = []; const handler = createEdgeHandler({ sites, gateways, + sleep: async (ms) => { slept.push(ms); }, fetchImpl: (async (input: string | URL | Request) => { if (!up) throw new Error("ECONNREFUSED"); if (String(input).startsWith("http://g1")) return new Response("bad", { status: 502 }); @@ -411,9 +416,9 @@ test("plain http serves the same sites, GET and HEAD only, 502 when no gateway a const srv = await listen((req, res) => { void handler(req, res); }); t.after(() => srv.server.close()); const port = Number(new URL(srv.url).port); - const call = (method: string, host: string) => new Promise<{ status: number; text: string }>((resolve, reject) => { - const req = httpRequest({ host: "127.0.0.1", port, method, path: "/", headers: { host } }, (res) => { - let text = ""; res.on("data", (d) => { text += d; }); res.on("end", () => resolve({ status: res.statusCode ?? 0, text })); + const call = (method: string, host: string, path = "/") => new Promise<{ status: number; text: string; stale?: string }>((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, method, path, headers: { host } }, (res) => { + let text = ""; res.on("data", (d) => { text += d; }); res.on("end", () => resolve({ status: res.statusCode ?? 0, text, ...(res.headers["x-c0mpute-stale"] ? { stale: String(res.headers["x-c0mpute-stale"]) } : {}) })); }); req.on("error", reject); req.end(); }); @@ -425,5 +430,47 @@ test("plain http serves the same sites, GET and HEAD only, 502 when no gateway a assert.equal((await call("GET", "red.eggs")).status, 421, "not a c0mpute name: misdirected, not a missing page"); assert.equal((await call("GET", "127.0.0.1")).status, 400); up = false; - assert.equal((await call("GET", "blue.eggs")).status, 502); + assert.deepEqual(await call("GET", "blue.eggs"), { status: 200, text: "hello", stale: "1" }, "the last good copy, marked stale"); + assert.deepEqual(slept, [500, 500, 1000, 1500], "retried on a Fibonacci schedule first"); + const miss = await call("GET", "blue.eggs", "/never-seen"); + assert.equal(miss.status, 503, "nothing kept: a graceful 503, not a bare 502"); + assert.match(miss.text, /back in a moment/); +}); + +test("a gateway that comes back mid-retry serves the request", async () => { + let calls = 0; + const handler = createEdgeHandler({ + sites: { site: async () => SITE }, + gateways: { pick: () => ["http://g1:7777"], succeeded: () => {}, failed: () => {} }, + sleep: async () => {}, + fetchImpl: (async () => { + calls++; + if (calls < 3) throw new Error("ECONNREFUSED"); + return new Response("back", { status: 200 }); + }) as typeof fetch, + }); + const srv = await listen((req, res) => { void handler(req, res); }); + const port = Number(new URL(srv.url).port); + const status = await new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/", headers: { host: "blue.eggs" } }, (res) => { res.resume(); res.on("end", () => resolve(res.statusCode ?? 0)); }); + req.on("error", reject); req.end(); + }); + srv.server.close(); + assert.equal(status, 200); + assert.equal(calls, 3); +}); + +test("fibonacci delays and the LRU of last good pages", () => { + assert.deepEqual(fibonacciDelays(7, 100), [100, 100, 200, 300, 500, 800, 1300]); + assert.deepEqual(fibonacciDelays(4, 5000, 9000), [5000, 5000, 9000, 9000]); + assert.equal(fibonacciDelay(5, 5000, 300_000), 25_000); + const lg = new LastGood({ maxEntryBytes: 4, maxBytes: 6 }); + lg.put("a", 200, {}, Buffer.from("aaa")); + lg.put("b", 200, {}, Buffer.from("bbb")); + lg.put("big", 200, {}, Buffer.from("too big")); + assert.equal(lg.get("big"), null, "over the per-page cap: not kept"); + lg.get("a"); + lg.put("c", 200, {}, Buffer.from("ccc")); + assert.equal(lg.get("b"), null, "least recently used went first"); + assert.ok(lg.get("a") && lg.get("c")); });