From 3fad78dbf6e6ffcd6b8fda429654c7547ea5c0fb Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 9 Nov 2026 12:08:17 +0000 Subject: [PATCH] c0mpute edge: keep last good copies on disk, retry ~10 s before giving up A live drill (SIGKILL dev2's gateway worker) showed two gaps: the 3.5 s retry window was shorter than a worker restart (~6 s), and the in-memory copies were gone because the edge had just restarted. Now 6 Fibonacci retries (0.5..4 s) and copies written through to EDGE_STALE_DIR (default /stale, pruned oldest first past 1 GiB on start). Co-Authored-By: Claude Opus 5.5 --- README.md | 5 ++- bin/moshpit-c0mpute-edge.ts | 9 ++++- lib/c0mpute-edge.ts | 73 +++++++++++++++++++++++++++++++++---- tests/c0mpute-edge.test.ts | 19 +++++++++- 4 files changed, 92 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 28dac2d..9178f50 100644 --- a/README.md +++ b/README.md @@ -280,9 +280,10 @@ browser --TLS (Moshpit CA leaf)--> edge --HTTP--> any healthy c0mpute gateway -- kept when discovery fails. No c0mpute server is on the path: any gateway can serve any site. - When every gateway fails, the edge retries the round on a Fibonacci schedule - (0.5 s, 0.5 s, 1 s, 1.5 s), long enough to ride out a gateway restart or + (0.5, 0.5, 1, 1.5, 2.5, 4 s), long enough to ride out a gateway restart or self-update. If they are still down it serves the last good copy of the page - (kept in memory, 4 MiB per page, 128 MiB in all, marked `x-c0mpute-stale: 1`), + (4 MiB per page, 128 MiB in memory, also on disk in `EDGE_STALE_DIR` so it + survives a restart, 1 GiB, marked `x-c0mpute-stale: 1`), and only then a 503 page with `Retry-After: 30` that reloads itself. A failing gateway sits out 5 s, 5 s, 10 s, 15 s, 25 s ... up to 5 minutes. - Plain `http://` serves the same sites rather than redirecting, for the reason diff --git a/bin/moshpit-c0mpute-edge.ts b/bin/moshpit-c0mpute-edge.ts index d388800..d12f29d 100644 --- a/bin/moshpit-c0mpute-edge.ts +++ b/bin/moshpit-c0mpute-edge.ts @@ -5,7 +5,7 @@ import { realpathSync } from "node:fs"; import { fileURLToPath } from "node:url"; -import { CertStore, GatewayPool, SiteLookup, startEdge } from "../lib/c0mpute-edge.ts"; +import { CertStore, GatewayPool, LastGood, SiteLookup, startEdge } from "../lib/c0mpute-edge.ts"; /** `host:port` or `[v6]:port`, or null. */ export function hostPort(v: string | undefined): { host: string; port: number } | null { @@ -33,6 +33,9 @@ export function edgeConfig(env: Record = process.env proxyProtocol: /^(1|true|yes|on)$/i.test(env.EDGE_PROXY_PROTOCOL || ""), fallback: hostPort(env.EDGE_FALLBACK), refreshMs: int(env.C0MPUTE_REFRESH_SECONDS, 60) * 1000, + // Last good copy of every page, served stale when every gateway is down. + // On disk so it survives a restart of the edge; "off" keeps it in memory only. + staleDir: env.EDGE_STALE_DIR === "off" ? null : env.EDGE_STALE_DIR || `${env.EDGE_CERT_DIR || "/var/lib/moshpit/c0mpute-edge"}/stale`, }; } @@ -47,10 +50,12 @@ async function main() { const timer = setInterval(() => { void gateways.refresh(); }, cfg.refreshMs); timer.unref(); + const lastGood = new LastGood({ dir: cfg.staleDir }); + await lastGood.load(); const sites = new SiteLookup({ registryBase: cfg.registryBase }); const certs = new CertStore({ dir: cfg.certDir, registryBase: cfg.registryBase, apiKey: cfg.apiKey, log }); await startEdge({ sites, gateways, certs, host: cfg.host, httpsPort: cfg.httpsPort, httpPort: cfg.httpPort, - proxyProtocol: cfg.proxyProtocol, fallback: cfg.fallback, log }); + proxyProtocol: cfg.proxyProtocol, fallback: cfg.fallback, lastGood, log }); log(`[edge] serving c0mpute sites on https :${cfg.httpsPort}${cfg.httpPort ? ` and http :${cfg.httpPort}` : ""}; ${gateways.size} gateway(s) known${cfg.fallback ? `; other names to ${cfg.fallback.host}:${cfg.fallback.port}` : ""}`); } diff --git a/lib/c0mpute-edge.ts b/lib/c0mpute-edge.ts index 71479a7..2cff010 100644 --- a/lib/c0mpute-edge.ts +++ b/lib/c0mpute-edge.ts @@ -29,12 +29,12 @@ // Retry-After. A failing gateway also sits out on a Fibonacci schedule. import { execFile } from "node:child_process"; -import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { mkdir, readdir, readFile, rename, stat, unlink, writeFile } from "node:fs/promises"; import { createServer as createHttpServer } from "node:http"; import type { IncomingMessage, Server as HttpServer, ServerResponse } from "node:http"; import { createServer as createHttpsServer } from "node:https"; import type { Server as HttpsServer } from "node:https"; -import { X509Certificate } from "node:crypto"; +import { createHash, X509Certificate } from "node:crypto"; import { createSecureContext } from "node:tls"; import type { SecureContext, TLSSocket } from "node:tls"; import { join } from "node:path"; @@ -394,7 +394,7 @@ export interface EdgeHandlerOptions { fetchImpl?: typeof fetch; timeoutMs?: number; attempts?: number; - /** Extra rounds over the gateways after the first fails, Fibonacci spaced. Default 4. */ + /** Extra rounds over the gateways after the first fails, Fibonacci spaced. Default 6 (~10 s, a gateway restart). */ retries?: number; /** The first retry's wait; later ones are Fibonacci multiples. Default 500 ms. */ retryBaseMs?: number; @@ -417,11 +417,20 @@ export class LastGood { private bytes = 0; readonly maxEntryBytes: number; readonly maxBytes: number; + readonly maxDiskBytes: number; + private dir: string | null; private now: () => number; - constructor(opts: { maxEntryBytes?: number; maxBytes?: number; now?: () => number } = {}) { + /** + * `dir` keeps every copy on disk too, so they outlive a restart of the edge + * (the moment they are most needed is right after one). Pruned oldest first + * past `maxDiskBytes` on `load()`. + */ + constructor(opts: { maxEntryBytes?: number; maxBytes?: number; maxDiskBytes?: number; dir?: string | null; now?: () => number } = {}) { this.maxEntryBytes = opts.maxEntryBytes ?? 4 * 1024 * 1024; this.maxBytes = opts.maxBytes ?? 128 * 1024 * 1024; + this.maxDiskBytes = opts.maxDiskBytes ?? 1024 * 1024 * 1024; + this.dir = opts.dir ?? null; this.now = opts.now ?? Date.now; } @@ -429,6 +438,26 @@ export class LastGood { get size(): number { return this.pages.size; } + private file(key: string): string { + return join(this.dir!, `${createHash("sha256").update(key).digest("hex")}.page`); + } + + /** Create the directory and prune it to `maxDiskBytes`, oldest first. Never throws. */ + async load(): Promise { + if (!this.dir) return; + try { + await mkdir(this.dir, { recursive: true }); + const files = await Promise.all((await readdir(this.dir)).filter((f) => f.endsWith(".page")) + .map(async (f) => ({ f, st: await stat(join(this.dir!, f)) }))); + files.sort((a, b) => b.st.mtimeMs - a.st.mtimeMs); + let total = 0; + for (const { f, st } of files) { + total += st.size; + if (total > this.maxDiskBytes) await unlink(join(this.dir, f)).catch(() => {}); + } + } catch { /* no disk copies, memory only */ } + } + get(key: string): Kept | null { const k = this.pages.get(key); if (!k) return null; @@ -437,12 +466,40 @@ export class LastGood { return k; } + /** From memory, else from disk (and back into memory). */ + async recall(key: string): Promise { + const hit = this.get(key); + if (hit || !this.dir) return hit; + try { + const raw = await readFile(this.file(key)); + const nl = raw.indexOf(10); + const meta = JSON.parse(raw.subarray(0, nl).toString("utf8")) as { key: string; status: number; headers: Record; at: number }; + if (meta.key !== key) return null; + const body = raw.subarray(nl + 1); + this.remember(key, { status: meta.status, headers: meta.headers, body, at: meta.at }); + return this.pages.get(key) ?? null; + } catch { + return null; + } + } + put(key: string, status: number, headers: Record, body: Buffer): void { if (body.length > this.maxEntryBytes) return; + const kept = { status, headers, body, at: this.now() }; + this.remember(key, kept); + if (this.dir) { + const file = this.file(key); + const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}`; + const head = Buffer.from(`${JSON.stringify({ key, status, headers, at: kept.at })}\n`); + writeFile(tmp, Buffer.concat([head, body])).then(() => rename(tmp, file)).catch(() => { unlink(tmp).catch(() => {}); }); + } + } + + private remember(key: string, kept: Kept): void { const old = this.pages.get(key); if (old) { this.bytes -= old.body.length; this.pages.delete(key); } - this.pages.set(key, { status, headers, body, at: this.now() }); - this.bytes += body.length; + this.pages.set(key, kept); + this.bytes += kept.body.length; while (this.bytes > this.maxBytes && this.pages.size) { const [k, v] = this.pages.entries().next().value as [string, Kept]; this.pages.delete(k); @@ -470,7 +527,7 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) { const fetchImpl = opts.fetchImpl ?? fetch; const timeoutMs = opts.timeoutMs ?? 10_000; const attempts = opts.attempts ?? 3; - const retries = opts.retries ?? 4; + const retries = opts.retries ?? 6; const delays = fibonacciDelays(retries, opts.retryBaseMs ?? 500, 10_000); const sleep = opts.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); const lastGood = opts.lastGood === undefined ? new LastGood() : opts.lastGood; @@ -574,7 +631,7 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) { if (await tryRound()) return; } - const stale = lastGood?.get(key); + const stale = lastGood ? await lastGood.recall(key) : null; if (stale) { log(`[edge] ${host}${path}: every gateway failed ${delays.length + 1} rounds; serving the last good copy`); const headers: Record = { ...stale.headers, "x-c0mpute-stale": "1", diff --git a/tests/c0mpute-edge.test.ts b/tests/c0mpute-edge.test.ts index 4f7d767..0db623b 100644 --- a/tests/c0mpute-edge.test.ts +++ b/tests/c0mpute-edge.test.ts @@ -10,7 +10,7 @@ import type { IncomingMessage, Server, ServerResponse } from "node:http"; import { request as httpsRequest } from "node:https"; import { request as httpRequest } from "node:http"; import type { AddressInfo } from "node:net"; -import { readFile, writeFile } from "node:fs/promises"; +import { readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -431,7 +431,7 @@ test("plain http serves the same sites, GET and HEAD only, stale then 503 when n assert.equal((await call("GET", "127.0.0.1")).status, 400); up = false; assert.deepEqual(await call("GET", "blue.eggs"), { status: 200, text: "hello", stale: "1" }, "the last good copy, marked stale"); - assert.deepEqual(slept, [500, 500, 1000, 1500], "retried on a Fibonacci schedule first"); + assert.deepEqual(slept, [500, 500, 1000, 1500, 2500, 4000], "retried on a Fibonacci schedule first"); const miss = await call("GET", "blue.eggs", "/never-seen"); assert.equal(miss.status, 503, "nothing kept: a graceful 503, not a bare 502"); assert.match(miss.text, /back in a moment/); @@ -474,3 +474,18 @@ test("fibonacci delays and the LRU of last good pages", () => { assert.equal(lg.get("b"), null, "least recently used went first"); assert.ok(lg.get("a") && lg.get("c")); }); + +test("last good copies outlive a restart when kept on disk", async (t) => { + const dir = await tempDir("moshpit-edge-stale-"); + t.after(() => rm(dir, { recursive: true, force: true })); + const a = new LastGood({ dir }); + await a.load(); + a.put("site/x", 200, { "content-type": "text/html" }, Buffer.from("

kept

")); + await new Promise((r) => setTimeout(r, 50)); + const b = new LastGood({ dir }); + await b.load(); + const got = await b.recall("site/x"); + assert.equal(got?.body.toString(), "

kept

"); + assert.equal(got?.headers["content-type"], "text/html"); + assert.equal(await b.recall("site/never"), null); +});