c0mpute edge: keep last good copies on disk, retry ~10 s before giving up

README.md

@@ -280,9 +280,10 @@ browser --TLS (Moshpit CA leaf)--> edge --HTTP--> any healthy c0mpute gateway --
   kept when discovery fails. No c0mpute server is on the path: any gateway can
   serve any site.
 - When every gateway fails, the edge retries the round on a Fibonacci schedule
-  (0.5 s, 0.5 s, 1 s, 1.5 s), long enough to ride out a gateway restart or
+  (0.5, 0.5, 1, 1.5, 2.5, 4 s), long enough to ride out a gateway restart or
   self-update. If they are still down it serves the last good copy of the page
-  (kept in memory, 4 MiB per page, 128 MiB in all, marked `x-c0mpute-stale: 1`),
+  (4 MiB per page, 128 MiB in memory, also on disk in `EDGE_STALE_DIR` so it
+  survives a restart, 1 GiB, marked `x-c0mpute-stale: 1`),
   and only then a 503 page with `Retry-After: 30` that reloads itself. A failing
   gateway sits out 5 s, 5 s, 10 s, 15 s, 25 s ... up to 5 minutes.
 - Plain `http://` serves the same sites rather than redirecting, for the reason

bin/moshpit-c0mpute-edge.ts

@@ -5,7 +5,7 @@
 
 import { realpathSync } from "node:fs";
 import { fileURLToPath } from "node:url";
-import { CertStore, GatewayPool, SiteLookup, startEdge } from "../lib/c0mpute-edge.ts";
+import { CertStore, GatewayPool, LastGood, SiteLookup, startEdge } from "../lib/c0mpute-edge.ts";
 
 /** `host:port` or `[v6]:port`, or null. */
 export function hostPort(v: string | undefined): { host: string; port: number } | null {
@@ -33,6 +33,9 @@ export function edgeConfig(env: Record<string, string | undefined> = process.env
     proxyProtocol: /^(1|true|yes|on)$/i.test(env.EDGE_PROXY_PROTOCOL || ""),
     fallback: hostPort(env.EDGE_FALLBACK),
     refreshMs: int(env.C0MPUTE_REFRESH_SECONDS, 60) * 1000,
+    // Last good copy of every page, served stale when every gateway is down.
+    // On disk so it survives a restart of the edge; "off" keeps it in memory only.
+    staleDir: env.EDGE_STALE_DIR === "off" ? null : env.EDGE_STALE_DIR || `${env.EDGE_CERT_DIR || "/var/lib/moshpit/c0mpute-edge"}/stale`,
   };
 }
 
@@ -47,10 +50,12 @@ async function main() {
   const timer = setInterval(() => { void gateways.refresh(); }, cfg.refreshMs);
   timer.unref();
 
+  const lastGood = new LastGood({ dir: cfg.staleDir });
+  await lastGood.load();
   const sites = new SiteLookup({ registryBase: cfg.registryBase });
   const certs = new CertStore({ dir: cfg.certDir, registryBase: cfg.registryBase, apiKey: cfg.apiKey, log });
   await startEdge({ sites, gateways, certs, host: cfg.host, httpsPort: cfg.httpsPort, httpPort: cfg.httpPort,
-    proxyProtocol: cfg.proxyProtocol, fallback: cfg.fallback, log });
+    proxyProtocol: cfg.proxyProtocol, fallback: cfg.fallback, lastGood, log });
   log(`[edge] serving c0mpute sites on https :${cfg.httpsPort}${cfg.httpPort ? ` and http :${cfg.httpPort}` : ""}; ${gateways.size} gateway(s) known${cfg.fallback ? `; other names to ${cfg.fallback.host}:${cfg.fallback.port}` : ""}`);
 }
 

lib/c0mpute-edge.ts

@@ -29,12 +29,12 @@
 // Retry-After. A failing gateway also sits out on a Fibonacci schedule.
 
 import { execFile } from "node:child_process";
-import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { mkdir, readdir, readFile, rename, stat, unlink, writeFile } from "node:fs/promises";
 import { createServer as createHttpServer } from "node:http";
 import type { IncomingMessage, Server as HttpServer, ServerResponse } from "node:http";
 import { createServer as createHttpsServer } from "node:https";
 import type { Server as HttpsServer } from "node:https";
-import { X509Certificate } from "node:crypto";
+import { createHash, X509Certificate } from "node:crypto";
 import { createSecureContext } from "node:tls";
 import type { SecureContext, TLSSocket } from "node:tls";
 import { join } from "node:path";
@@ -394,7 +394,7 @@ export interface EdgeHandlerOptions {
   fetchImpl?: typeof fetch;
   timeoutMs?: number;
   attempts?: number;
-  /** Extra rounds over the gateways after the first fails, Fibonacci spaced. Default 4. */
+  /** Extra rounds over the gateways after the first fails, Fibonacci spaced. Default 6 (~10 s, a gateway restart). */
   retries?: number;
   /** The first retry's wait; later ones are Fibonacci multiples. Default 500 ms. */
   retryBaseMs?: number;
@@ -417,18 +417,47 @@ export class LastGood {
   private bytes = 0;
   readonly maxEntryBytes: number;
   readonly maxBytes: number;
+  readonly maxDiskBytes: number;
+  private dir: string | null;
   private now: () => number;
 
-  constructor(opts: { maxEntryBytes?: number; maxBytes?: number; now?: () => number } = {}) {
+  /**
+   * `dir` keeps every copy on disk too, so they outlive a restart of the edge
+   * (the moment they are most needed is right after one). Pruned oldest first
+   * past `maxDiskBytes` on `load()`.
+   */
+  constructor(opts: { maxEntryBytes?: number; maxBytes?: number; maxDiskBytes?: number; dir?: string | null; now?: () => number } = {}) {
     this.maxEntryBytes = opts.maxEntryBytes ?? 4 * 1024 * 1024;
     this.maxBytes = opts.maxBytes ?? 128 * 1024 * 1024;
+    this.maxDiskBytes = opts.maxDiskBytes ?? 1024 * 1024 * 1024;
+    this.dir = opts.dir ?? null;
     this.now = opts.now ?? Date.now;
   }
 
   static key(site: string, path: string): string { return `${site}${path}`; }
 
   get size(): number { return this.pages.size; }
 
+  private file(key: string): string {
+    return join(this.dir!, `${createHash("sha256").update(key).digest("hex")}.page`);
+  }
+
+  /** Create the directory and prune it to `maxDiskBytes`, oldest first. Never throws. */
+  async load(): Promise<void> {
+    if (!this.dir) return;
+    try {
+      await mkdir(this.dir, { recursive: true });
+      const files = await Promise.all((await readdir(this.dir)).filter((f) => f.endsWith(".page"))
+        .map(async (f) => ({ f, st: await stat(join(this.dir!, f)) })));
+      files.sort((a, b) => b.st.mtimeMs - a.st.mtimeMs);
+      let total = 0;
+      for (const { f, st } of files) {
+        total += st.size;
+        if (total > this.maxDiskBytes) await unlink(join(this.dir, f)).catch(() => {});
+      }
+    } catch { /* no disk copies, memory only */ }
+  }
+
   get(key: string): Kept | null {
     const k = this.pages.get(key);
     if (!k) return null;
@@ -437,12 +466,40 @@ export class LastGood {
     return k;
   }
 
+  /** From memory, else from disk (and back into memory). */
+  async recall(key: string): Promise<Kept | null> {
+    const hit = this.get(key);
+    if (hit || !this.dir) return hit;
+    try {
+      const raw = await readFile(this.file(key));
+      const nl = raw.indexOf(10);
+      const meta = JSON.parse(raw.subarray(0, nl).toString("utf8")) as { key: string; status: number; headers: Record<string, string>; at: number };
+      if (meta.key !== key) return null;
+      const body = raw.subarray(nl + 1);
+      this.remember(key, { status: meta.status, headers: meta.headers, body, at: meta.at });
+      return this.pages.get(key) ?? null;
+    } catch {
+      return null;
+    }
+  }
+
   put(key: string, status: number, headers: Record<string, string>, body: Buffer): void {
     if (body.length > this.maxEntryBytes) return;
+    const kept = { status, headers, body, at: this.now() };
+    this.remember(key, kept);
+    if (this.dir) {
+      const file = this.file(key);
+      const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}`;
+      const head = Buffer.from(`${JSON.stringify({ key, status, headers, at: kept.at })}\n`);
+      writeFile(tmp, Buffer.concat([head, body])).then(() => rename(tmp, file)).catch(() => { unlink(tmp).catch(() => {}); });
+    }
+  }
+
+  private remember(key: string, kept: Kept): void {
     const old = this.pages.get(key);
     if (old) { this.bytes -= old.body.length; this.pages.delete(key); }
-    this.pages.set(key, { status, headers, body, at: this.now() });
-    this.bytes += body.length;
+    this.pages.set(key, kept);
+    this.bytes += kept.body.length;
     while (this.bytes > this.maxBytes && this.pages.size) {
       const [k, v] = this.pages.entries().next().value as [string, Kept];
       this.pages.delete(k);
@@ -470,7 +527,7 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) {
   const fetchImpl = opts.fetchImpl ?? fetch;
   const timeoutMs = opts.timeoutMs ?? 10_000;
   const attempts = opts.attempts ?? 3;
-  const retries = opts.retries ?? 4;
+  const retries = opts.retries ?? 6;
   const delays = fibonacciDelays(retries, opts.retryBaseMs ?? 500, 10_000);
   const sleep = opts.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
   const lastGood = opts.lastGood === undefined ? new LastGood() : opts.lastGood;
@@ -574,7 +631,7 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) {
       if (await tryRound()) return;
     }
 
-    const stale = lastGood?.get(key);
+    const stale = lastGood ? await lastGood.recall(key) : null;
     if (stale) {
       log(`[edge] ${host}${path}: every gateway failed ${delays.length + 1} rounds; serving the last good copy`);
       const headers: Record<string, string> = { ...stale.headers, "x-c0mpute-stale": "1",

tests/c0mpute-edge.test.ts

@@ -10,7 +10,7 @@ import type { IncomingMessage, Server, ServerResponse } from "node:http";
 import { request as httpsRequest } from "node:https";
 import { request as httpRequest } from "node:http";
 import type { AddressInfo } from "node:net";
-import { readFile, writeFile } from "node:fs/promises";
+import { readFile, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { promisify } from "node:util";
 
@@ -431,7 +431,7 @@ test("plain http serves the same sites, GET and HEAD only, stale then 503 when n
   assert.equal((await call("GET", "127.0.0.1")).status, 400);
   up = false;
   assert.deepEqual(await call("GET", "blue.eggs"), { status: 200, text: "hello", stale: "1" }, "the last good copy, marked stale");
-  assert.deepEqual(slept, [500, 500, 1000, 1500], "retried on a Fibonacci schedule first");
+  assert.deepEqual(slept, [500, 500, 1000, 1500, 2500, 4000], "retried on a Fibonacci schedule first");
   const miss = await call("GET", "blue.eggs", "/never-seen");
   assert.equal(miss.status, 503, "nothing kept: a graceful 503, not a bare 502");
   assert.match(miss.text, /back in a moment/);
@@ -474,3 +474,18 @@ test("fibonacci delays and the LRU of last good pages", () => {
   assert.equal(lg.get("b"), null, "least recently used went first");
   assert.ok(lg.get("a") && lg.get("c"));
 });
+
+test("last good copies outlive a restart when kept on disk", async (t) => {
+  const dir = await tempDir("moshpit-edge-stale-");
+  t.after(() => rm(dir, { recursive: true, force: true }));
+  const a = new LastGood({ dir });
+  await a.load();
+  a.put("site/x", 200, { "content-type": "text/html" }, Buffer.from("<p>kept</p>"));
+  await new Promise((r) => setTimeout(r, 50));
+  const b = new LastGood({ dir });
+  await b.load();
+  const got = await b.recall("site/x");
+  assert.equal(got?.body.toString(), "<p>kept</p>");
+  assert.equal(got?.headers["content-type"], "text/html");
+  assert.equal(await b.recall("site/never"), null);
+});