Skip to content

c0mpute edge: Fibonacci retries, serve the last good copy, 503 page instead of 502 #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ralyodio merged 1 commit into from
Nov 9, 2026
Merged

c0mpute edge: Fibonacci retries, serve the last good copy, 503 page instead of 502 #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,12 @@ browser --TLS (Moshpit CA leaf)--> edge --HTTP--> any healthy c0mpute gateway --
that announce a gateway URL), re-checked every minute; the last good list is
kept when discovery fails. No c0mpute server is on the path: any gateway can
serve any site.
- When every gateway fails, the edge retries the round on a Fibonacci schedule
(0.5 s, 0.5 s, 1 s, 1.5 s), long enough to ride out a gateway restart or
self-update. If they are still down it serves the last good copy of the page
(kept in memory, 4 MiB per page, 128 MiB in all, marked `x-c0mpute-stale: 1`),
and only then a 503 page with `Retry-After: 30` that reloads itself. A failing
gateway sits out 5 s, 5 s, 10 s, 15 s, 25 s ... up to 5 minutes.
- Plain `http://` serves the same sites rather than redirecting, for the reason
`nginx/moshpit-origin.conf` gives: a visitor without the Moshpit root gets a
page instead of a certificate error.
Expand Down
224 changes: 176 additions & 48 deletions lib/c0mpute-edge.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@
// site, so there is no c0mpute server on the path to lose. Gateways are
// discovered from seeds and from the public node list (nodes that announce a
// gateway URL), and the last good list is kept when discovery fails.
//
// When every gateway fails a request the edge does not give up at once: it
// retries the round a few times on a Fibonacci schedule (a gateway restarting
// or re-executing for an update is back within seconds), then serves the last
// good copy of that page it saw (marked stale), and only then a 503 page with
// Retry-After. A failing gateway also sits out on a Fibonacci schedule.

import { execFile } from "node:child_process";
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
Expand Down Expand Up @@ -50,6 +56,27 @@ export function normalizeHost(input: string | undefined | null): string | null {
return NAME_RE.test(host) ? host : null;
}

/* -------------------------------------------------------------- backoff */

/**
* The first `n` Fibonacci multiples of `baseMs` (1, 1, 2, 3, 5, 8 ...), each
* capped at `capMs`. The retry schedule everywhere in the edge.
*/
export function fibonacciDelays(n: number, baseMs: number, capMs = Number.MAX_SAFE_INTEGER): number[] {
const out: number[] = [];
let a = 1, b = 1;
for (let i = 0; i < n; i++) {
out.push(Math.min(capMs, a * baseMs));
[a, b] = [b, a + b];
}
return out;
}

/** The `k`th (1-based) Fibonacci multiple of `baseMs`, capped. */
export function fibonacciDelay(k: number, baseMs: number, capMs = Number.MAX_SAFE_INTEGER): number {
return fibonacciDelays(Math.max(1, Math.min(k, 60)), baseMs, capMs).at(-1)!;
}

/* ------------------------------------------------------------- gateways */

/** An http(s) origin with no path, or null. */
Expand Down Expand Up @@ -162,8 +189,8 @@ export class GatewayPool {
if (!s) return;
s.failures += 1;
s.healthy = false;
// 10 s, 20 s, 40 s ... capped at 5 minutes.
s.downUntil = this.now() + Math.min(300_000, 10_000 * 2 ** (s.failures - 1));
// 5 s, 5 s, 10 s, 15 s, 25 s, 40 s ... capped at 5 minutes.
s.downUntil = this.now() + fibonacciDelay(s.failures, 5_000, 300_000);
}

/**
Expand Down Expand Up @@ -367,9 +394,71 @@ export interface EdgeHandlerOptions {
fetchImpl?: typeof fetch;
timeoutMs?: number;
attempts?: number;
/** Extra rounds over the gateways after the first fails, Fibonacci spaced. Default 4. */
retries?: number;
/** The first retry's wait; later ones are Fibonacci multiples. Default 500 ms. */
retryBaseMs?: number;
/** Last good copies to serve when every gateway fails; null disables. */
lastGood?: LastGood | null;
sleep?: (ms: number) => Promise<void>;
log?: (line: string) => void;
}

/** One page as a gateway last served it. */
type Kept = { status: number; headers: Record<string, string>; body: Buffer; at: number };

/**
* The last good copy of each page, so a visitor gets the site (marked stale)
* instead of an error while every gateway is down. In memory, least recently
* used out first, bounded per page and in total.
*/
export class LastGood {
private pages = new Map<string, Kept>();
private bytes = 0;
readonly maxEntryBytes: number;
readonly maxBytes: number;
private now: () => number;

constructor(opts: { maxEntryBytes?: number; maxBytes?: number; now?: () => number } = {}) {
this.maxEntryBytes = opts.maxEntryBytes ?? 4 * 1024 * 1024;
this.maxBytes = opts.maxBytes ?? 128 * 1024 * 1024;
this.now = opts.now ?? Date.now;
}

static key(site: string, path: string): string { return `${site}${path}`; }

get size(): number { return this.pages.size; }

get(key: string): Kept | null {
const k = this.pages.get(key);
if (!k) return null;
this.pages.delete(key);
this.pages.set(key, k);
return k;
}

put(key: string, status: number, headers: Record<string, string>, body: Buffer): void {
if (body.length > this.maxEntryBytes) return;
const old = this.pages.get(key);
if (old) { this.bytes -= old.body.length; this.pages.delete(key); }
this.pages.set(key, { status, headers, body, at: this.now() });
this.bytes += body.length;
while (this.bytes > this.maxBytes && this.pages.size) {
const [k, v] = this.pages.entries().next().value as [string, Kept];
this.pages.delete(k);
this.bytes -= v.body.length;
}
}
}

const FAIL_PAGE = `<!doctype html><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Back in a moment</title><meta http-equiv="refresh" content="30">
<body style="background:#070806;color:#edf2e4;font-family:monospace;padding:14vh 24px;text-align:center">
<h1 style="color:#a6ff1a">back in a moment</h1>
<p>no c0mpute gateway could serve this site just now; the network is healing itself.</p>
<p>this page retries on its own every 30 seconds.</p></body>
`;

/** The gateway URL for a request path on a site. */
export function upstreamUrl(gateway: string, site: string, path: string): string {
const rel = path.startsWith("/") ? path : `/${path}`;
Expand All @@ -381,6 +470,10 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) {
const fetchImpl = opts.fetchImpl ?? fetch;
const timeoutMs = opts.timeoutMs ?? 10_000;
const attempts = opts.attempts ?? 3;
const retries = opts.retries ?? 4;
const delays = fibonacciDelays(retries, opts.retryBaseMs ?? 500, 10_000);
const sleep = opts.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
const lastGood = opts.lastGood === undefined ? new LastGood() : opts.lastGood;
const log = opts.log ?? (() => {});

return async function handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
Expand Down Expand Up @@ -409,56 +502,91 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) {
if (!site) return send(421, `${host} does not point at a c0mpute site`);

const path = (req.url || "/").startsWith("/") ? (req.url || "/") : "/";
for (const gw of opts.gateways.pick(attempts)) {
let upstream: Response;
try {
upstream = await fetchImpl(upstreamUrl(gw, site, path), {
method: req.method,
headers: {
...(req.headers["if-none-match"] ? { "if-none-match": String(req.headers["if-none-match"]) } : {}),
...(req.headers["accept-encoding"] ? { "accept-encoding": String(req.headers["accept-encoding"]) } : {}),
},
redirect: "manual",
signal: AbortSignal.timeout(timeoutMs),
});
} catch (e) {
opts.gateways.failed(gw);
log(`[edge] ${host} via ${gw}: ${(e as Error).message}`);
continue;
}
if (upstream.status >= 500) {
await upstream.body?.cancel();
opts.gateways.failed(gw);
log(`[edge] ${host} via ${gw}: ${upstream.status}`);
continue;
}
opts.gateways.succeeded(gw);
const headers: Record<string, string> = { "x-c0mpute-gateway": new URL(gw).host };
for (const h of PASS_HEADERS) {
const v = upstream.headers.get(h);
if (v !== null) headers[h] = v;
}
res.writeHead(upstream.status, headers);
if (req.method === "HEAD" || !upstream.body) {
await upstream.body?.cancel();
res.end();
return;
}
const reader = upstream.body.getReader();
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
if (!res.write(value)) await new Promise((r) => res.once("drain", r));
const key = LastGood.key(site, path);
const conditional = Boolean(req.headers["if-none-match"]);

// One pass over the gateways; true when a response was sent.
const tryRound = async (): Promise<boolean> => {
for (const gw of opts.gateways.pick(attempts)) {
let upstream: Response;
try {
upstream = await fetchImpl(upstreamUrl(gw, site, path), {
method: req.method,
headers: {
...(conditional ? { "if-none-match": String(req.headers["if-none-match"]) } : {}),
...(req.headers["accept-encoding"] ? { "accept-encoding": String(req.headers["accept-encoding"]) } : {}),
},
redirect: "manual",
signal: AbortSignal.timeout(timeoutMs),
});
} catch (e) {
opts.gateways.failed(gw);
log(`[edge] ${host} via ${gw}: ${(e as Error).message}`);
continue;
}
if (upstream.status >= 500) {
await upstream.body?.cancel();
opts.gateways.failed(gw);
log(`[edge] ${host} via ${gw}: ${upstream.status}`);
continue;
}
opts.gateways.succeeded(gw);
const headers: Record<string, string> = { "x-c0mpute-gateway": new URL(gw).host };
for (const h of PASS_HEADERS) {
const v = upstream.headers.get(h);
if (v !== null) headers[h] = v;
}
res.writeHead(upstream.status, headers);
if (req.method === "HEAD" || !upstream.body) {
await upstream.body?.cancel();
res.end();
return true;
}
// Keep a copy of a full 200 for the day every gateway is down.
const keep = lastGood && upstream.status === 200 && !headers["content-encoding"] ? [] as Buffer[] : null;
let kept = 0;
const reader = upstream.body.getReader();
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
if (keep) {
kept += value.length;
if (kept <= lastGood!.maxEntryBytes) keep.push(Buffer.from(value));
}
if (!res.write(value)) await new Promise((r) => res.once("drain", r));
}
} catch {
res.destroy();
return true;
}
} catch {
res.destroy();
return;
res.end();
if (keep && kept <= lastGood!.maxEntryBytes) lastGood!.put(key, 200, headers, Buffer.concat(keep));
return true;
}
res.end();
return false;
};

if (await tryRound()) return;
for (const ms of delays) {
if (res.destroyed || req.socket.destroyed) return;
await sleep(ms);
if (await tryRound()) return;
}

const stale = lastGood?.get(key);
if (stale) {
log(`[edge] ${host}${path}: every gateway failed ${delays.length + 1} rounds; serving the last good copy`);
const headers: Record<string, string> = { ...stale.headers, "x-c0mpute-stale": "1",
"cache-control": "no-store", "content-length": String(stale.body.length) };
delete headers["content-encoding"];
res.writeHead(stale.status, headers);
res.end(req.method === "HEAD" ? undefined : stale.body);
return;
}
send(502, "no c0mpute gateway answered for this site; try again shortly");
log(`[edge] ${host}${path}: every gateway failed ${delays.length + 1} rounds; no copy kept`);
res.writeHead(503, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store", "retry-after": "30" });
res.end(req.method === "HEAD" ? undefined : FAIL_PAGE);
};
}

Expand Down
57 changes: 52 additions & 5 deletions tests/c0mpute-edge.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ import {
GatewayPool,
SiteLookup,
createEdgeHandler,
fibonacciDelay,
fibonacciDelays,
LastGood,
gatewaysFromNodes,
normalizeGateway,
normalizeHost,
Expand Down Expand Up @@ -394,14 +397,16 @@ test("https://<name> is served from a healthy gateway with a registry-signed lea
});
});

test("plain http serves the same sites, GET and HEAD only, 502 when no gateway answers", async (t) => {
test("plain http serves the same sites, GET and HEAD only, stale then 503 when no gateway answers", async (t) => {
const sites = { site: async (name: string) => (name === "blue.eggs" ? SITE : null) };
let failures = 0;
const gateways = { pick: () => ["http://g1:7777", "http://g2:7777"], succeeded: () => {}, failed: () => { failures++; } };
let up = true;
const slept: number[] = [];
const handler = createEdgeHandler({
sites,
gateways,
sleep: async (ms) => { slept.push(ms); },
fetchImpl: (async (input: string | URL | Request) => {
if (!up) throw new Error("ECONNREFUSED");
if (String(input).startsWith("http://g1")) return new Response("bad", { status: 502 });
Expand All @@ -411,9 +416,9 @@ test("plain http serves the same sites, GET and HEAD only, 502 when no gateway a
const srv = await listen((req, res) => { void handler(req, res); });
t.after(() => srv.server.close());
const port = Number(new URL(srv.url).port);
const call = (method: string, host: string) => new Promise<{ status: number; text: string }>((resolve, reject) => {
const req = httpRequest({ host: "127.0.0.1", port, method, path: "/", headers: { host } }, (res) => {
let text = ""; res.on("data", (d) => { text += d; }); res.on("end", () => resolve({ status: res.statusCode ?? 0, text }));
const call = (method: string, host: string, path = "/") => new Promise<{ status: number; text: string; stale?: string }>((resolve, reject) => {
const req = httpRequest({ host: "127.0.0.1", port, method, path, headers: { host } }, (res) => {
let text = ""; res.on("data", (d) => { text += d; }); res.on("end", () => resolve({ status: res.statusCode ?? 0, text, ...(res.headers["x-c0mpute-stale"] ? { stale: String(res.headers["x-c0mpute-stale"]) } : {}) }));
});
req.on("error", reject); req.end();
});
Expand All @@ -425,5 +430,47 @@ test("plain http serves the same sites, GET and HEAD only, 502 when no gateway a
assert.equal((await call("GET", "red.eggs")).status, 421, "not a c0mpute name: misdirected, not a missing page");
assert.equal((await call("GET", "127.0.0.1")).status, 400);
up = false;
assert.equal((await call("GET", "blue.eggs")).status, 502);
assert.deepEqual(await call("GET", "blue.eggs"), { status: 200, text: "hello", stale: "1" }, "the last good copy, marked stale");
assert.deepEqual(slept, [500, 500, 1000, 1500], "retried on a Fibonacci schedule first");
const miss = await call("GET", "blue.eggs", "/never-seen");
assert.equal(miss.status, 503, "nothing kept: a graceful 503, not a bare 502");
assert.match(miss.text, /back in a moment/);
});

test("a gateway that comes back mid-retry serves the request", async () => {
let calls = 0;
const handler = createEdgeHandler({
sites: { site: async () => SITE },
gateways: { pick: () => ["http://g1:7777"], succeeded: () => {}, failed: () => {} },
sleep: async () => {},
fetchImpl: (async () => {
calls++;
if (calls < 3) throw new Error("ECONNREFUSED");
return new Response("back", { status: 200 });
}) as typeof fetch,
});
const srv = await listen((req, res) => { void handler(req, res); });
const port = Number(new URL(srv.url).port);
const status = await new Promise<number>((resolve, reject) => {
const req = httpRequest({ host: "127.0.0.1", port, path: "/", headers: { host: "blue.eggs" } }, (res) => { res.resume(); res.on("end", () => resolve(res.statusCode ?? 0)); });
req.on("error", reject); req.end();
});
srv.server.close();
assert.equal(status, 200);
assert.equal(calls, 3);
});

test("fibonacci delays and the LRU of last good pages", () => {
assert.deepEqual(fibonacciDelays(7, 100), [100, 100, 200, 300, 500, 800, 1300]);
assert.deepEqual(fibonacciDelays(4, 5000, 9000), [5000, 5000, 9000, 9000]);
assert.equal(fibonacciDelay(5, 5000, 300_000), 25_000);
const lg = new LastGood({ maxEntryBytes: 4, maxBytes: 6 });
lg.put("a", 200, {}, Buffer.from("aaa"));
lg.put("b", 200, {}, Buffer.from("bbb"));
lg.put("big", 200, {}, Buffer.from("too big"));
assert.equal(lg.get("big"), null, "over the per-page cap: not kept");
lg.get("a");
lg.put("c", 200, {}, Buffer.from("ccc"));
assert.equal(lg.get("b"), null, "least recently used went first");
assert.ok(lg.get("a") && lg.get("c"));
});
Loading