diff --git a/README.md b/README.md index f1f45a9..84c9670 100644 --- a/README.md +++ b/README.md @@ -291,9 +291,33 @@ browser --TLS (Moshpit CA leaf)--> edge --HTTP--> any healthy c0mpute gateway -- | `C0MPUTE_NODES_URL` | `https://c0mpute.com/api/nodes` | gateway discovery; empty disables | | `EDGE_CERT_DIR` | `/var/lib/moshpit/c0mpute-edge` | | | `EDGE_HTTPS_PORT` / `EDGE_HTTP_PORT` | `443` / `80` | `EDGE_HTTP_PORT=0` disables http | +| `EDGE_HOST` | `::` | listen address (`127.0.0.1` behind a router) | +| `EDGE_PROXY_PROTOCOL` | off | `1`: expect a PROXY protocol header (v1 or v2) on https | +| `EDGE_FALLBACK` | | `host:port` to splice every TLS connection that is not a c0mpute name to | -It needs 80 and 443 to itself: run it on its own box or address, not behind the +It wants 80 and 443 to itself: run it on its own box or address, not behind the origin nginx (which would answer with its own default certificate). + +**Sharing :443 with an origin.** On a box whose :443 already belongs to an SNI +router (nginx `stream {}` with `ssl_preread`), the router cannot know which +Moshpit names are c0mpute sites, since any holder can point one at c0mpute at +any time. Route every Moshpit-shaped name the router has no other route for to +the edge, and give the edge the old destination as `EDGE_FALLBACK`. The edge +reads the PROXY header and the ClientHello's server name without terminating +anything, terminates TLS only for names the registry says are c0mpute sites, +and splices every other connection to the fallback byte for byte, PROXY header +included, so names the box serves itself answer exactly as before. Over plain +http a name that is not a c0mpute site gets `421 Misdirected Request` (a +site's own missing page is a 404), so the box's :80 server can intercept the +421 and do what it did before, such as parking the name: + +``` +EDGE_HOST=127.0.0.1 +EDGE_HTTPS_PORT=8456 +EDGE_HTTP_PORT=8457 # the box's :80 server proxies Moshpit names here +EDGE_PROXY_PROTOCOL=1 +EDGE_FALLBACK=127.0.0.1:8444 # the router's old default backend +``` `systemd/moshpit-c0mpute-edge.service` runs it with its variables in `/etc/moshpit/c0mpute-edge.env`. Several edges behind one `MOSHPIT_C0MPUTE_EDGE` hostname (several A/AAAA records) is how it gets redundancy. diff --git a/bin/moshpit-c0mpute-edge.ts b/bin/moshpit-c0mpute-edge.ts index 413205e..d388800 100644 --- a/bin/moshpit-c0mpute-edge.ts +++ b/bin/moshpit-c0mpute-edge.ts @@ -7,6 +7,14 @@ import { realpathSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { CertStore, GatewayPool, SiteLookup, startEdge } from "../lib/c0mpute-edge.ts"; +/** `host:port` or `[v6]:port`, or null. */ +export function hostPort(v: string | undefined): { host: string; port: number } | null { + const m = String(v || "").trim().match(/^(?:\[([0-9a-fA-F:.]+)\]|([^:\s\[\]]+)):(\d{1,5})$/); + if (!m) return null; + const port = Number(m[3]); + return port > 0 && port < 65536 ? { host: m[1] ?? m[2], port } : null; +} + export function edgeConfig(env: Record = process.env) { const int = (v: string | undefined, d: number) => (v !== undefined && /^\d+$/.test(v) ? Number(v) : d); return { @@ -20,6 +28,10 @@ export function edgeConfig(env: Record = process.env // The public node list, for gateways that announce a URL. Empty disables // discovery and the edge uses the seeds alone. nodesUrl: env.C0MPUTE_NODES_URL === undefined ? "https://c0mpute.com/api/nodes" : env.C0MPUTE_NODES_URL || null, + // Behind an SNI router (see the README): a PROXY protocol header on every + // https connection, and where the names that are not c0mpute sites go. + proxyProtocol: /^(1|true|yes|on)$/i.test(env.EDGE_PROXY_PROTOCOL || ""), + fallback: hostPort(env.EDGE_FALLBACK), refreshMs: int(env.C0MPUTE_REFRESH_SECONDS, 60) * 1000, }; } @@ -37,8 +49,9 @@ async function main() { const sites = new SiteLookup({ registryBase: cfg.registryBase }); const certs = new CertStore({ dir: cfg.certDir, registryBase: cfg.registryBase, apiKey: cfg.apiKey, log }); - await startEdge({ sites, gateways, certs, host: cfg.host, httpsPort: cfg.httpsPort, httpPort: cfg.httpPort, log }); - log(`[edge] serving c0mpute sites on https :${cfg.httpsPort}${cfg.httpPort ? ` and http :${cfg.httpPort}` : ""}; ${gateways.size} gateway(s) known`); + await startEdge({ sites, gateways, certs, host: cfg.host, httpsPort: cfg.httpsPort, httpPort: cfg.httpPort, + proxyProtocol: cfg.proxyProtocol, fallback: cfg.fallback, log }); + log(`[edge] serving c0mpute sites on https :${cfg.httpsPort}${cfg.httpPort ? ` and http :${cfg.httpPort}` : ""}; ${gateways.size} gateway(s) known${cfg.fallback ? `; other names to ${cfg.fallback.host}:${cfg.fallback.port}` : ""}`); } const invoked = (() => { try { return realpathSync(process.argv[1] ?? "") === fileURLToPath(import.meta.url); } catch { return false; } })(); diff --git a/lib/c0mpute-edge.ts b/lib/c0mpute-edge.ts index 04f3a32..61bac04 100644 --- a/lib/c0mpute-edge.ts +++ b/lib/c0mpute-edge.ts @@ -32,6 +32,8 @@ import { X509Certificate } from "node:crypto"; import { createSecureContext } from "node:tls"; import type { SecureContext, TLSSocket } from "node:tls"; import { join } from "node:path"; +import type { Server as NetServer } from "node:net"; +import { createSniFront } from "./sni-front.ts"; import { promisify } from "node:util"; const run = promisify(execFile); @@ -402,7 +404,9 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) { } const site = await opts.sites.site(host); - if (!site) return send(404, `${host} does not point at a c0mpute site`); + // 421, not 404: "this server does not answer for that name", which a front + // proxy can tell apart from a c0mpute site's own 404 (and park the name). + if (!site) return send(421, `${host} does not point at a c0mpute site`); const path = (req.url || "/").startsWith("/") ? (req.url || "/") : "/"; for (const gw of opts.gateways.pick(attempts)) { @@ -466,6 +470,17 @@ export interface EdgeServerOptions extends EdgeHandlerOptions { httpsPort?: number; /** 0 disables plain HTTP. */ httpPort?: number; + /** + * Behind an SNI router that prepends a PROXY protocol header (v1 or v2) to + * every connection it hands the https port. + */ + proxyProtocol?: boolean; + /** + * Where to splice TLS connections for names that are not c0mpute sites, + * instead of refusing them: the origin that owned them before the router + * started sending them here. See lib/sni-front.ts. + */ + fallback?: { host: string; port: number } | null; } /** @@ -474,7 +489,7 @@ export interface EdgeServerOptions extends EdgeHandlerOptions { * reason moshpit-origin.conf gives: a visitor who has not installed the Moshpit * root gets a working page over http instead of a certificate error. */ -export async function startEdge(opts: EdgeServerOptions): Promise<{ https: HttpsServer; http: HttpServer | null; close(): Promise }> { +export async function startEdge(opts: EdgeServerOptions): Promise<{ https: HttpsServer | NetServer; http: HttpServer | null; close(): Promise }> { const handler = createEdgeHandler(opts); const onRequest = (req: IncomingMessage, res: ServerResponse) => { handler(req, res).catch(() => { try { res.destroy(); } catch { /* gone */ } }); @@ -493,22 +508,39 @@ export async function startEdge(opts: EdgeServerOptions): Promise<{ https: Https }, }, onRequest); - const listen = (server: HttpServer | HttpsServer, port: number) => new Promise((resolve, reject) => { + // In front of the TLS server when the edge shares :443 with an origin: the + // TLS server itself never listens, it is handed the sockets that are ours. + const front = opts.proxyProtocol || opts.fallback + ? createSniFront({ + proxyProtocol: Boolean(opts.proxyProtocol), + fallback: opts.fallback ?? null, + serves: async (sni) => { + const name = normalizeHost(sni); + return Boolean(name && (await opts.sites.site(name))); + }, + terminate: (socket) => { https.emit("connection", socket); }, + log: opts.log, + }) + : null; + + const listen = (server: HttpServer | HttpsServer | NetServer, port: number) => new Promise((resolve, reject) => { server.once("error", reject); server.listen(port, opts.host ?? "::", () => { server.off("error", reject); resolve(); }); }); - await listen(https, opts.httpsPort ?? 443); + const outer = front ?? https; + await listen(outer, opts.httpsPort ?? 443); let http: HttpServer | null = null; if (opts.httpPort !== 0) { http = createHttpServer(onRequest); await listen(http, opts.httpPort ?? 80); } return { - https, + https: outer, http, close: async () => { - await Promise.all([https, http].filter(Boolean).map((s) => new Promise((r) => s!.close(() => r())))); + if (front) https.close(); + await Promise.all([outer, http].filter(Boolean).map((s) => new Promise((r) => s!.close(() => r())))); }, }; } diff --git a/lib/sni-front.ts b/lib/sni-front.ts new file mode 100644 index 0000000..68250c9 --- /dev/null +++ b/lib/sni-front.ts @@ -0,0 +1,152 @@ +// Sharing :443 with something else: the c0mpute edge behind an SNI router. +// +// A box whose :443 already belongs to an SNI router (nginx `stream {}` with +// ssl_preread, like dev2's) can hand the edge the names it does not serve +// itself, but the router cannot know which Moshpit names are c0mpute sites: +// anyone can point one at c0mpute at any time. So the router sends the edge +// every Moshpit-shaped name it has no route for, and the edge sorts them: +// +// - a name the registry says is a c0mpute site is terminated here, as usual; +// - anything else is spliced, byte for byte, to a fallback (the origin that +// answered those names before), so a name the box serves itself keeps +// working exactly as it did. +// +// The router usually prepends a PROXY protocol header (v1 or v2). It is +// dropped on the TLS path (the edge has no use for the visitor's address) and +// passed on untouched to the fallback, which expects it. + +import { connect, createServer } from "node:net"; +import type { Server, Socket } from "node:net"; + +/** Bytes of a PROXY protocol header at the start of `buf`: >0 found, 0 none, -1 need more. */ +export function proxyHeaderLength(buf: Buffer): number { + const V2 = Buffer.from([0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54, 0x0a]); + const v1 = Buffer.from("PROXY "); + const n = Math.min(buf.length, 12); + if (buf.subarray(0, n).equals(V2.subarray(0, n))) { + if (buf.length < 16) return -1; + const total = 16 + buf.readUInt16BE(14); + return buf.length < total ? -1 : total; + } + const m = Math.min(buf.length, 6); + if (buf.subarray(0, m).equals(v1.subarray(0, m))) { + const end = buf.indexOf("\r\n"); + if (end === -1) return buf.length >= 107 ? 0 : -1; // v1 lines are at most 107 bytes + return end + 2; + } + return 0; +} + +/** + * The server name in a TLS ClientHello at the start of `buf`. + * A string, null (a ClientHello without SNI, or not TLS at all), or undefined + * when more bytes are needed. + */ +export function clientHelloSni(buf: Buffer): string | null | undefined { + if (buf.length < 5) return undefined; + if (buf[0] !== 0x16) return null; // not a handshake record + const recLen = buf.readUInt16BE(3); + if (buf.length < 5 + recLen) return undefined; + const hs = buf.subarray(5, 5 + recLen); + if (hs.length < 4 || hs[0] !== 0x01) return null; // not a ClientHello + const hsLen = (hs[1] << 16) | (hs[2] << 8) | hs[3]; + // A ClientHello split across records is legal but no browser sends one + // under 16 KiB; treat it as "no SNI" rather than reassembling. + if (hsLen + 4 > hs.length) return null; + let p = 4 + 2 + 32; // version, random + const need = (k: number) => p + k <= 4 + hsLen; + if (!need(1)) return null; + p += 1 + hs[p]; // session id + if (!need(2)) return null; + p += 2 + hs.readUInt16BE(p); // cipher suites + if (!need(1)) return null; + p += 1 + hs[p]; // compression methods + if (!need(2)) return null; + const extEnd = p + 2 + hs.readUInt16BE(p); + p += 2; + while (p + 4 <= extEnd && p + 4 <= hs.length) { + const type = hs.readUInt16BE(p); + const len = hs.readUInt16BE(p + 2); + p += 4; + if (type === 0x0000) { + // server_name_list: u16 length, then (u8 type, u16 len, name)* + let q = p + 2; + while (q + 3 <= p + len) { + const kind = hs[q]; + const nlen = hs.readUInt16BE(q + 1); + if (kind === 0) return hs.subarray(q + 3, q + 3 + nlen).toString("latin1"); + q += 3 + nlen; + } + return null; + } + p += len; + } + return null; +} + +export interface SniFrontOptions { + /** Expect a PROXY protocol header before the ClientHello. */ + proxyProtocol: boolean; + /** Whether this edge terminates `servername` (a c0mpute site). */ + serves: (servername: string | null) => Promise; + /** Where every other connection goes, PROXY header included. Null closes them. */ + fallback: { host: string; port: number } | null; + /** Hand a socket (header stripped, ClientHello unread) to the TLS server. */ + terminate: (socket: Socket) => void; + timeoutMs?: number; + log?: (line: string) => void; +} + +const MAX_PEEK = 5 + 16384 + 232; // one full TLS record plus the largest PROXY header + +/** A plain TCP server that reads the PROXY header and SNI, then terminates or splices. */ +export function createSniFront(opts: SniFrontOptions): Server { + const timeoutMs = opts.timeoutMs ?? 10_000; + const log = opts.log ?? (() => {}); + + return createServer((socket) => { + let buf = Buffer.alloc(0); + let decided = false; + const timer = setTimeout(() => { if (!decided) socket.destroy(); }, timeoutMs); + socket.on("error", () => socket.destroy()); + + const onData = (chunk: Buffer) => { + buf = Buffer.concat([buf, chunk]); + const hdr = opts.proxyProtocol ? proxyHeaderLength(buf) : 0; + if (hdr === -1 && buf.length < MAX_PEEK) return; + const rest = buf.subarray(Math.max(hdr, 0)); + const sni = clientHelloSni(rest); + if (sni === undefined && buf.length < MAX_PEEK) return; + decided = true; + clearTimeout(timer); + socket.off("data", onData); + socket.pause(); + route(socket, buf, rest, sni ?? null); + }; + socket.on("data", onData); + }); + + function route(socket: Socket, all: Buffer, rest: Buffer, sni: string | null) { + opts.serves(sni).catch(() => false).then((mine) => { + if (socket.destroyed) return; + if (mine) { + socket.unshift(rest); + opts.terminate(socket); + return; + } + if (!opts.fallback) { socket.destroy(); return; } + const up = connect(opts.fallback.port, opts.fallback.host); + up.setNoDelay(true); + up.on("error", (e) => { log(`[edge] fallback for ${sni ?? "(no sni)"}: ${e.message}`); socket.destroy(); }); + socket.on("error", () => up.destroy()); + socket.on("close", () => up.destroy()); + up.on("close", () => socket.destroy()); + up.once("connect", () => { + up.write(all); + socket.pipe(up); + up.pipe(socket); + socket.resume(); + }); + }); + } +} diff --git a/tests/c0mpute-edge.test.ts b/tests/c0mpute-edge.test.ts index 6023540..29d514f 100644 --- a/tests/c0mpute-edge.test.ts +++ b/tests/c0mpute-edge.test.ts @@ -25,7 +25,10 @@ import { startEdge, upstreamUrl, } from "../lib/c0mpute-edge.ts"; -import { edgeConfig } from "../bin/moshpit-c0mpute-edge.ts"; +import { edgeConfig, hostPort } from "../bin/moshpit-c0mpute-edge.ts"; +import { clientHelloSni, proxyHeaderLength } from "../lib/sni-front.ts"; +import { connect as netConnect, createServer as createNetServer } from "node:net"; +import { connect as tlsConnect } from "node:tls"; import { tempDir } from "./helpers.ts"; const run = promisify(execFile); @@ -86,6 +89,120 @@ test("edge config reads the environment", () => { assert.equal(c.httpPort, 0); assert.equal(edgeConfig({}).nodesUrl, "https://c0mpute.com/api/nodes"); assert.equal(edgeConfig({}).httpsPort, 443); + assert.equal(edgeConfig({}).proxyProtocol, false); + assert.equal(edgeConfig({}).fallback, null); + const behind = edgeConfig({ EDGE_PROXY_PROTOCOL: "1", EDGE_FALLBACK: "127.0.0.1:8444" }); + assert.equal(behind.proxyProtocol, true); + assert.deepEqual(behind.fallback, { host: "127.0.0.1", port: 8444 }); + assert.deepEqual(hostPort("[::1]:443"), { host: "::1", port: 443 }); + assert.equal(hostPort("127.0.0.1"), null); + assert.equal(hostPort("x:70000"), null); +}); + +/* ------------------------------------------------------- behind a router */ + +/** The first bytes a real TLS client sends for `servername`. */ +function captureClientHello(servername?: string): Promise { + return new Promise((resolve) => { + const srv = createNetServer((sock) => { + sock.once("data", (d: Buffer) => { resolve(d); sock.destroy(); srv.close(); }); + }); + srv.listen(0, "127.0.0.1", () => { + const port = (srv.address() as AddressInfo).port; + const c = tlsConnect({ host: "127.0.0.1", port, ...(servername ? { servername } : {}), rejectUnauthorized: false }); + c.on("error", () => {}); + }); + }); +} + +test("PROXY headers and the SNI are read off the front of a connection", async () => { + const v1 = Buffer.from("PROXY TCP4 1.2.3.4 5.6.7.8 1111 443\r\nrest"); + assert.equal(proxyHeaderLength(v1), v1.indexOf("rest")); + assert.equal(proxyHeaderLength(Buffer.from("PROXY TCP4 1.2")), -1, "need more"); + assert.equal(proxyHeaderLength(Buffer.from([0x16, 3, 1])), 0, "no header"); + const v2 = Buffer.concat([Buffer.from([0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54, 0x0a, 0x21, 0x11, 0, 12]), Buffer.alloc(12), Buffer.from("x")]); + assert.equal(proxyHeaderLength(v2), 28); + assert.equal(proxyHeaderLength(v2.subarray(0, 20)), -1); + + const hello = await captureClientHello("alt.2600"); + assert.equal(clientHelloSni(hello), "alt.2600"); + assert.equal(clientHelloSni(hello.subarray(0, 40)), undefined, "need more"); + assert.equal(clientHelloSni(await captureClientHello()), null, "no SNI"); + assert.equal(clientHelloSni(Buffer.from("GET / HTTP/1.1\r\n\r\n")), null, "not TLS"); +}); + +test("behind an SNI router: c0mpute names terminate here, every other name is spliced to the fallback", async (t) => { + const dir = await tempDir("moshpit-edge-front-"); + const ca = await makeCa(dir); + const good = await listen((req, res) => { + if (req.url === "/healthz") { res.writeHead(200); return res.end("ok"); } + res.writeHead(200, { "content-type": "text/plain" }); + res.end(`gateway ${req.url}`); + }); + const registry = await listen(async (req, res) => { + const url = new URL(req.url ?? "/", "http://x"); + if (url.pathname === "/api/moshpit/resolve") { + const name = url.searchParams.get("name"); + return res.end(JSON.stringify({ name, c0mpute: name === "blue.eggs" ? SITE : null })); + } + if (req.method === "POST" && url.pathname === "/api/moshpit/tlds/eggs/certs") { + const { label, csr } = JSON.parse(await body(req)); + res.writeHead(201, { "content-type": "application/json" }); + return res.end(JSON.stringify({ chain: await sign(dir, ca, `${label}.eggs`, csr) })); + } + res.writeHead(404); res.end("{}"); + }); + // The fallback records exactly what it was sent, then answers. + const seen: Buffer[] = []; + const fallback = createNetServer((sock) => { + sock.once("data", (d: Buffer) => { seen.push(d); sock.end("FALLBACK"); }); + }); + await new Promise((r) => fallback.listen(0, "127.0.0.1", () => r())); + const fbPort = (fallback.address() as AddressInfo).port; + + const gateways = new GatewayPool({ seeds: [good.url] }); + await gateways.refresh(); + const sites = new SiteLookup({ registryBase: registry.url }); + const certs = new CertStore({ dir: join(dir, "certs"), registryBase: registry.url, apiKey: "k" }); + const edge = await startEdge({ sites, gateways, certs, host: "127.0.0.1", httpsPort: 0, httpPort: 0, + proxyProtocol: true, fallback: { host: "127.0.0.1", port: fbPort } }); + const port = (edge.https.address() as AddressInfo).port; + t.after(async () => { + await edge.close(); + for (const s of [good.server, registry.server, fallback]) s.close(); + }); + const PROXY = "PROXY TCP4 203.0.113.9 127.0.0.1 40000 443\r\n"; + + await t.test("a c0mpute name, after a PROXY header, gets its leaf and its site", async () => { + const text = await new Promise((resolve, reject) => { + const raw = netConnect(port, "127.0.0.1", () => { + raw.write(PROXY); + const tls = tlsConnect({ socket: raw, servername: "blue.eggs", ca: ca.pem }, () => { + tls.write("GET /x HTTP/1.1\r\nHost: blue.eggs\r\nConnection: close\r\n\r\n"); + }); + let out = ""; + tls.on("data", (d) => { out += d; }); + tls.on("end", () => resolve(out)); + tls.on("error", reject); + }); + }); + assert.match(text, /^HTTP\/1\.1 200/); + assert.match(text, new RegExp(`gateway /s/${SITE}/x`)); + }); + + await t.test("any other name reaches the fallback byte for byte, PROXY header first", async () => { + const hello = await captureClientHello("red.eggs"); + const reply = await new Promise((resolve, reject) => { + const raw = netConnect(port, "127.0.0.1", () => { raw.write(Buffer.concat([Buffer.from(PROXY), hello])); }); + let out = ""; + raw.on("data", (d) => { out += d; }); + raw.on("end", () => resolve(out)); + raw.on("error", reject); + }); + assert.equal(reply, "FALLBACK"); + assert.equal(seen.length, 1); + assert.ok(seen[0].equals(Buffer.concat([Buffer.from(PROXY), hello])), "PROXY header and ClientHello untouched"); + }); }); /* ------------------------------------------------------------ gateways */ @@ -305,7 +422,7 @@ test("plain http serves the same sites, GET and HEAD only, 502 when no gateway a assert.equal(failures, 1, "g1's 502 counted against it"); assert.equal((await call("HEAD", "blue.eggs")).text, ""); assert.equal((await call("POST", "blue.eggs")).status, 405); - assert.equal((await call("GET", "red.eggs")).status, 404); + assert.equal((await call("GET", "red.eggs")).status, 421, "not a c0mpute name: misdirected, not a missing page"); assert.equal((await call("GET", "127.0.0.1")).status, 400); up = false; assert.equal((await call("GET", "blue.eggs")).status, 502);