c0mpute edge: share :443 behind an SNI router (PROXY protocol + fallback)

README.md

@@ -291,9 +291,33 @@ browser --TLS (Moshpit CA leaf)--> edge --HTTP--> any healthy c0mpute gateway --
 | `C0MPUTE_NODES_URL` | `https://c0mpute.com/api/nodes` | gateway discovery; empty disables |
 | `EDGE_CERT_DIR` | `/var/lib/moshpit/c0mpute-edge` | |
 | `EDGE_HTTPS_PORT` / `EDGE_HTTP_PORT` | `443` / `80` | `EDGE_HTTP_PORT=0` disables http |
+| `EDGE_HOST` | `::` | listen address (`127.0.0.1` behind a router) |
+| `EDGE_PROXY_PROTOCOL` | off | `1`: expect a PROXY protocol header (v1 or v2) on https |
+| `EDGE_FALLBACK` | | `host:port` to splice every TLS connection that is not a c0mpute name to |
 
-It needs 80 and 443 to itself: run it on its own box or address, not behind the
+It wants 80 and 443 to itself: run it on its own box or address, not behind the
 origin nginx (which would answer with its own default certificate).
+
+**Sharing :443 with an origin.** On a box whose :443 already belongs to an SNI
+router (nginx `stream {}` with `ssl_preread`), the router cannot know which
+Moshpit names are c0mpute sites, since any holder can point one at c0mpute at
+any time. Route every Moshpit-shaped name the router has no other route for to
+the edge, and give the edge the old destination as `EDGE_FALLBACK`. The edge
+reads the PROXY header and the ClientHello's server name without terminating
+anything, terminates TLS only for names the registry says are c0mpute sites,
+and splices every other connection to the fallback byte for byte, PROXY header
+included, so names the box serves itself answer exactly as before. Over plain
+http a name that is not a c0mpute site gets `421 Misdirected Request` (a
+site's own missing page is a 404), so the box's :80 server can intercept the
+421 and do what it did before, such as parking the name:
+
+```
+EDGE_HOST=127.0.0.1
+EDGE_HTTPS_PORT=8456
+EDGE_HTTP_PORT=8457          # the box's :80 server proxies Moshpit names here
+EDGE_PROXY_PROTOCOL=1
+EDGE_FALLBACK=127.0.0.1:8444 # the router's old default backend
+```
 `systemd/moshpit-c0mpute-edge.service` runs it with its variables in
 `/etc/moshpit/c0mpute-edge.env`. Several edges behind one `MOSHPIT_C0MPUTE_EDGE`
 hostname (several A/AAAA records) is how it gets redundancy.

bin/moshpit-c0mpute-edge.ts

@@ -7,6 +7,14 @@ import { realpathSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { CertStore, GatewayPool, SiteLookup, startEdge } from "../lib/c0mpute-edge.ts";
 
+/** `host:port` or `[v6]:port`, or null. */
+export function hostPort(v: string | undefined): { host: string; port: number } | null {
+  const m = String(v || "").trim().match(/^(?:\[([0-9a-fA-F:.]+)\]|([^:\s\[\]]+)):(\d{1,5})$/);
+  if (!m) return null;
+  const port = Number(m[3]);
+  return port > 0 && port < 65536 ? { host: m[1] ?? m[2], port } : null;
+}
+
 export function edgeConfig(env: Record<string, string | undefined> = process.env) {
   const int = (v: string | undefined, d: number) => (v !== undefined && /^\d+$/.test(v) ? Number(v) : d);
   return {
@@ -20,6 +28,10 @@ export function edgeConfig(env: Record<string, string | undefined> = process.env
     // The public node list, for gateways that announce a URL. Empty disables
     // discovery and the edge uses the seeds alone.
     nodesUrl: env.C0MPUTE_NODES_URL === undefined ? "https://c0mpute.com/api/nodes" : env.C0MPUTE_NODES_URL || null,
+    // Behind an SNI router (see the README): a PROXY protocol header on every
+    // https connection, and where the names that are not c0mpute sites go.
+    proxyProtocol: /^(1|true|yes|on)$/i.test(env.EDGE_PROXY_PROTOCOL || ""),
+    fallback: hostPort(env.EDGE_FALLBACK),
     refreshMs: int(env.C0MPUTE_REFRESH_SECONDS, 60) * 1000,
   };
 }
@@ -37,8 +49,9 @@ async function main() {
 
   const sites = new SiteLookup({ registryBase: cfg.registryBase });
   const certs = new CertStore({ dir: cfg.certDir, registryBase: cfg.registryBase, apiKey: cfg.apiKey, log });
-  await startEdge({ sites, gateways, certs, host: cfg.host, httpsPort: cfg.httpsPort, httpPort: cfg.httpPort, log });
-  log(`[edge] serving c0mpute sites on https :${cfg.httpsPort}${cfg.httpPort ? ` and http :${cfg.httpPort}` : ""}; ${gateways.size} gateway(s) known`);
+  await startEdge({ sites, gateways, certs, host: cfg.host, httpsPort: cfg.httpsPort, httpPort: cfg.httpPort,
+    proxyProtocol: cfg.proxyProtocol, fallback: cfg.fallback, log });
+  log(`[edge] serving c0mpute sites on https :${cfg.httpsPort}${cfg.httpPort ? ` and http :${cfg.httpPort}` : ""}; ${gateways.size} gateway(s) known${cfg.fallback ? `; other names to ${cfg.fallback.host}:${cfg.fallback.port}` : ""}`);
 }
 
 const invoked = (() => { try { return realpathSync(process.argv[1] ?? "") === fileURLToPath(import.meta.url); } catch { return false; } })();

lib/c0mpute-edge.ts

@@ -32,6 +32,8 @@ import { X509Certificate } from "node:crypto";
 import { createSecureContext } from "node:tls";
 import type { SecureContext, TLSSocket } from "node:tls";
 import { join } from "node:path";
+import type { Server as NetServer } from "node:net";
+import { createSniFront } from "./sni-front.ts";
 import { promisify } from "node:util";
 
 const run = promisify(execFile);
@@ -402,7 +404,9 @@ export function createEdgeHandler(opts: EdgeHandlerOptions) {
     }
 
     const site = await opts.sites.site(host);
-    if (!site) return send(404, `${host} does not point at a c0mpute site`);
+    // 421, not 404: "this server does not answer for that name", which a front
+    // proxy can tell apart from a c0mpute site's own 404 (and park the name).
+    if (!site) return send(421, `${host} does not point at a c0mpute site`);
 
     const path = (req.url || "/").startsWith("/") ? (req.url || "/") : "/";
     for (const gw of opts.gateways.pick(attempts)) {
@@ -466,6 +470,17 @@ export interface EdgeServerOptions extends EdgeHandlerOptions {
   httpsPort?: number;
   /** 0 disables plain HTTP. */
   httpPort?: number;
+  /**
+   * Behind an SNI router that prepends a PROXY protocol header (v1 or v2) to
+   * every connection it hands the https port.
+   */
+  proxyProtocol?: boolean;
+  /**
+   * Where to splice TLS connections for names that are not c0mpute sites,
+   * instead of refusing them: the origin that owned them before the router
+   * started sending them here. See lib/sni-front.ts.
+   */
+  fallback?: { host: string; port: number } | null;
 }
 
 /**
@@ -474,7 +489,7 @@ export interface EdgeServerOptions extends EdgeHandlerOptions {
  * reason moshpit-origin.conf gives: a visitor who has not installed the Moshpit
  * root gets a working page over http instead of a certificate error.
  */
-export async function startEdge(opts: EdgeServerOptions): Promise<{ https: HttpsServer; http: HttpServer | null; close(): Promise<void> }> {
+export async function startEdge(opts: EdgeServerOptions): Promise<{ https: HttpsServer | NetServer; http: HttpServer | null; close(): Promise<void> }> {
   const handler = createEdgeHandler(opts);
   const onRequest = (req: IncomingMessage, res: ServerResponse) => {
     handler(req, res).catch(() => { try { res.destroy(); } catch { /* gone */ } });
@@ -493,22 +508,39 @@ export async function startEdge(opts: EdgeServerOptions): Promise<{ https: Https
     },
   }, onRequest);
 
-  const listen = (server: HttpServer | HttpsServer, port: number) => new Promise<void>((resolve, reject) => {
+  // In front of the TLS server when the edge shares :443 with an origin: the
+  // TLS server itself never listens, it is handed the sockets that are ours.
+  const front = opts.proxyProtocol || opts.fallback
+    ? createSniFront({
+      proxyProtocol: Boolean(opts.proxyProtocol),
+      fallback: opts.fallback ?? null,
+      serves: async (sni) => {
+        const name = normalizeHost(sni);
+        return Boolean(name && (await opts.sites.site(name)));
+      },
+      terminate: (socket) => { https.emit("connection", socket); },
+      log: opts.log,
+    })
+    : null;
+
+  const listen = (server: HttpServer | HttpsServer | NetServer, port: number) => new Promise<void>((resolve, reject) => {
     server.once("error", reject);
     server.listen(port, opts.host ?? "::", () => { server.off("error", reject); resolve(); });
   });
 
-  await listen(https, opts.httpsPort ?? 443);
+  const outer = front ?? https;
+  await listen(outer, opts.httpsPort ?? 443);
   let http: HttpServer | null = null;
   if (opts.httpPort !== 0) {
     http = createHttpServer(onRequest);
     await listen(http, opts.httpPort ?? 80);
   }
   return {
-    https,
+    https: outer,
     http,
     close: async () => {
-      await Promise.all([https, http].filter(Boolean).map((s) => new Promise<void>((r) => s!.close(() => r()))));
+      if (front) https.close();
+      await Promise.all([outer, http].filter(Boolean).map((s) => new Promise<void>((r) => s!.close(() => r()))));
     },
   };
 }

lib/sni-front.ts

@@ -0,0 +1,152 @@
+// Sharing :443 with something else: the c0mpute edge behind an SNI router.
+//
+// A box whose :443 already belongs to an SNI router (nginx `stream {}` with
+// ssl_preread, like dev2's) can hand the edge the names it does not serve
+// itself, but the router cannot know which Moshpit names are c0mpute sites:
+// anyone can point one at c0mpute at any time. So the router sends the edge
+// every Moshpit-shaped name it has no route for, and the edge sorts them:
+//
+//   - a name the registry says is a c0mpute site is terminated here, as usual;
+//   - anything else is spliced, byte for byte, to a fallback (the origin that
+//     answered those names before), so a name the box serves itself keeps
+//     working exactly as it did.
+//
+// The router usually prepends a PROXY protocol header (v1 or v2). It is
+// dropped on the TLS path (the edge has no use for the visitor's address) and
+// passed on untouched to the fallback, which expects it.
+
+import { connect, createServer } from "node:net";
+import type { Server, Socket } from "node:net";
+
+/** Bytes of a PROXY protocol header at the start of `buf`: >0 found, 0 none, -1 need more. */
+export function proxyHeaderLength(buf: Buffer): number {
+  const V2 = Buffer.from([0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54, 0x0a]);
+  const v1 = Buffer.from("PROXY ");
+  const n = Math.min(buf.length, 12);
+  if (buf.subarray(0, n).equals(V2.subarray(0, n))) {
+    if (buf.length < 16) return -1;
+    const total = 16 + buf.readUInt16BE(14);
+    return buf.length < total ? -1 : total;
+  }
+  const m = Math.min(buf.length, 6);
+  if (buf.subarray(0, m).equals(v1.subarray(0, m))) {
+    const end = buf.indexOf("\r\n");
+    if (end === -1) return buf.length >= 107 ? 0 : -1; // v1 lines are at most 107 bytes
+    return end + 2;
+  }
+  return 0;
+}
+
+/**
+ * The server name in a TLS ClientHello at the start of `buf`.
+ * A string, null (a ClientHello without SNI, or not TLS at all), or undefined
+ * when more bytes are needed.
+ */
+export function clientHelloSni(buf: Buffer): string | null | undefined {
+  if (buf.length < 5) return undefined;
+  if (buf[0] !== 0x16) return null; // not a handshake record
+  const recLen = buf.readUInt16BE(3);
+  if (buf.length < 5 + recLen) return undefined;
+  const hs = buf.subarray(5, 5 + recLen);
+  if (hs.length < 4 || hs[0] !== 0x01) return null; // not a ClientHello
+  const hsLen = (hs[1] << 16) | (hs[2] << 8) | hs[3];
+  // A ClientHello split across records is legal but no browser sends one
+  // under 16 KiB; treat it as "no SNI" rather than reassembling.
+  if (hsLen + 4 > hs.length) return null;
+  let p = 4 + 2 + 32; // version, random
+  const need = (k: number) => p + k <= 4 + hsLen;
+  if (!need(1)) return null;
+  p += 1 + hs[p]; // session id
+  if (!need(2)) return null;
+  p += 2 + hs.readUInt16BE(p); // cipher suites
+  if (!need(1)) return null;
+  p += 1 + hs[p]; // compression methods
+  if (!need(2)) return null;
+  const extEnd = p + 2 + hs.readUInt16BE(p);
+  p += 2;
+  while (p + 4 <= extEnd && p + 4 <= hs.length) {
+    const type = hs.readUInt16BE(p);
+    const len = hs.readUInt16BE(p + 2);
+    p += 4;
+    if (type === 0x0000) {
+      // server_name_list: u16 length, then (u8 type, u16 len, name)*
+      let q = p + 2;
+      while (q + 3 <= p + len) {
+        const kind = hs[q];
+        const nlen = hs.readUInt16BE(q + 1);
+        if (kind === 0) return hs.subarray(q + 3, q + 3 + nlen).toString("latin1");
+        q += 3 + nlen;
+      }
+      return null;
+    }
+    p += len;
+  }
+  return null;
+}
+
+export interface SniFrontOptions {
+  /** Expect a PROXY protocol header before the ClientHello. */
+  proxyProtocol: boolean;
+  /** Whether this edge terminates `servername` (a c0mpute site). */
+  serves: (servername: string | null) => Promise<boolean>;
+  /** Where every other connection goes, PROXY header included. Null closes them. */
+  fallback: { host: string; port: number } | null;
+  /** Hand a socket (header stripped, ClientHello unread) to the TLS server. */
+  terminate: (socket: Socket) => void;
+  timeoutMs?: number;
+  log?: (line: string) => void;
+}
+
+const MAX_PEEK = 5 + 16384 + 232; // one full TLS record plus the largest PROXY header
+
+/** A plain TCP server that reads the PROXY header and SNI, then terminates or splices. */
+export function createSniFront(opts: SniFrontOptions): Server {
+  const timeoutMs = opts.timeoutMs ?? 10_000;
+  const log = opts.log ?? (() => {});
+
+  return createServer((socket) => {
+    let buf = Buffer.alloc(0);
+    let decided = false;
+    const timer = setTimeout(() => { if (!decided) socket.destroy(); }, timeoutMs);
+    socket.on("error", () => socket.destroy());
+
+    const onData = (chunk: Buffer) => {
+      buf = Buffer.concat([buf, chunk]);
+      const hdr = opts.proxyProtocol ? proxyHeaderLength(buf) : 0;
+      if (hdr === -1 && buf.length < MAX_PEEK) return;
+      const rest = buf.subarray(Math.max(hdr, 0));
+      const sni = clientHelloSni(rest);
+      if (sni === undefined && buf.length < MAX_PEEK) return;
+      decided = true;
+      clearTimeout(timer);
+      socket.off("data", onData);
+      socket.pause();
+      route(socket, buf, rest, sni ?? null);
+    };
+    socket.on("data", onData);
+  });
+
+  function route(socket: Socket, all: Buffer, rest: Buffer, sni: string | null) {
+    opts.serves(sni).catch(() => false).then((mine) => {
+      if (socket.destroyed) return;
+      if (mine) {
+        socket.unshift(rest);
+        opts.terminate(socket);
+        return;
+      }
+      if (!opts.fallback) { socket.destroy(); return; }
+      const up = connect(opts.fallback.port, opts.fallback.host);
+      up.setNoDelay(true);
+      up.on("error", (e) => { log(`[edge] fallback for ${sni ?? "(no sni)"}: ${e.message}`); socket.destroy(); });
+      socket.on("error", () => up.destroy());
+      socket.on("close", () => up.destroy());
+      up.on("close", () => socket.destroy());
+      up.once("connect", () => {
+        up.write(all);
+        socket.pipe(up);
+        up.pipe(socket);
+        socket.resume();
+      });
+    });
+  }
+}