Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 87 additions & 3 deletions dev2/dev2-edge
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@ How it works (agreed with Anthony 2026-10-19, "pit.world first"):
visitor :443 -> nginx stream {} (ssl_preread, SNI only, TLS not terminated)
pit.world, *.pit.world, Moshpit names in edge.json -> 127.0.0.1:8453 Caddy
everything else -> 127.0.0.1:8444 nginx http
both hops carry a PROXY protocol header, so nginx ($remote_addr via real_ip_header
other Moshpit-shaped names (label.ending, ending not an IANA TLD)
-> c0mpute_edge.tls (moshpit-proxy's c0mpute edge), which terminates the
names the registry says are c0mpute sites and splices every other
connection, PROXY header included, back to 127.0.0.1:8444
every hop carries a PROXY protocol header, so nginx ($remote_addr via real_ip_header
proxy_protocol) and Caddy (proxy_protocol listener wrapper) see the visitor's address.
Port 80 is untouched: nginx keeps answering ACME and http->https for every name.
Port 80 stays on nginx (ACME, http->https). The one addition: Moshpit-shaped names
with no :80 vhost of their own are proxied to the c0mpute edge's plain http port.

Certificates are never issued by Caddy (auto_https off). caddy-edge-certs copies what
certbot, pit-proxy-cert and moshpit-renew already keep current into bundles Caddy
Expand All @@ -38,6 +43,7 @@ import datetime as dt
import hashlib
import json
import os
import re
import shlex
import shutil
import subprocess
Expand Down Expand Up @@ -86,6 +92,32 @@ def stream_pattern(host):
return host


TLDS = os.path.join(EDGE, "iana-tlds.txt")


def iana_tlds(path=TLDS):
"""Every top-level domain IANA delegates (data.iana.org/TLD/tlds-alpha-by-domain.txt)."""
with open(path) as f:
tlds = [l.strip().lower() for l in f if l.strip() and not l.startswith("#")]
if len(tlds) < 1000:
die(f"{path}: only {len(tlds)} TLDs, refusing to treat the rest of the namespace as Moshpit")
return tlds


# Moshpit names are label.ending; an ending is never a real TLD. nginx cannot hold a
# negative-lookahead over 1,400 TLDs in one token (4 KiB limit), so the real TLDs are
# map entries (`.com` covers com and *.com, and a wildcard beats any regex) and a short
# regex catches the two-label names left over.
MOSHPIT_NAME_RE = r"~^[a-z0-9-]+\.[a-z0-9-]+$"
TLD_MAP_DIR = "etc/nginx/dev2-edge"


def render_tld_map(value):
rows = [HEADER.rstrip("\n"), f"# every IANA TLD ({len(iana_tlds())}), from dev2/edge/iana-tlds.txt: never a Moshpit name"]
rows += [f".{t} {value};" for t in iana_tlds()]
return "\n".join(rows) + "\n"


def render_stream(s, on=True):
p = s["ports"]
caddy, nginx = f"127.0.0.1:{p['caddy']}", f"127.0.0.1:{p['nginx']}"
Expand All @@ -102,13 +134,23 @@ def render_stream(s, on=True):
if h.startswith("//"):
continue
rows.append(f" {h:<32} {nginx}; # {why[:90]}")
ce = s.get("c0mpute_edge")
if ce:
# Last, and a regex: nginx tries exact names and wildcards first, so every
# name listed above (Caddy's Moshpit names included) keeps its route.
rows.append(f" include /{TLD_MAP_DIR}/iana-tlds.stream.map; # real TLDs stay on nginx")
rows.append(" # every other Moshpit name: the c0mpute edge (it splices non-c0mpute names back to nginx)")
rows.append(f' "{MOSHPIT_NAME_RE}" 127.0.0.1:{ce["tls"]};')
state = "on: the names below go to Caddy" if on else "off: every name goes to nginx (dev2-edge route off)"
return HEADER + f"""# The edge ({state}).
# nginx's stream module owns :443. It reads the TLS server name (ssl_preread; the TLS
# itself is not terminated here) and passes the connection on, with a PROXY protocol
# header carrying the visitor's address, to Caddy or to nginx's own https servers.
stream {{
map $ssl_preread_server_name $dev2_edge_backend {{
""" + (""" # ~1,500 keys with the real-TLD map: more than the default 2048/64 hash holds.
map_hash_max_size 4096;
map_hash_bucket_size 128;
""" if on and s.get("c0mpute_edge") else "") + f""" map $ssl_preread_server_name $dev2_edge_backend {{
hostnames;
default {nginx};
""" + "\n".join(rows) + ("\n" if rows else "") + f""" }}
Expand All @@ -128,6 +170,44 @@ stream {{
"""


def render_c0mpute_http(s):
ce = s.get("c0mpute_edge")
if not ce:
return None
return HEADER + f"""# Plain http for Moshpit names pointed at c0mpute sites: the c0mpute edge serves them on
# 127.0.0.1:{ce['http']}. A regex server_name, so every name with its own :80 vhost (an
# exact or wildcard server_name) keeps it. Everything else this regex catches gets what
# the port-80 catch-all in userdirs.conf (cli-tools/root-ubuntu.sh) gave it: ACME
# challenges, an https upgrade for a real domain, and parking for a Moshpit name the edge
# says is not a c0mpute site (its 421). $moshpit_parked comes from that tool's
# conf.d/moshpit-parking.conf; without it `nginx -t` fails and apply changes nothing.
server {{
listen 80;
listen [::]:80;
server_name "{MOSHPIT_NAME_RE}";

location ^~ /.well-known/acme-challenge/ {{
root {ce.get('acme_webroot', '/var/www/acme')};
default_type "text/plain";
}}

location / {{
if ($moshpit_parked = 0) {{ return 301 https://$host$request_uri; }}
proxy_pass http://127.0.0.1:{ce['http']};
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_http_version 1.1;
proxy_intercept_errors on;
error_page 421 = @parked;
}}

location @parked {{
return 302 {ce.get('parking', 'https://pit.moshcode.sh/n/')}$host;
}}
}}
"""


def render_realip(s):
return HEADER + f"""# Every https server listens on 127.0.0.1:{s['ports']['nginx']} behind the edge stream proxy,
# which sends the visitor's address in a PROXY protocol header. Without these two lines
Expand Down Expand Up @@ -242,6 +322,10 @@ def render(out, s, on=True):
"etc/caddy/Caddyfile": render_caddyfile(s),
"etc/caddy/edge-cert-sources": render_cert_sources(s),
}
c0http = render_c0mpute_http(s)
if c0http:
files["etc/nginx/conf.d/c0mpute-edge-http.conf"] = c0http
files[f"{TLD_MAP_DIR}/iana-tlds.stream.map"] = render_tld_map(f"127.0.0.1:{s['ports']['nginx']}")
for u in UNITS:
with open(os.path.join(EDGE, "systemd", u)) as f:
files[f"etc/systemd/system/{u}"] = f.read()
Expand Down
16 changes: 13 additions & 3 deletions dev2/edge/edge-box
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ BACKUPS = "/home/anthony/backups"
NGINX_CONF = "/etc/nginx/nginx.conf"
STREAM_DIR = "/etc/nginx/stream-enabled"
STREAM_CONF = f"{STREAM_DIR}/dev2-edge.conf"
C0MPUTE_HTTP = "etc/nginx/conf.d/c0mpute-edge-http.conf"
# The real-TLD map the stream map includes, then the :80 server: installed before the stream map.
C0MPUTE_FILES = ["etc/nginx/dev2-edge/iana-tlds.stream.map", C0MPUTE_HTTP]
REALIP_CONF = "/etc/nginx/conf.d/00-edge-realip.conf"
CF_CONF = "/etc/nginx/conf.d/cloudflare-realip.conf"
DROPIN = "/etc/systemd/system/nginx.service.d/dev2-edge.conf"
Expand Down Expand Up @@ -299,8 +302,9 @@ def apply(bundle):
say(f"Caddy on 127.0.0.1:{spec['ports']['caddy']} answers like nginx for {len(checked)} names")

# ---- the switch: one nginx reload hands :443 from the http servers to the stream.
install_files(bundle, ["etc/nginx/stream-enabled/dev2-edge.conf", "etc/nginx/conf.d/00-edge-realip.conf",
"etc/systemd/system/nginx.service.d/dev2-edge.conf"])
install_files(bundle, [f for f in C0MPUTE_FILES if os.path.exists(f"{bundle}/root/{f}")]
+ ["etc/nginx/stream-enabled/dev2-edge.conf", "etc/nginx/conf.d/00-edge-realip.conf",
"etc/systemd/system/nginx.service.d/dev2-edge.conf"])
edit_cloudflare()
edit_nginx_conf(spec)
out = run(["/usr/local/sbin/nginx-edge-listen"], capture=True).stdout
Expand Down Expand Up @@ -355,9 +359,12 @@ def new_sites(spec):

def restore_live(tgz):
"""Undo a failed re-apply on a live edge: the Caddyfile, cert sources and stream map from the backup."""
keep = {"etc/caddy/Caddyfile", "etc/caddy/edge-cert-sources", "etc/nginx/stream-enabled/dev2-edge.conf"}
keep = {"etc/caddy/Caddyfile", "etc/caddy/edge-cert-sources", "etc/nginx/stream-enabled/dev2-edge.conf", C0MPUTE_HTTP}
with tarfile.open(tgz) as tf:
members = [m for m in tf.getmembers() if m.name in keep]
# Not in the backup means this run added it: take it away again.
if C0MPUTE_HTTP not in {m.name for m in members} and os.path.exists("/" + C0MPUTE_HTTP):
os.remove("/" + C0MPUTE_HTTP)
tf.extractall("/", members=members, **({"filter": "fully_trusted"} if hasattr(tarfile, "data_filter") else {}))
run("nginx -t -q", capture=True)
run("systemctl reload nginx")
Expand Down Expand Up @@ -440,6 +447,9 @@ def rollback(tgz=None):


def route(bundle):
for f in C0MPUTE_FILES[:1]: # the stream map includes the real-TLD map
if os.path.exists(f"{bundle}/root/{f}"):
put(f"{bundle}/root/{f}", "/" + f)
put(f"{bundle}/root/etc/nginx/stream-enabled/dev2-edge.conf", STREAM_CONF)
run("nginx -t -q", capture=True)
run("systemctl reload nginx")
Expand Down
3 changes: 2 additions & 1 deletion dev2/edge/edge.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
{
"//": "dev2 edge: which TLS names the nginx stream proxy hands to Caddy, and what Caddy does with them. Rendered by dev2/dev2-edge. Every upstream is the one nginx used before the move (2026-10-19). Certificates are NOT issued by Caddy: caddy-edge-certs copies the files the existing renewals (certbot, pit-proxy-cert, moshpit-renew) already keep current, and reloads Caddy when they change.",
"ports": { "nginx": 8444, "caddy": 8453, "caddy_http": 8454, "caddy_admin": 2019 },
"c0mpute_edge": { "tls": 8456, "http": 8457,
"//": "moshpit-proxy's c0mpute edge (systemd moshpit-c0mpute-edge, EDGE_PROXY_PROTOCOL=1, EDGE_FALLBACK=127.0.0.1:8444): every Moshpit-shaped name not listed here goes to it; it serves the c0mpute sites (edge.c0mpute.com is MOSHPIT_C0MPUTE_EDGE on the registry) and splices the rest back to nginx. iana-tlds.txt is what counts as not Moshpit." },
"caddy_version": "2.11.7",
"stream_timeout": "24h",
"nginx": { "worker_connections": 4096, "worker_rlimit_nofile": 16384 },
Expand Down Expand Up @@ -69,7 +71,6 @@
{ "hosts": ["offtherecord.podcast", "otr.podcast"], "proxy": 8814, "max_body": "1MiB", "moshpit": true, "//": "Off The Record (moshcoder/offtherecord): pages, podcast feed, mp3 with ranges; the app 301s otr.podcast to offtherecord.podcast.pit.world" },
{ "hosts": ["search.peptides"], "proxy": 8802, "max_body": "1MiB", "moshpit": true },
{ "hosts": ["seo.rank"], "proxy": 8790, "max_body": "64KiB", "moshpit": true },
{ "hosts": ["alt.2600"], "root": "/var/www/alt.2600", "moshpit": true, "access_log": true },
{ "hosts": ["auto.hacker"], "proxy": 8793, "max_body": "1MiB", "moshpit": true },
{ "hosts": ["chovy.hacker"], "root": "/var/www/chovy.hacker", "moshpit": true, "access_log": true },
{ "hosts": ["profullstack.agent"], "root": "/var/www/profullstack.agent", "moshpit": true, "access_log": true }
Expand Down
Loading
Loading