fix(mail): unbreak join@ registration when the Mailu cert lapses

deploy/mailu/refresh-certs.sh

@@ -1,28 +1,60 @@
 #!/usr/bin/env bash
 #
 # refresh-certs.sh — copy Caddy's Let's Encrypt cert for mail.$DOMAIN into the
-# Mailu certs dir (TLS_FLAVOR=mail), so Postfix/Dovecot TLS on 465/587/993 track
-# Caddy's auto-renewals. Mirrors deploy/news-refresh-certs.sh: Caddy is the only
-# ACME client on the box (it serves the mail.$DOMAIN site block), and we reuse
-# that cert rather than running a second ACME client inside Mailu.
+# Mailu certs dir (TLS_FLAVOR=mail), so Postfix/Dovecot TLS on 25/465/587/993
+# track Caddy's auto-renewals. Mirrors deploy/news-refresh-certs.sh: Caddy is the
+# only ACME client on the box (it serves the mail.$DOMAIN site block), and we
+# reuse that cert rather than running a second ACME client inside Mailu.
 #
 # Install to /usr/local/bin/agentbbs-mailu-certs and run from a timer. Reloads
-# the Mailu front/smtp/imap so the new cert is picked up. Exits non-zero
-# (touching nothing) until Caddy has issued the cert.
+# the Mailu front so the new cert is picked up. Exits non-zero (touching
+# nothing) until Caddy has issued the cert.
+#
+# Copying the file is not the whole job. Mailu keeps serving whatever it loaded
+# at container start, so a copy whose reload silently failed leaves a fresh cert
+# on disk and an expiring one on the wire. That is exactly how join@ stopped
+# being able to email confirmation codes: Caddy renewed on 2026-08-14, Mailu went
+# on serving the Jun 15 cert, and registration broke when it expired on Sep 13.
+# So we also compare what the relay ACTUALLY serves against the file and force a
+# reload when they disagree, and we no longer swallow reload failures.
 set -euo pipefail
 
 DOMAIN="${DOMAIN:?set DOMAIN}"
 MAIL_HOST="${MAIL_HOST:-mail.${DOMAIN}}"
 MAILU_DIR="${MAILU_DIR:-/opt/agentbbs/deploy/mailu}"
 CERT_DIR="${CERT_DIR:-$MAILU_DIR/certs}"
 CADDY_DATA="${CADDY_DATA:-/var/lib/caddy/.local/share/caddy}"
+# Port used to read back the cert the relay is really serving (loopback SMTP).
+PROBE_ADDR="${PROBE_ADDR:-127.0.0.1:25}"
+
+# notAfter of a PEM file, or empty if it can't be read.
+cert_not_after() {
+  openssl x509 -noout -enddate -in "$1" 2>/dev/null | sed 's/^notAfter=//'
+}
+
+# notAfter of the cert the running relay serves over STARTTLS, or empty if the
+# probe can't be made (openssl missing, port closed, Mailu down).
+served_not_after() {
+  command -v openssl >/dev/null 2>&1 || return 0
+  printf 'QUIT\r\n' \
+    | timeout 10 openssl s_client -quiet -starttls smtp \
+        -connect "$PROBE_ADDR" -servername "$MAIL_HOST" 2>/dev/null \
+    | openssl x509 -noout -enddate 2>/dev/null | sed 's/^notAfter=//'
+}
 
 # Caddy stores certs under certificates/<acme-dir>/<host>/<host>.{crt,key};
 # the ACME directory segment varies (prod vs staging), so glob for it.
 crt="$(ls "$CADDY_DATA"/certificates/*/"$MAIL_HOST"/"$MAIL_HOST".crt 2>/dev/null | head -1 || true)"
 key="$(ls "$CADDY_DATA"/certificates/*/"$MAIL_HOST"/"$MAIL_HOST".key 2>/dev/null | head -1 || true)"
 if [ -z "$crt" ] || [ -z "$key" ]; then
-  echo "no Caddy cert for $MAIL_HOST yet (looked under $CADDY_DATA/certificates)"
+  echo "no Caddy cert for $MAIL_HOST yet (looked under $CADDY_DATA/certificates)" >&2
+  exit 1
+fi
+
+# A source cert that is itself expired means Caddy's renewal is broken, which is
+# a different fault and one no amount of copying fixes. Say so loudly.
+if ! openssl x509 -checkend 0 -noout -in "$crt" >/dev/null 2>&1; then
+  echo "Caddy's cert for $MAIL_HOST is EXPIRED ($(cert_not_after "$crt")) — fix Caddy's renewal; not copying" >&2
   exit 1
 fi
 
@@ -33,9 +65,21 @@ changed=0
 if ! cmp -s "$crt" "$CERT_DIR/cert.pem"; then install -m 0644 "$crt" "$CERT_DIR/cert.pem"; changed=1; fi
 if ! cmp -s "$key" "$CERT_DIR/key.pem"; then install -m 0640 "$key" "$CERT_DIR/key.pem"; changed=1; fi
 
-if [ "$changed" = 1 ]; then
-  echo "updated Mailu TLS cert for $MAIL_HOST; reloading Mailu"
-  ( cd "$MAILU_DIR" && docker compose restart front smtp imap >/dev/null 2>&1 || true )
+# The file can be current while the running container still serves an older one
+# (a reload that never happened, or failed). Trust the wire, not the filesystem.
+on_disk="$(cert_not_after "$CERT_DIR/cert.pem")"
+on_wire="$(served_not_after)"
+stale_on_wire=0
+if [ -n "$on_wire" ] && [ -n "$on_disk" ] && [ "$on_wire" != "$on_disk" ]; then
+  echo "Mailu is serving a cert that expires '$on_wire' but /certs holds one that expires '$on_disk' — forcing reload" >&2
+  stale_on_wire=1
+fi
+
+if [ "$changed" = 1 ] || [ "$stale_on_wire" = 1 ]; then
+  echo "updating Mailu TLS cert for $MAIL_HOST (expires $on_disk); reloading Mailu"
+  # No `|| true`: a reload that fails is the failure mode this whole script
+  # exists to prevent, so it must surface in `systemctl status` / the journal.
+  ( cd "$MAILU_DIR" && docker compose restart front )
 else
-  echo "Mailu TLS cert for $MAIL_HOST already current"
+  echo "Mailu TLS cert for $MAIL_HOST already current (expires $on_disk)"
 fi

docs/credentials.md

@@ -130,6 +130,7 @@ on any failure.
 | `AGENTBBS_SET_IRC_SUDO` | `1` | chat — invoke the helper via `sudo` (set `0` if the BBS already runs as root, e.g. in tests) |
 | `AGENTBBS_SMTP_HOST` / `_FROM` | unset | **sending** all of the above emails (required to actually send) |
 | `AGENTBBS_SMTP_PORT` / `_USER` / `_PASS` | `587` / unset / unset | SMTP submission (STARTTLS) |
+| `AGENTBBS_SMTP_SERVERNAME` | unset (= `_HOST`) | name STARTTLS certs are verified against when the relay is dialled on loopback; verification is skipped entirely for a loopback relay |
 
 ## Two SMTP paths (and why one is `:25`)
 

docs/mail.md

@@ -147,9 +147,37 @@ them at the local Mailu relay so codes actually send:
 AGENTBBS_SMTP_HOST=127.0.0.1
 AGENTBBS_SMTP_PORT=25
 AGENTBBS_SMTP_FROM=bbs@bbs.profullstack.com
+AGENTBBS_SMTP_SERVERNAME=mail.profullstack.com   # set by setup.sh
 # user/pass omitted: the co-located relay accepts local submission unauthenticated
 ```
 
+`AGENTBBS_SMTP_SERVERNAME` is the name STARTTLS certificates are verified
+against when it differs from the dialled host — the relay answers on
+`127.0.0.1` but presents a cert for the mail host. It mirrors
+`AGENTBBS_MAIL_SMTP_SERVERNAME` on the mailbox gateway and `setup.sh` sets it
+whenever the Mailu stack is enabled.
+
+On a **loopback** relay the sender skips certificate verification outright. The
+connection never leaves the host, so there is nothing to intercept — and tying
+`join@` registration to an on-box cert being both name-matched and unexpired is
+precisely what broke signups for nine days in September 2026 (see below).
+
+### When confirmation codes stop sending
+
+`join@` reporting *"couldn't email the code"* means `internal/mail` could not
+hand the message to the relay. The error is in the journal
+(`journalctl -u agentbbs -g "send code"`), and it now names the address and the
+failing stage. The usual cause is the mail host's TLS cert: Caddy owns ACME for
+`mail.$DOMAIN` and `deploy/mailu/refresh-certs.sh` copies it into Mailu, but
+Mailu keeps serving whatever it loaded at container start. Check what is
+actually on the wire rather than what is on disk:
+
+```bash
+printf 'QUIT\r\n' | openssl s_client -quiet -starttls smtp \
+  -connect 127.0.0.1:25 -servername mail.$DOMAIN 2>&1 | grep -i notAfter
+sudo /usr/local/bin/agentbbs-mailu-certs   # copies + reloads; loud on failure
+```
+
 ## Provisioning member mailboxes
 
 Provisioning is automatic at `join@` verification. To create or backfill by hand:

internal/mail/mail.go

@@ -5,7 +5,9 @@
 package mail
 
 import (
+	"crypto/tls"
 	"fmt"
+	"net"
 	"net/smtp"
 	"os"
 	"strings"
@@ -18,43 +20,118 @@
 	User string // auth user; empty = no auth
 	Pass string
 	From string // envelope + From: header
+
+	// ServerName is the name STARTTLS certificates are verified against when it
+	// differs from Host. A co-located relay is dialled on loopback but presents a
+	// certificate for its public mail host, never for 127.0.0.1. Mirrors
+	// AGENTBBS_MAIL_SMTP_SERVERNAME on the mailbox gateway.
+	ServerName string
 }
 
-// ConfigFromEnv reads AGENTBBS_SMTP_{HOST,PORT,USER,PASS,FROM}.
+// ConfigFromEnv reads AGENTBBS_SMTP_{HOST,PORT,USER,PASS,FROM,SERVERNAME}.
 func ConfigFromEnv() Config {
 	return Config{
-		Host: os.Getenv("AGENTBBS_SMTP_HOST"),
-		Port: os.Getenv("AGENTBBS_SMTP_PORT"),
-		User: os.Getenv("AGENTBBS_SMTP_USER"),
-		Pass: os.Getenv("AGENTBBS_SMTP_PASS"),
-		From: os.Getenv("AGENTBBS_SMTP_FROM"),
+		Host:       os.Getenv("AGENTBBS_SMTP_HOST"),
+		Port:       os.Getenv("AGENTBBS_SMTP_PORT"),
+		User:       os.Getenv("AGENTBBS_SMTP_USER"),
+		Pass:       os.Getenv("AGENTBBS_SMTP_PASS"),
+		From:       os.Getenv("AGENTBBS_SMTP_FROM"),
+		ServerName: os.Getenv("AGENTBBS_SMTP_SERVERNAME"),
 	}
 }
 
 // Configured reports whether email can actually be sent.
 func (c Config) Configured() bool { return c.Host != "" && c.From != "" }
 
-// Send delivers a plain-text message. net/smtp negotiates STARTTLS when the
-// server advertises it (the common case on :587). Implicit-TLS :465 is not
-// supported — use a STARTTLS port.
+// port is the submission port, defaulting to STARTTLS 587.
+func (c Config) port() string {
+	if c.Port == "" {
+		return "587"
+	}
+	return c.Port
+}
+
+// tlsServerName is the identity STARTTLS certificates are checked against.
+// AGENTBBS_SMTP_SERVERNAME wins; otherwise the dialled host is used, which is
+// only meaningful when that host is a real name.
+func (c Config) tlsServerName() string {
+	if c.ServerName != "" {
+		return c.ServerName
+	}
+	return c.Host
+}
+
+// isLoopback reports whether the relay lives on this host. A loopback
+// connection never leaves the box, so there is nothing for certificate
+// verification to defend against — the same reasoning docs/mail.md already
+// applies to the plaintext Dovecot hand-off.
+func isLoopback(host string) bool {
+	if strings.EqualFold(host, "localhost") {
+		return true
+	}
+	ip := net.ParseIP(strings.Trim(host, "[]"))
+	return ip != nil && ip.IsLoopback()
+}
+
+// Send delivers a plain-text message. STARTTLS is negotiated whenever the
+// server advertises it (the common case on :587 and on a co-located :25).
+// Implicit-TLS :465 is not supported — use a STARTTLS port.
+//
+// Certificates are verified against tlsServerName() EXCEPT on a loopback relay,
+// where verification is skipped deliberately. An on-box MTA serves a cert for
+// its public mail host and renews it on its own schedule; making join@ signups
+// depend on that cert being both name-matched and unexpired is how registration
+// silently died for nine days when the Mailu cert lapsed. Loopback traffic is
+// not interceptable, so the check bought nothing and cost everything.
 func (c Config) Send(to, subject, body string) error {
 	if !c.Configured() {
 		return fmt.Errorf("smtp not configured")
 	}
-	port := c.Port
-	if port == "" {
-		port = "587"
-	}
-	var auth smtp.Auth
-	if c.User != "" {
-		auth = smtp.PlainAuth("", c.User, c.Pass, c.Host)
-	}
+	addr := net.JoinHostPort(c.Host, c.port())
 	msg := "From: " + c.From + "\r\n" +
 		"To: " + to + "\r\n" +
 		"Subject: " + subject + "\r\n" +
 		"MIME-Version: 1.0\r\n" +
 		"Content-Type: text/plain; charset=utf-8\r\n" +
 		"\r\n" +
 		strings.ReplaceAll(body, "\n", "\r\n") + "\r\n"
-	return smtp.SendMail(c.Host+":"+port, auth, c.From, []string{to}, []byte(msg))
+
+	cl, err := smtp.Dial(addr)
+	if err != nil {
+		return fmt.Errorf("smtp dial %s: %w", addr, err)
+	}
+	defer func() { _ = cl.Close() }()
+
+	if ok, _ := cl.Extension("STARTTLS"); ok {
+		name := c.tlsServerName()
+		conf := &tls.Config{ServerName: name}
+		if isLoopback(c.Host) {
+			conf = &tls.Config{ServerName: name, InsecureSkipVerify: true} // #nosec G402 -- loopback relay, see doc comment
+		}
+		if err := cl.StartTLS(conf); err != nil {
+			return fmt.Errorf("smtp starttls %s (servername %q): %w", addr, name, err)
+		}
+	}
+	if c.User != "" {
+		if err := cl.Auth(smtp.PlainAuth("", c.User, c.Pass, c.tlsServerName())); err != nil {
+			return fmt.Errorf("smtp auth %s: %w", addr, err)
+		}
+	}
+	if err := cl.Mail(c.From); err != nil {
+		return fmt.Errorf("smtp mail from %s: %w", c.From, err)
+	}
+	if err := cl.Rcpt(to); err != nil {
+		return fmt.Errorf("smtp rcpt to %s: %w", to, err)
+	}
+	w, err := cl.Data()
+	if err != nil {
+		return fmt.Errorf("smtp data: %w", err)
+	}
+	if _, err := w.Write([]byte(msg)); err != nil {
+		return fmt.Errorf("smtp write: %w", err)
+	}
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("smtp close: %w", err)
+	}
+	return cl.Quit()
 }