From ad8e64858485b3296548cf2556a9c00485656587 Mon Sep 17 00:00:00 2001 From: pythonlearner1025 Date: Fri, 28 Aug 2026 05:04:41 +0000 Subject: [PATCH] fix(bake): the builder script has been truncated since it was written No golden snapshot has ever been a complete build. Three bugs, stacked so that each hid the next. 1. A backtick ends a template literal, including inside a shell comment. # `|| true` throughout: an absent unit is not a bake failure. That line closed builderUserData's template 49 lines early. What follows parses as `template || true`tagged-template``, which is valid JavaScript and short-circuits to the first string -- so `node --check` passes, the function returns, nothing throws, and the builder is handed a script a third shorter than the file looks. Lost with it: lever 2, the root-password fix, the identity strip, the marker, and `shutdown -h now`. 2. `/usr/sbin/sshd -t` fails with "Missing privilege separation directory: /run/sshd". /run is a tmpfs that only ssh.service populates, and socket activation has not started it yet at first boot. `set -Eeuo pipefail` and the ERR trap then power the builder off at line 49. This masked bug 1 completely. The truncated script had no shutdown of its own, so the trap's `shutdown -h now` was the only reason a bake ever finished at all -- and the bake reads "powered off" as "finished". 3. `chage` was never passed -d, so sp_lstchg stays at Hetzner's 0. That does not mean "old", it means "must be changed at next login", and PAM refuses a key login with "Your password has expired" however open the other aging fields are. Confirmed by booting both snapshots and reading /var/log/blitz-bake.log, which survives only because the cleanup never ran: 425047509 -- live on canary and client prod -- FAILED at line 49 425198277 -- baked today -- FAILED at line 49 Fixing 2 alone turned the symptom into "builder never powered off" after the full 30-minute timeout, which is bug 1 with nothing left to hide behind. Fixes: escape the backticks; create /run/sshd before the test; pass `chage -d "$(date +%F)"`. Verified for 3 by booting 425047509 and running only that command -- a refused login becomes a working one. assertWholeScript then refuses to spend a VM on a script missing any of its five landmarks or not ending in `shutdown -h now`, because the failure mode of bug 1 is silence. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J6fUBY1B27EzvDwbhfBf52 --- .../scripts/bake-golden-image.mjs | 59 ++++++++++++++++++- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/control-plane/scripts/bake-golden-image.mjs b/packages/control-plane/scripts/bake-golden-image.mjs index 18cdc691..c7b73add 100644 --- a/packages/control-plane/scripts/bake-golden-image.mjs +++ b/packages/control-plane/scripts/bake-golden-image.mjs @@ -110,13 +110,20 @@ install -d -m 0755 /etc/ssh/sshd_config.d cat >/etc/ssh/sshd_config.d/00-blitz.conf <<'SSHD_CONFIG' Port 2222 SSHD_CONFIG +# sshd -t refuses to run without its privilege separation directory, and /run +# is a tmpfs that only ssh.service populates -- which socket activation has not +# started yet at this point in first boot. Without this line the test exits 255 +# on "Missing privilege separation directory: /run/sshd", the ERR trap fires, +# and the builder powers off HERE: after the expensive image pull and before +# every line below. That is what produced the first two golden snapshots. +install -d -m 0755 /run/sshd /usr/sbin/sshd -t systemctl disable ssh.socket 2>/dev/null || true systemctl mask ssh.socket systemctl enable ssh # Lever 2: units a workspace never uses, and which cost seconds of every boot. -# `|| true` throughout: an absent unit is not a bake failure. +# \`|| true\` throughout: an absent unit is not a bake failure. for unit in snapd.service snapd.socket snapd.seeded.service unattended-upgrades.service \ multipathd.service multipathd.socket apt-daily.timer apt-daily-upgrade.timer \ motd-news.timer man-db.timer e2scrub_all.timer fstrim.timer; do @@ -128,7 +135,13 @@ done # host cannot be debugged. Lock the password instead: no password login, no # expiry, keys still work. usermod -p '*' root -chage -I -1 -m 0 -M 99999 -E -1 root +# -d is the one that matters and the one that was missing. The other flags +# clear the inactivity, min, max and account-expiry fields; none of them touch +# sp_lstchg, and Hetzner's image ships it as 0. A 0 there does not mean "old", +# it means "must be changed at next login", so PAM refuses a key login with +# "Your password has expired" no matter how open the rest of the aging fields +# are. Stamping today's date is what actually lets a golden box be debugged. +chage -d "$(date +%F)" -I -1 -m 0 -M 99999 -E -1 root # Undoing the expiry once is not enough: cloud-init runs again on every clone # and re-expires root, because the Ubuntu cloud image defaults to # a chpasswd expire default. Turning that default off is what survives. @@ -295,6 +308,43 @@ async function verifySnapshot(token, imageId, location, serverType, deadline) { } } +/** + * The builder script is a template literal, so an unescaped backtick anywhere + * inside it silently ends the string early and the rest parses as JavaScript + * that is never evaluated. That is not a syntax error and not a runtime error: + * `node --check` passes, the function returns, and the builder simply receives + * a shorter script than the file appears to contain. + * + * It happened, and it cost two unusable snapshots. A comment reading + * "`|| true` throughout" cut the script off just before lever 2, taking the + * root-password fix, the identity stripping, the marker and the final + * `shutdown -h now` with it. The bake only ever finished because a separate + * failure tripped the ERR trap, whose handler powers the machine off. + * + * So assert the two ends of the script are present before spending a VM on it. + */ +function assertWholeScript(userData) { + const required = [ + ["the box image setup", "bake: box image present as"], + ["lever 2", "unattended-upgrades.service"], + ["the root-password fix", "chage -d"], + ["the identity strip", "truncate -s 0 /etc/machine-id"], + ["the golden marker", "/etc/blitz-golden-image"], + ]; + const missing = required.filter(([, needle]) => !userData.includes(needle)); + if (missing.length > 0) { + throw new Error( + `builder script is truncated -- missing ${missing.map(([name]) => name).join(", ")}. ` + + "An unescaped backtick inside the template literal ends it early.", + ); + } + // Without this line the builder cannot stop itself, and the bake waits out + // its whole timeout for a shutdown that was never in the script. + if (!userData.trimEnd().endsWith("shutdown -h now")) { + throw new Error("builder script does not end in `shutdown -h now`"); + } +} + async function main() { const token = requireEnv("HETZNER_API_TOKEN"); const image = { @@ -306,6 +356,9 @@ async function main() { const serverType = argument("server-type", "cx23"); const deadline = Date.now() + BUILD_TIMEOUT_MS; + const userData = builderUserData(image); + assertWholeScript(userData); + console.log(`bake: builder ${serverType}@${location} for ${image.boxImageRef}`); const created = await hetzner(token, "/servers", { method: "POST", @@ -314,7 +367,7 @@ async function main() { server_type: serverType, image: "ubuntu-24.04", location, - user_data: builderUserData(image), + user_data: userData, labels: { "blitz-purpose": "golden-builder" }, }), });