diff --git a/packages/control-plane/scripts/bake-golden-image.mjs b/packages/control-plane/scripts/bake-golden-image.mjs index 18cdc691..c7b73add 100644 --- a/packages/control-plane/scripts/bake-golden-image.mjs +++ b/packages/control-plane/scripts/bake-golden-image.mjs @@ -110,13 +110,20 @@ install -d -m 0755 /etc/ssh/sshd_config.d cat >/etc/ssh/sshd_config.d/00-blitz.conf <<'SSHD_CONFIG' Port 2222 SSHD_CONFIG +# sshd -t refuses to run without its privilege separation directory, and /run +# is a tmpfs that only ssh.service populates -- which socket activation has not +# started yet at this point in first boot. Without this line the test exits 255 +# on "Missing privilege separation directory: /run/sshd", the ERR trap fires, +# and the builder powers off HERE: after the expensive image pull and before +# every line below. That is what produced the first two golden snapshots. +install -d -m 0755 /run/sshd /usr/sbin/sshd -t systemctl disable ssh.socket 2>/dev/null || true systemctl mask ssh.socket systemctl enable ssh # Lever 2: units a workspace never uses, and which cost seconds of every boot. -# `|| true` throughout: an absent unit is not a bake failure. +# \`|| true\` throughout: an absent unit is not a bake failure. for unit in snapd.service snapd.socket snapd.seeded.service unattended-upgrades.service \ multipathd.service multipathd.socket apt-daily.timer apt-daily-upgrade.timer \ motd-news.timer man-db.timer e2scrub_all.timer fstrim.timer; do @@ -128,7 +135,13 @@ done # host cannot be debugged. Lock the password instead: no password login, no # expiry, keys still work. usermod -p '*' root -chage -I -1 -m 0 -M 99999 -E -1 root +# -d is the one that matters and the one that was missing. The other flags +# clear the inactivity, min, max and account-expiry fields; none of them touch +# sp_lstchg, and Hetzner's image ships it as 0. A 0 there does not mean "old", +# it means "must be changed at next login", so PAM refuses a key login with +# "Your password has expired" no matter how open the rest of the aging fields +# are. Stamping today's date is what actually lets a golden box be debugged. +chage -d "$(date +%F)" -I -1 -m 0 -M 99999 -E -1 root # Undoing the expiry once is not enough: cloud-init runs again on every clone # and re-expires root, because the Ubuntu cloud image defaults to # a chpasswd expire default. Turning that default off is what survives. @@ -295,6 +308,43 @@ async function verifySnapshot(token, imageId, location, serverType, deadline) { } } +/** + * The builder script is a template literal, so an unescaped backtick anywhere + * inside it silently ends the string early and the rest parses as JavaScript + * that is never evaluated. That is not a syntax error and not a runtime error: + * `node --check` passes, the function returns, and the builder simply receives + * a shorter script than the file appears to contain. + * + * It happened, and it cost two unusable snapshots. A comment reading + * "`|| true` throughout" cut the script off just before lever 2, taking the + * root-password fix, the identity stripping, the marker and the final + * `shutdown -h now` with it. The bake only ever finished because a separate + * failure tripped the ERR trap, whose handler powers the machine off. + * + * So assert the two ends of the script are present before spending a VM on it. + */ +function assertWholeScript(userData) { + const required = [ + ["the box image setup", "bake: box image present as"], + ["lever 2", "unattended-upgrades.service"], + ["the root-password fix", "chage -d"], + ["the identity strip", "truncate -s 0 /etc/machine-id"], + ["the golden marker", "/etc/blitz-golden-image"], + ]; + const missing = required.filter(([, needle]) => !userData.includes(needle)); + if (missing.length > 0) { + throw new Error( + `builder script is truncated -- missing ${missing.map(([name]) => name).join(", ")}. ` + + "An unescaped backtick inside the template literal ends it early.", + ); + } + // Without this line the builder cannot stop itself, and the bake waits out + // its whole timeout for a shutdown that was never in the script. + if (!userData.trimEnd().endsWith("shutdown -h now")) { + throw new Error("builder script does not end in `shutdown -h now`"); + } +} + async function main() { const token = requireEnv("HETZNER_API_TOKEN"); const image = { @@ -306,6 +356,9 @@ async function main() { const serverType = argument("server-type", "cx23"); const deadline = Date.now() + BUILD_TIMEOUT_MS; + const userData = builderUserData(image); + assertWholeScript(userData); + console.log(`bake: builder ${serverType}@${location} for ${image.boxImageRef}`); const created = await hetzner(token, "/servers", { method: "POST", @@ -314,7 +367,7 @@ async function main() { server_type: serverType, image: "ubuntu-24.04", location, - user_data: builderUserData(image), + user_data: userData, labels: { "blitz-purpose": "golden-builder" }, }), });