From da0f2b61c22df14e34a460f40c9c521883b3b655 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 2 Oct 2026 06:44:13 +0000 Subject: [PATCH] dev2: deploy-app.sh rolls back on any failure after the swap starts (0.51.1) Under set -e, a failed `compose up` or `compose down` exited the script before its health check and rollback ran, so a deploy that had already removed the old container could leave the site down. The rollback path also rebuilt the old revision, which fails the same way when the build is what broke. Now the images the running containers came from are recorded (and pinned as :rollback) before anything is touched, and an ERR trap plus the health check failure both re-tag them and `compose up --no-build`. A failed build exits before the snapshot with the running stack untouched. The subnet grep no longer kills the script when a compose file has no subnet. test/dev2-deploy-app.test.ts drives the script against a fake docker: build failure, failed up, unhealthy image, healthy deploy. Co-Authored-By: Claude Opus 5.5 --- dev2/README.md | 2 +- dev2/templates/deploy-app.sh | 79 +++++++++++++++++++---- package.json | 2 +- test/dev2-deploy-app.test.ts | 122 +++++++++++++++++++++++++++++++++++ 4 files changed, 190 insertions(+), 15 deletions(-) create mode 100644 test/dev2-deploy-app.test.ts diff --git a/dev2/README.md b/dev2/README.md index a2d3163..53f2edc 100644 --- a/dev2/README.md +++ b/dev2/README.md @@ -23,7 +23,7 @@ dev2-site migrate --yes the whole move, resumable; each step below is a | `db` | (kind `pg`) database in the shared Supabase cluster on dev2 (one database per app, role `postgres`, like nichedb and rssamplifier); `pg_dump -Fc` inside the Railway Postgres container over `ssh.railway.com`, streamed to dev2, `pg_restore`, extensions pre-created, row counts compared | | `provision` | `/home/anthony/www//`: `app.env` rendered from Railway's variables (RAILWAY_* dropped, DATABASE_URL / PG* / REDIS_URL re-pointed), one `.env` per companion service, `deploy.env`, `docker-compose.app.yml`, `deploy-app.sh`; one deploy key per repo authorised for `anthony`; `DEV2_*` secrets set on the repo | | `volumes` | Railway volume contents tar-streamed into `volumes/`, bind-mounted at the same path | -| `deploy` | `deploy-app.sh ` as `anthony`: build on the box, `compose up`, health on `127.0.0.1:` | +| `deploy` | `deploy-app.sh ` as `anthony`: build on the box, `compose up`, health on `127.0.0.1:`. A failed build leaves the running stack alone; any failure after that (compose up/down, health) re-tags and restarts the images that were running | | `cert` | acme.sh, Let's Encrypt, **Porkbun DNS-01**, every custom domain (+ www), into `/etc/nginx/ssl//`; issued before DNS moves so there is no TLS gap | | `vhost` | nginx server block -> `127.0.0.1:`, plus the site's maintenance page (below). Takes several sites or `--all`; `--dry-run` diffs the rendered vhost against the live one | | `verify` | curl each domain `--resolve`d to dev2; `--public` after the flip | diff --git a/dev2/templates/deploy-app.sh b/dev2/templates/deploy-app.sh index 65ac926..ad3d903 100755 --- a/dev2/templates/deploy-app.sh +++ b/dev2/templates/deploy-app.sh @@ -30,6 +30,51 @@ die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } compose() { (cd "$ROOT" && docker compose -f docker-compose.app.yml --env-file "$ROOT/deploy.env" "$@"); } +# Rollback state. snapshot() records the image each running service container was +# started from (and pins it as :rollback so a prune cannot take it); once the +# running stack is touched, any failure (the ERR trap, or a failed health check) +# re-tags those images and brings the old stack back up without building. +SNAP=() +CURRENT="" +snapshot() { + local svc cid id ref + for svc in $BUILD_SERVICES; do + cid=$(compose ps -q "$svc" 2>/dev/null | head -n1) || true + [ -n "$cid" ] || continue + id=$(docker inspect --format '{{.Image}}' "$cid" 2>/dev/null) || continue + ref=$(docker inspect --format '{{.Config.Image}}' "$cid" 2>/dev/null) || continue + case $ref in *@*) continue ;; *:*) ;; *) ref="$ref:latest" ;; esac + docker tag "$id" "${ref%:*}:rollback" >/dev/null 2>&1 || true + SNAP+=("$id $ref") + done +} + +restore() { + trap - ERR + set +e + if [ "${#SNAP[@]}" -eq 0 ]; then log "Nothing to roll back to (no running containers before this deploy)"; return 1; fi + log "Rolling back to the previous image(s)" + local p + for p in "${SNAP[@]}"; do docker tag "${p%% *}" "${p#* }"; done + [ -z "$CURRENT" ] || git -C "$APP_DIR" checkout -q --detach "$CURRENT" + compose up -d --no-build --remove-orphans + if health; then log "Restored the previous deploy"; return 0; fi + log "ROLLBACK ALSO UNHEALTHY - site is down" + return 1 +} + +on_err() { + local rc=$? line=$1 + # errtrace runs this inside compose()'s subshell and $(...) too; leave the + # rollback to the top-level shell, which sees the same failure next. + [ "$BASH_SUBSHELL" = 0 ] || exit "$rc" + trap - ERR + log "Failed (exit $rc, line $line) after the running stack was touched" + compose logs --tail 60 || true + restore || true + exit 1 +} + health() { local i for i in $(seq 1 "$((HEALTH_TIMEOUT / 5))"); do @@ -64,18 +109,21 @@ TARGET=${1:?usage: deploy-app.sh | --rollback | --status} if [ "${IMAGE_ONLY:-0}" = 1 ]; then # A stock image with Railway's start command: nothing to clone or build. log "Pulling image and starting" + snapshot compose pull -q || true + set -E; trap 'on_err $LINENO' ERR compose up -d --remove-orphans if health; then log "Healthy on 127.0.0.1:$APP_PORT$HEALTH_PATH" { echo "DEPLOYED_SHA=image"; echo "PREVIOUS_SHA="; echo "DEPLOYED_AT=$(date -u +%FT%TZ)"; } > "$STATE" compose ps; exit 0 fi + trap - ERR compose logs --tail 60 || true + restore || true die "image-only deploy failed health check" fi -CURRENT="" [ -d "$APP_DIR/.git" ] && CURRENT=$(git -C "$APP_DIR" rev-parse HEAD 2>/dev/null || echo "") if [ ! -d "$APP_DIR/.git" ]; then @@ -112,24 +160,33 @@ while IFS= read -r line || [ -n "$line" ]; do export "$key=$val" ;; esac done < "$ROOT/app.env" +# A failed build exits here (set -e) before anything running is touched: `compose +# build` only moves the :latest tags when it succeeds. # shellcheck disable=SC2086 -compose build $BUILD_SERVICES +compose build $BUILD_SERVICES || die "build of $SHA failed; the running stack was not touched" # If the compose network's subnet changed (the kit moved sites to explicit # 10.200.x.0/24 subnets), `up -d` recreates the network in place and Docker's # embedded DNS then answers SERVFAIL for service aliases like `redis` while # container names still resolve. A clean down/up avoids that. -WANT_SUBNET=$(grep -oE 'subnet: [0-9./]+' "$ROOT/docker-compose.app.yml" | awk '{print $2}' | head -n1) -APP_CID=$(compose ps -q app 2>/dev/null | head -n1) +WANT_SUBNET=$(grep -oE 'subnet: [0-9./]+' "$ROOT/docker-compose.app.yml" | awk '{print $2}' | head -n1 || true) +APP_CID=$(compose ps -q app 2>/dev/null | head -n1 || true) +RECREATE=0 if [ -n "$WANT_SUBNET" ] && [ -n "$APP_CID" ]; then - NET=$(docker inspect --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$APP_CID" 2>/dev/null | head -n1) + NET=$(docker inspect --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$APP_CID" 2>/dev/null | head -n1 || true) HAVE_SUBNET=$(docker network inspect "$NET" --format '{{range .IPAM.Config}}{{.Subnet}}{{end}}' 2>/dev/null || true) if [ -n "$HAVE_SUBNET" ] && [ "$HAVE_SUBNET" != "$WANT_SUBNET" ]; then log "Network subnet changes ($HAVE_SUBNET -> $WANT_SUBNET): recreating the stack" - compose down --remove-orphans + RECREATE=1 fi fi +# From here on the running stack is touched: any failure rolls back to the images +# it was running. +snapshot +set -E; trap 'on_err $LINENO' ERR +[ "$RECREATE" = 0 ] || compose down --remove-orphans + log "Starting" compose up -d --remove-orphans @@ -152,6 +209,7 @@ if [ -n "$APP_CID" ]; then fi if health; then + trap - ERR log "Healthy on 127.0.0.1:$APP_PORT$HEALTH_PATH" { echo "DEPLOYED_SHA=$SHA" @@ -160,14 +218,9 @@ if health; then } > "$STATE" compose ps else + trap - ERR log "UNHEALTHY after ${HEALTH_TIMEOUT}s - last 60 lines:" compose logs --tail 60 || true - if [ -n "$CURRENT" ] && [ "$CURRENT" != "$SHA" ]; then - log "Restoring $CURRENT" - git -C "$APP_DIR" checkout -q --detach "$CURRENT" - # shellcheck disable=SC2086 - compose build $BUILD_SERVICES && compose up -d - health && log "Restored to $CURRENT" || log "ROLLBACK ALSO UNHEALTHY - site is down" - fi + restore || true die "deploy of $SHA failed health check" fi diff --git a/package.json b/package.json index 8222fe4..b9d94f6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/cli-tools", - "version": "0.51.0", + "version": "0.51.1", "private": true, "description": "Local command-line tools, in TypeScript, exposed on PATH.", "type": "module", diff --git a/test/dev2-deploy-app.test.ts b/test/dev2-deploy-app.test.ts new file mode 100644 index 0000000..eaa206a --- /dev/null +++ b/test/dev2-deploy-app.test.ts @@ -0,0 +1,122 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +/** + * dev2/templates/deploy-app.sh against a fake docker. The fake keeps two facts: + * which image the running app container came from (`running`, empty = no + * container) and what site-app:latest points at (`latest`). The old stack runs + * img-old; a successful build moves :latest to img-new. + */ +const SCRIPT = resolve(__dirname, '../dev2/templates/deploy-app.sh'); + +const FAKE_DOCKER = `#!/usr/bin/env bash +F=$FAKE; echo "$*" >> "$F/log" +if [ "$1" = compose ]; then + shift + while [ $# -gt 0 ]; do case $1 in -f|--env-file) shift 2 ;; *) break ;; esac; done + cmd=$1; shift + case $cmd in + build) [ "\${FAIL_BUILD:-0}" = 1 ] && exit 1; echo img-new > "$F/latest" ;; + ps) if [ "\${1:-}" = -q ] && [ -s "$F/running" ]; then echo "cid-\${2:-app}"; fi ;; + up) + n=$(cat "$F/ups" 2>/dev/null || echo 0); echo $((n + 1)) > "$F/ups" + # Compose removed the old container, then failed to start the new one. + if [ "\${FAIL_UP:-0}" = 1 ] && [ "$n" = 0 ]; then : > "$F/running"; exit 1; fi + cat "$F/latest" > "$F/running" ;; + down) : > "$F/running" ;; + esac + exit 0 +fi +case $1 in + inspect) case $3 in '{{.Image}}') cat "$F/running" ;; '{{.Config.Image}}') echo site-app:latest ;; *) echo net ;; esac ;; + tag) [ "$3" = site-app:latest ] && echo "$2" > "$F/latest"; exit 0 ;; + exec) exit 1 ;; +esac +exit 0 +`; + +const FAKE_CURL = `#!/usr/bin/env bash +r=$(cat "$FAKE/running") +if [ -n "$r" ] && ! { [ "\${UNHEALTHY_NEW:-0}" = 1 ] && [ "$r" = img-new ]; }; then printf 200; else printf 000; fi +`; + +let dir = ''; +afterEach(() => { if (dir) rmSync(dir, { recursive: true, force: true }); dir = ''; }); + +function setup() { + dir = mkdtempSync(join(tmpdir(), 'deploy-app-')); + const bin = join(dir, 'bin'), fake = join(dir, 'fake'), root = join(dir, 'www'), origin = join(dir, 'origin'); + for (const d of [bin, fake, root, origin]) mkdirSync(d); + for (const [n, body] of [['docker', FAKE_DOCKER], ['curl', FAKE_CURL], ['sleep', '#!/bin/sh\n']] as const) { + writeFileSync(join(bin, n), body); chmodSync(join(bin, n), 0o755); + } + const git = (...a: string[]) => execFileSync('git', ['-c', 'user.name=t', '-c', 'user.email=t@t', ...a], { cwd: origin, encoding: 'utf8' }).trim(); + git('init', '-q', '-b', 'main'); + writeFileSync(join(origin, 'f'), '1'); git('add', 'f'); git('commit', '-qm', 'one'); + const oldSha = git('rev-parse', 'HEAD'); + writeFileSync(join(origin, 'f'), '2'); git('commit', '-qam', 'two'); + const newSha = git('rev-parse', 'HEAD'); + execFileSync('git', ['clone', '-q', origin, join(root, 'app')]); + execFileSync('git', ['-C', join(root, 'app'), 'checkout', '-q', '--detach', oldSha]); + writeFileSync(join(root, 'deploy.env'), `REPO=${origin}\nAPP_PORT=3999\nBUILD_SERVICES=app\nHEALTH_PATH=/\nHEALTH_TIMEOUT=5\n`); + writeFileSync(join(root, 'app.env'), 'NEXT_PUBLIC_X=1\n'); + writeFileSync(join(root, 'docker-compose.app.yml'), 'services:\n app:\n image: site-app:latest\n'); + writeFileSync(join(fake, 'running'), 'img-old\n'); + writeFileSync(join(fake, 'latest'), 'img-old\n'); + writeFileSync(join(fake, 'log'), ''); + return { bin, fake, root, oldSha, newSha }; +} + +function deploy(env: Record = {}) { + const t = setup(); + const r = spawnSync('bash', [SCRIPT, t.newSha], { + encoding: 'utf8', + env: { ...process.env, PATH: `${t.bin}:${process.env.PATH}`, ROOT: t.root, FAKE: t.fake, ...env }, + }); + const read = (f: string) => readFileSync(join(t.fake, f), 'utf8').trim(); + return { ...t, status: r.status, out: r.stdout + r.stderr, running: read('running'), latest: read('latest'), log: read('log'), + state: existsSync(join(t.root, '.deploy-state')) ? readFileSync(join(t.root, '.deploy-state'), 'utf8') : '' }; +} + +describe('dev2 deploy-app.sh', () => { + it('a failed build never touches the running stack', () => { + const r = deploy({ FAIL_BUILD: '1' }); + expect(r.status).not.toBe(0); + expect(r.out).toContain('the running stack was not touched'); + expect(r.running).toBe('img-old'); + expect(r.log).not.toMatch(/compose .*\b(up|down)\b/); + expect(r.log).not.toMatch(/^tag /m); + expect(r.state).toBe(''); + }); + + it('a failed `compose up` after the old container is gone rolls back to the previous image', () => { + const r = deploy({ FAIL_UP: '1' }); + expect(r.status).not.toBe(0); + expect(r.out).toContain('Rolling back to the previous image'); + expect(r.out).toContain('Restored the previous deploy'); + expect(r.running).toBe('img-old'); + expect(r.latest).toBe('img-old'); + expect(r.log).toMatch(/compose .*up -d --no-build/); + expect(r.out.match(/Rolling back/g)).toHaveLength(1); + expect(r.state).toBe(''); + }); + + it('an unhealthy new image rolls back to the previous image without rebuilding', () => { + const r = deploy({ UNHEALTHY_NEW: '1' }); + expect(r.status).not.toBe(0); + expect(r.running).toBe('img-old'); + expect(r.log.match(/compose .* build /g)).toHaveLength(1); + expect(r.state).toBe(''); + }); + + it('a healthy deploy switches to the new image and records it', () => { + const r = deploy(); + expect(r.status, r.out).toBe(0); + expect(r.running).toBe('img-new'); + expect(r.state).toContain(`DEPLOYED_SHA=${r.newSha}`); + expect(r.state).toContain(`PREVIOUS_SHA=${r.oldSha}`); + }); +});