Add S4 recovery checkpoints and isolated restore

.dockerignore

@@ -0,0 +1,10 @@
+.git
+.env
+ghost.env
+.ghost-*
+data
+node_modules
+caddy/sites
+caddy/custom
+caddy/global
+caddy/.staging

.editorconfig

@@ -27,3 +27,7 @@ indent_style = tab
 
 [Caddyfile]
 indent_style = tab
+
+# Match Ghost's JavaScript/TypeScript formatting without changing shell style.
+[*.{js,mjs,ts,mts}]
+indent_size = 2

.github/workflows/manager.yml

@@ -0,0 +1,27 @@
+---
+name: Recovery manager image
+on:
+  push:
+    tags: ['v*']
+permissions:
+  contents: read
+  packages: write
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
+      - name: Publish release manager
+        env:
+          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          printf '%s' "$GHCR_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+          image="ghcr.io/${GITHUB_REPOSITORY,,}/manager"
+          docker buildx create --use
+          docker buildx build --platform linux/amd64,linux/arm64 \
+            --file manager/Dockerfile --tag "$image:$GITHUB_REF_NAME" \
+            --metadata-file /tmp/manager-image.json --push .
+          digest=$(jq -r '.["containerimage.digest"]' /tmp/manager-image.json)
+          printf 'Release manager: `%s@%s`\n' "$image" "$digest" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/test.yml

@@ -24,6 +24,15 @@ jobs:
         with:
           node-version: "22"
 
+      - name: Install development tools
+        run: npm ci --ignore-scripts
+
+      - name: Check JavaScript and TypeScript
+        run: |
+          npm run format:check
+          npm run lint
+          npm run typecheck
+
       - name: Check the runtime prerequisites
         run: |
           set -eu
@@ -157,3 +166,17 @@ jobs:
         run: |
           docker ps -a
           docker compose ls || true
+
+  recovery:
+    name: Recovery checkpoint and restore drill
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
+        with:
+          node-version: "22"
+      - name: Restore a real site into an isolated destination
+        env:
+          GD_TEST_RECOVERY: "1"
+        run: node --test --test-timeout=1800000 tests/recovery-e2e.test.mjs

.gitignore

@@ -93,3 +93,16 @@ caddy/.staging/
 caddy/Caddyfile.local
 # Backups written by the helpers
 *.bak.*
+
+# Recovery checkpoints contain credentials and full site data.
+.ghost-backups/
+.ghost-operation.json
+.ghost-operation-lock/
+.ghost-manager-image
+.ghost-operation.json.tmp
+.ghost-docker.json.tmp*
+.env.tmp.*
+ghost.env.tmp.*
+
+# Development tooling only; never part of the manager runtime.
+node_modules/

.oxfmtrc.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "./node_modules/oxfmt/configuration_schema.json",
+  "singleQuote": true,
+  "embeddedLanguageFormatting": "off",
+  "ignorePatterns": ["tests/fixtures/**", "**/node_modules/**"]
+}

.oxlintrc.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "./node_modules/oxlint/configuration_schema.json",
+  "plugins": ["typescript", "unicorn", "oxc"],
+  "categories": { "correctness": "error" },
+  "rules": { "curly": ["error", "all"] },
+  "ignorePatterns": ["tests/fixtures/**"]
+}

CLAUDE.md

@@ -42,6 +42,8 @@ curl -fsSL .../bootstrap.sh | bash -s -- --domain example.com   # release-select
 ./install.sh --local --no-prompt --no-start                     # checkout-owned installer
 scripts/site.sh check                   # doctor: config, health, DB, ingress
 scripts/site.sh list                    # every managed container on this host
+scripts/recovery.sh backup --keep 5      # private recovery checkpoint
+scripts/recovery.sh status              # inspect an unfinished operation
 
 # Core operations
 docker compose up -d                    # Start the services for the selected mode
@@ -73,8 +75,12 @@ scripts/config.sh unset ghost.env KEY
 # Caddy routes
 scripts/caddy.sh apply                  # Render, validate, install, reload, verify
 
-# Tests (Node 20+ built-in runner, no dependencies and no package.json;
+# Development (Node 22.18+ on the 22.x line, or Node 24+;
 # docker tests skip without a daemon)
+npm ci --ignore-scripts
+npm run format:check
+npm run lint
+npm run typecheck
 node --test --test-timeout=120000 tests/*.test.mjs
 GD_TEST_INGRESS=1 node --test --test-timeout=900000 tests/ingress.test.mjs
 GD_TEST_INSTALL=1 node --test --test-timeout=1800000 tests/install-e2e.test.mjs
@@ -135,8 +141,7 @@ The repository includes comprehensive migration tools:
   - Sets up Docker Compose environment
 
 - `scripts/config-to-env.js` - Converts Ghost JSON config to ghost.env format.
-  CommonJS; there is no package.json in this repository, so `.js` is CommonJS
-  by default. This is the only host Node dependency, and `install.sh --import`
+  CommonJS; the root package keeps the default CommonJS module mode. This is the only host Node dependency, and `install.sh --import`
   removes it
 
 ## Installer
@@ -165,7 +170,9 @@ Rules that must not regress:
   `--image-registry`, `--ghost-channel`, `--without`) exit 3 naming the step,
   not as unknown options.
 
-See `docs/install.md`.
+See `docs/install.md`. Recovery checkpoints, isolated restore and the manager
+image are documented in `docs/recovery.md`. Supported mutating scripts share the
+operation lock; new mutating commands must acquire it and refuse unresolved journals.
 
 ## Development Workflow
 
@@ -183,6 +190,12 @@ in `tests/` run by Node's built-in runner: the shell libraries are exercised
 through a real shell via `tests/helpers.mjs`, while fixtures, structured-output
 parsing and assertions are JavaScript.
 
+JavaScript and TypeScript use oxfmt with Ghost's two-space, single-quote style.
+Oxlint requires braces around control flow. The manager is strict TypeScript using
+Node's native type stripping; `npm run typecheck` uses `tsc --noEmit` with
+`erasableSyntaxOnly`. No compiler or npm dependencies are needed in the manager
+image. See `docs/development.md`.
+
 For analytics setup, see `TINYBIRD.md` for detailed instructions.
 
 ## Implementation plan

docs/development.md

@@ -0,0 +1,41 @@
+# Development tools
+
+Use Node 22.18 or later on the 22.x line, or Node 24+. These are developer/test
+requirements; site operators still use the shell dispatcher and Docker.
+
+```sh
+npm ci --ignore-scripts
+npm run format
+npm run format:check
+npm run lint
+npm run typecheck
+npm test
+```
+
+The dependencies are pinned in `package-lock.json`. CI runs the formatting, lint
+and type checks before the helper suite. Docker integration tests skip without a
+working daemon; the opt-in restore drills are described in `docs/recovery.md`.
+
+Formatting uses Ghost's oxfmt version and conventions: two spaces for JavaScript
+and TypeScript, single quotes, and no embedded-language formatting. The formatter
+covers the manager, tests, the legacy JavaScript helper and tooling configuration;
+fixture data is excluded. Shell files retain their existing four-space style and
+ShellCheck validation. Oxlint checks correctness and requires braces around control
+flow so dense one-line conditions do not return.
+
+The manager is strict TypeScript with explicit checkpoint, journal, ownership,
+Docker response and subprocess types. Its local `package.json` selects ESM; the
+repository root keeps CommonJS for the legacy `.js` helper. Tests remain `.mjs`
+and import the manager's `.ts` modules directly.
+
+Node strips erasable types at runtime; it does **not** type-check the program.
+`npm run typecheck` runs TypeScript with `noEmit`, `erasableSyntaxOnly`,
+`verbatimModuleSyntax` and `allowImportingTsExtensions`. Use type-only imports and
+explicit `.ts` extensions, and avoid enums, parameter properties, decorators,
+path aliases or other constructs that require transformation. Runtime JSON and
+checkpoint integrity checks still apply; TypeScript types are not input validation.
+
+The manager Dockerfile copies the TypeScript source and its ESM package metadata,
+then runs `node /opt/manager/main.ts`. It does not install the formatter, linter,
+compiler, or any npm runtime dependencies. The pinned Node image is tested by the
+same local, production HTTPS and ActivityPub restore drills as the recovery code.

docs/ghost-cli-replacement.md

@@ -876,6 +876,15 @@ blockers for dependent steps.
 
 ### S4 — Backup, restore, locks, and recovery journal
 
+Implementation: see `docs/recovery.md`. The initial manager supports stack-managed
+MySQL and Ghost/ActivityPub, private directory checkpoints, a shared host lock and
+durable recovery journals. Analytics and external database/override configurations
+fail preflight pending explicit state contracts. Maintenance stops all application
+and ingress containers. Restore targets a fresh checkout and remains isolated until
+explicit activation; S7 must add in-place transactional upgrade orchestration around
+these primitives. Source checkouts build and execute an immutable manager image ID;
+release tags publish multi-platform manager images selectable by digest.
+
 Repo: ghost-docker. Deps: S1. Implement the reusable §2.5 checkpoint/restore contract,
 explicit DB connection abstraction, operation lock, maintenance handling, retention,
 and journals. Backups include required local application/configuration state and

docs/recovery.md

@@ -0,0 +1,144 @@
+# Recovery checkpoints
+
+S4 introduces `scripts/recovery.sh` and a containerized manager. The host still
+needs only the installer prerequisites; Node runs inside the manager. The manager
+is TypeScript executed directly by the pinned Node 22 image, with no compilation
+step or npm runtime dependencies. See `docs/development.md` for type-checking and
+formatting. A source
+checkout builds its own manager before downtime and executes the resulting image
+ID. Releases publish `ghcr.io/tryghost/ghost-docker/manager`; set
+`GD_MANAGER_IMAGE=ghcr.io/tryghost/ghost-docker/manager@sha256:…` to use a published
+immutable digest. Mutable image tags are refused. The journal, checkpoint and installation metadata (when present) record the actual
+manager image ID. Publication is wired to release tags; no registry publication is
+needed to use a source checkout.
+
+```sh
+scripts/recovery.sh backup --keep 5
+scripts/recovery.sh status
+```
+
+Backups are private directory checkpoints under `.ghost-backups/`, **not migration
+bundles**. Copy the entire checkpoint directory to private off-host storage. It
+contains database credentials and all site data. Completed checkpoints have a
+versioned manifest and SHA-256 inventory. Incomplete `.partial-*` directories are
+never offered as completed backups or removed by retention. Retention removes only
+verified completed checkpoints, keeping the newest requested count (default five).
+A failed backup remains journaled; inspect it and run `recover` to restore the
+previous service state. Partial files remain available for inspection/removal.
+
+The initial supported configuration is the stack-managed MySQL database, local or
+production mode, optionally ActivityPub, and data bind mounts beneath
+`PROJECT_DIR/data`. External MySQL, Compose overrides, symlinks/hardlinks/special
+files, analytics, supervisor, rootless Docker and user namespaces are refused
+before downtime. Local Unix sockets are required; Docker Desktop/OrbStack must
+share the site's absolute path with the daemon. The manager mounts that same path,
+the socket, and (for restore) a read-only checkpoint. Docker socket access is
+host-privileged. No unrelated host directories are mounted writable.
+
+A checkpoint includes:
+
+- A logical SQL dump of the explicitly selected Ghost database, and ActivityPub's
+  database when enabled, including schema, triggers, routines and events.
+- The entire Ghost content tree, including hidden files, themes and ActivityPub
+  uploads, with recorded ownership and modes.
+- `.env`, `ghost.env`, installation metadata when present, the stack's Compose
+  definition, MySQL initialization scripts and the full Caddy configuration tree.
+- Exact registry image digests, manager identity and file checksums.
+
+Caddy's certificate and cache volumes are regenerated, so activation may need
+certificate issuance. Analytics is currently refused because its persistent queues
+and remote deployment state need their own recovery contract. Ghost/ActivityPub
+backups do not undo delivered email, payments, webhooks, federation messages or
+other remote effects. Extra databases unrelated to the enabled services are not
+included. External writers to the managed databases must be stopped by the
+operator; an enabled MySQL event scheduler with active events is refused.
+
+Backup disables restart policies and stops every application/ingress container,
+including background workers and custom Caddy routes, while leaving MySQL running.
+Requests fail to connect during maintenance. The stopped ingress is intentional:
+custom routes cannot bypass a maintenance response. After snapshot verification,
+the manager restores each container's previous restart policy and running/stopped
+state and verifies readiness before reporting completion. It checks available space
+with conservative database/content allowances and a 256 MiB reserve; actual write
+errors still fail the operation. Checkpoints are published only after file writes,
+manifest verification and directory synchronization.
+
+## Restore drill or recovery to a fresh checkout
+
+Restore into a separate checkout with no `.env`, metadata, existing data or
+containers belonging to the chosen destination project. Use a checkout containing
+S4. The source checkpoint's stack configuration and image versions are restored;
+upgrading is a separate operation.
+
+```sh
+# Run these from the fresh destination checkout.
+scripts/recovery.sh restore /absolute/path/to/checkpoint \
+    --project ghost-rehearsal --local --port 2468
+scripts/recovery.sh status
+# Once you intend to run this copy normally:
+scripts/recovery.sh activate
+```
+
+`--local` changes the destination's mode, URL and port and disables optional
+profiles for a rehearsal. Without it, the checkpoint's mode/URLs/profiles are
+retained. `--project` is always explicit and must not already exist on this daemon.
+Generated production routes are rendered for the new identity at activation;
+operator Caddy files are retained and must be appropriate for the destination.
+
+Restore checks the checkpoint before mutation, pulls its immutable images, restores
+SQL and files, compares every table row count and the content inventory, then boots a temporary Ghost without published ports on an internal
+Docker network. This verification container can reach MySQL but cannot send mail or
+webhooks to external services. It is removed after verification. Normal Ghost,
+ActivityPub, jobs and Caddy remain stopped. The journal remains at `verified` until
+`activate`; configuration helpers also remain locked out by that journal.
+
+**Activation enables normal outbound behavior and ingress.** Use it only when the
+copy is intended to become a running site. Production cutover, DNS and ensuring the
+old site no longer accepts writes are operator responsibilities. Activation verifies
+Compose health and production proxy routing before clearing the journal. Restore
+does not overwrite an existing installed site: preserve that site and use a fresh
+destination. It does not claim an automatic rollback of a live deployment.
+
+## Interrupted operations
+
+Every supported mutating entrypoint (install, config edits, Caddy edits and
+recovery) shares `.ghost-operation-lock`. The manager also keeps a durable
+`.ghost-operation.json` journal, written before side effects. `site.sh check`
+reports unresolved operations. Direct `docker compose` commands bypass these
+safeguards; do not run them concurrently with managed operations.
+
+```sh
+scripts/recovery.sh status
+scripts/recovery.sh recover
+```
+
+Locks never expire by age. Recovery checks the host PID, Docker daemon identity and any manager container;
+a live or ambiguous owner is not stolen. An unreachable daemon prevents reclamation.
+An incomplete owner record requires manual inspection. A reused PID can conservatively
+block recovery; inspect the owner rather than deleting a live lock.
+
+For interrupted backups, recovery reconciles saved container identities and returns
+them to their original state. It does not promote an unfinished checkpoint. For a
+restore that has never activated, recovery removes any verification container and
+repeats restoration from the same verified checkpoint into the same destination.
+Partial SQL restores cannot be mistaken for resumable imports. Once activation has
+begun, recovery only retries readiness/activation; it **never replays the checkpoint**
+because new writes may already exist. Failed restore verification leaves ingress
+blocked. Preserve the journal and checkpoint until recovery is resolved.
+
+## Verification
+
+`node --test tests/recovery.test.mjs` checks lock exclusion/reclamation, daemon
+identity, checkpoint integrity, retention, failed durable writes (`ENOSPC`) and
+SQL producer/consumer exit propagation. The standard helper suite also covers the
+host configuration interfaces that share the lock.
+
+`GD_TEST_RECOVERY=1 node --test --test-timeout=1800000 tests/recovery-e2e.test.mjs`
+runs real local, HTTPS production and ActivityPub restore drills. Each kills the
+host dispatcher while its backup manager is still alive, checks that recovery
+cannot steal that lock, kills the manager, and recovers. It also kills restore
+during verification and recovers without ingress. Assertions cover database values,
+active theme, hidden assets, application configuration including multiline literal
+values, private checkpoint permissions, isolation, activation and restart policy.
+These drills run in CI. The release publishing workflow itself requires a release
+tag and package publishing credentials.

help

@@ -48,6 +48,12 @@ ROUTING (production):
   Your own routes:   caddy/custom/*.caddy
   Global options:    caddy/global/*.caddy
 
+RECOVERY CHECKPOINTS:
+  scripts/recovery.sh backup --keep 5
+  scripts/recovery.sh status
+  scripts/recovery.sh recover
+  See docs/recovery.md for isolated restore and activation.
+
 TROUBLESHOOTING:
   docker compose exec ghost sh      # Access Ghost container shell
   docker compose logs --tail=100    # View last 100 log lines

install.sh

@@ -153,6 +153,9 @@ if [[ -n $dir ]]; then
 fi
 dir=$checkout
 
+operation_acquire "$dir"
+trap operation_release EXIT
+
 # --- Refuse to install over an existing site -------------------------------
 
 for existing in "$GD_ENV_FILE_NAME" "$GD_META_FILE_NAME"; do