krlex · Oct 7, 2026 · Oct 4, 2026 · Oct 5, 2026
diff --git a/README.md b/README.md
@@ -134,13 +134,17 @@ outside this repository:
 ## Backup & Restore
 
 ```bash
-# Backup
-./scripts/backup.sh
+# Backup (writes to the forail_backups volume)
+docker compose exec -T forail-task bash /etc/forail/backup.sh
 
-# Restore
-./scripts/restore.sh /path/to/backup.sql.gz
+# Restore: stop the writers, restore in a one-off container, start again
+docker compose stop forail-web forail-task
+docker compose run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes
+docker compose up -d
 ```
 
+Keep `.env` -- at least `FORAIL_SECRET_KEY` -- with your backups: without it, a restored database cannot decrypt any stored credential. Details, upgrade and rollback: the [Administrator Handbook](https://forail-platform.github.io/docs/admin-handbook.html#backup).
+
 ## Related Repositories
 
 - [forail-backend](https://github.com/forail-platform/forail-backend) — Django API + Task Engine

diff --git a/scripts/backup.sh b/scripts/backup.sh
@@ -1,27 +1,50 @@
 #!/bin/bash
-set -e
+# Back up the Forail database.
+#
+#   docker compose exec -T forail-task bash /etc/forail/backup.sh
+#
+# Writes, in $BACKUP_DIR (the forail_backups volume):
+#   forail_backup_<timestamp>.dump   pg_dump custom format (compressed)
+#   forail_backup_<timestamp>.meta   what restore.sh checks before it touches anything
+#
+# The database is not the whole backup. Credentials, tokens and other secrets
+# in it are encrypted with FORAIL_SECRET_KEY; without that key a restored
+# database cannot decrypt any of them. Keep .env (or at least
+# FORAIL_SECRET_KEY) with the backups, somewhere else than this host.
+set -euo pipefail
 
 BACKUP_DIR="${BACKUP_DIR:-/var/lib/awx/backups}"
 RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-7}"
-TIMESTAMP=$(date +%Y%m%d_%H%M%S)
-BACKUP_FILE="${BACKUP_DIR}/forail_backup_${TIMESTAMP}.sql.gz"
+STAMP=$(date +%Y%m%d_%H%M%S)
+BASE="${BACKUP_DIR}/forail_backup_${STAMP}"
+
+export PGPASSWORD="${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set}"
+PGARGS=(-h "${POSTGRES_HOST:-postgres}" -p "${POSTGRES_PORT:-5432}" -U "${POSTGRES_USER:-forail}" -d "${POSTGRES_DB:-forail}")
 
 mkdir -p "${BACKUP_DIR}"
+umask 077  # the dump holds encrypted secrets and every user's data
 
 echo "==> Starting backup..."
-PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \
-    -h "${POSTGRES_HOST:-postgres}" \
-    -p "${POSTGRES_PORT:-5432}" \
-    -U "${POSTGRES_USER:-forail}" \
-    -d "${POSTGRES_DB:-forail}" \
-    | gzip > "${BACKUP_FILE}"
+# Write to a temporary name first: an interrupted dump must never look like
+# the latest good backup to restore.sh.
+pg_dump "${PGARGS[@]}" --format=custom --compress=6 --file="${BASE}.dump.partial"
+mv "${BASE}.dump.partial" "${BASE}.dump"
+
+last_migration=$(psql "${PGARGS[@]}" -At -c "SELECT max(name) FROM django_migrations WHERE app = 'main'")
+key_fingerprint=$(printf '%s' "${FORAIL_SECRET_KEY:-${AWX_SECRET_KEY:-}}" | sha256sum | cut -c1-16)
+{
+    echo "created=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+    echo "forail_version=$(forail-manage version 2>/dev/null || echo unknown)"
+    echo "last_main_migration=${last_migration}"
+    echo "secret_key_sha256_16=${key_fingerprint}"
+} > "${BASE}.meta"
 
-echo "==> Backup saved to ${BACKUP_FILE}"
+echo "==> Backup saved to ${BASE}.dump"
 
 echo "==> Removing backups older than ${RETENTION_DAYS} days..."
-find "${BACKUP_DIR}" -name "forail_backup_*.sql.gz" -mtime "+${RETENTION_DAYS}" -delete
+find "${BACKUP_DIR}" \( -name 'forail_backup_*.dump' -o -name 'forail_backup_*.meta' -o -name 'forail_backup_*.sql.gz' \) -mtime "+${RETENTION_DAYS}" -delete
 
 echo "==> Current backups:"
-ls -lh "${BACKUP_DIR}"/forail_backup_*.sql.gz 2>/dev/null || echo "    (none)"
+ls -lh "${BACKUP_DIR}"/forail_backup_*.dump 2>/dev/null || echo "    (none)"
 
 echo "==> Backup complete."
diff --git a/scripts/restore.sh b/scripts/restore.sh
@@ -1,28 +1,102 @@
 #!/bin/bash
-set -e
+# Restore the Forail database from a backup made by backup.sh.
+#
+# Stop everything that writes to the database first, then run this in a
+# one-off container -- the forail-task container is one of the writers:
+#
+#   docker compose stop forail-web forail-task
+#   docker compose run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes [FILE]
+#   docker compose up -d
+#
+# FILE defaults to the newest backup in $BACKUP_DIR. The current database is
+# replaced, not merged, and kept under another name until you drop it. Exits
+# non-zero, and changes nothing, if any step fails.
+set -euo pipefail
 
 BACKUP_DIR="${BACKUP_DIR:-/var/lib/awx/backups}"
-BACKUP_FILE="${1}"
+CONFIRMED=no
+IGNORE_KEY=no
+FILE=""
+for arg in "$@"; do
+    case "$arg" in
+        --yes) CONFIRMED=yes ;;
+        --ignore-secret-key-mismatch) IGNORE_KEY=yes ;;
+        -*) echo "Unknown option: $arg" >&2; exit 2 ;;
+        *) FILE="$arg" ;;
+    esac
+done
 
-if [ -z "${BACKUP_FILE}" ]; then
-    BACKUP_FILE=$(ls -t "${BACKUP_DIR}"/forail_backup_*.sql.gz 2>/dev/null | head -1)
-    if [ -z "${BACKUP_FILE}" ]; then
-        echo "ERROR: No backup file specified and no backups found in ${BACKUP_DIR}"
+if [ -z "$FILE" ]; then
+    FILE=$(ls -t "${BACKUP_DIR}"/forail_backup_*.dump "${BACKUP_DIR}"/forail_backup_*.sql.gz 2>/dev/null | head -1 || true)
+    [ -n "$FILE" ] || { echo "ERROR: no backup given and none found in ${BACKUP_DIR}" >&2; exit 1; }
+    echo "==> No file given, using the newest: ${FILE}"
+fi
+[ -f "$FILE" ] || { echo "ERROR: backup not found: ${FILE}" >&2; exit 1; }
+
+export PGPASSWORD="${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set}"
+PGARGS=(-h "${POSTGRES_HOST:-postgres}" -p "${POSTGRES_PORT:-5432}" -U "${POSTGRES_USER:-forail}")
+DB="${POSTGRES_DB:-forail}"
+
+# A database restored under a different FORAIL_SECRET_KEY comes up fine and
+# then fails on every credential, token and encrypted setting.
+META="${FILE%.*}.meta"
+if [ -f "$META" ]; then
+    want=$(sed -n 's/^secret_key_sha256_16=//p' "$META")
+    have=$(printf '%s' "${FORAIL_SECRET_KEY:-${AWX_SECRET_KEY:-}}" | sha256sum | cut -c1-16)
+    if [ -n "$want" ] && [ "$want" != "$have" ] && [ "$IGNORE_KEY" != yes ]; then
+        echo "ERROR: this backup was taken with a different FORAIL_SECRET_KEY." >&2
+        echo "       Set the original key in .env before restoring, or every stored credential" >&2
+        echo "       will be unreadable. --ignore-secret-key-mismatch restores anyway." >&2
         exit 1
     fi
-    echo "==> No file specified, using latest: ${BACKUP_FILE}"
+    echo "==> Backup metadata:"; sed 's/^/    /' "$META"
+else
+    echo "==> No metadata next to ${FILE}; cannot check FORAIL_SECRET_KEY."
+fi
+
+active=$(psql "${PGARGS[@]}" -d "$DB" -At -c "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid()")
+if [ "$active" != 0 ]; then
+    echo "ERROR: ${active} other connection(s) to ${DB}. Stop forail-web and forail-task first:" >&2
+    echo "       docker compose stop forail-web forail-task" >&2
+    exit 1
 fi
 
-if [ ! -f "${BACKUP_FILE}" ]; then
-    echo "ERROR: Backup file not found: ${BACKUP_FILE}"
+if [ "$CONFIRMED" != yes ]; then
+    echo "This REPLACES the database ${DB} with ${FILE}. Re-run with --yes to proceed." >&2
     exit 1
 fi
 
-echo "==> Restoring from ${BACKUP_FILE}..."
-gunzip -c "${BACKUP_FILE}" | PGPASSWORD="${POSTGRES_PASSWORD}" psql \
-    -h "${POSTGRES_HOST:-postgres}" \
-    -p "${POSTGRES_PORT:-5432}" \
-    -U "${POSTGRES_USER:-forail}" \
-    -d "${POSTGRES_DB:-forail}"
+# Restore into a fresh database and swap it in by renaming. pg_restore
+# --clean cannot drop Forail's partitioned event tables, and a failed load
+# must not leave a half-restored database behind. The current database is
+# kept, renamed, until you drop it.
+STAMP=$(date +%Y%m%d_%H%M%S)
+NEW="${DB}_restoring"
+OLD="${DB}_before_restore_${STAMP}"
+admin() { psql "${PGARGS[@]}" -d postgres -v ON_ERROR_STOP=1 -q -c "$1"; }
+
+echo "==> Restoring ${FILE} into a new database ${NEW}..."
+admin "DROP DATABASE IF EXISTS \"${NEW}\""
+admin "CREATE DATABASE \"${NEW}\""
+cleanup() { admin "DROP DATABASE IF EXISTS \"${NEW}\"" || true; }
+trap 'echo "ERROR: restore failed; ${DB} is unchanged." >&2; cleanup' ERR
+case "$FILE" in
+    *.dump)
+        pg_restore "${PGARGS[@]}" -d "$NEW" --no-owner --exit-on-error "$FILE"
+        ;;
+    *.sql.gz)
+        # Plain dumps from the old backup.sh.
+        gunzip -c "$FILE" | psql "${PGARGS[@]}" -d "$NEW" -v ON_ERROR_STOP=1 -q
+        ;;
+    *) echo "ERROR: unknown backup format: ${FILE}" >&2; false ;;
+esac
+trap - ERR
+
+echo "==> Swapping: ${DB} -> ${OLD}, ${NEW} -> ${DB}"
+admin "ALTER DATABASE \"${DB}\" RENAME TO \"${OLD}\""
+admin "ALTER DATABASE \"${NEW}\" RENAME TO \"${DB}\""
 
-echo "==> Restore complete."
+echo "==> The previous database is kept as ${OLD}. Once the restore is verified:"
+echo "    docker compose exec postgres psql -U ${POSTGRES_USER:-forail} -d postgres -c 'DROP DATABASE \"${OLD}\"'"
+echo "==> Restore complete. Start the stack: docker compose up -d"
+echo "    If the backup is from an older Forail, forail-init applies the migrations on start."
diff --git a/scripts/test-backup-restore.sh b/scripts/test-backup-restore.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+# End-to-end check of backup.sh and restore.sh against a running stack.
+#
+#   docker compose up -d && bash scripts/test-backup-restore.sh
+#
+# Creates marker data -- an organization and a credential with a secret --
+# backs up, adds a second marker, restores, and checks that the first marker
+# is back, the second is gone, the secret still decrypts, and the stack
+# accepts new writes (restored sequences). Leaves the pre-restore database
+# behind under its kept name; drop it afterwards if you like.
+#
+# Destructive: run it against a throwaway stack, never production.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+dc() { docker compose "$@"; }
+manage() { dc exec -T forail-task forail-manage shell -c "$1" 2>/dev/null | tail -1; }
+fail() { echo "FAIL: $*" >&2; exit 1; }
+
+stamp=$(date +%s)
+before="bkp-before-${stamp}"
+after="bkp-after-${stamp}"
+secret="s3cret-${stamp}"
+
+echo "==> Seeding ${before}"
+manage "
+from forail.main.models import Organization, Credential, CredentialType
+o = Organization.objects.create(name='${before}')
+ct = CredentialType.objects.get(namespace='ssh', managed=True)
+Credential.objects.create(name='${before}', organization=o, credential_type=ct, inputs={'username': 'u', 'password': '${secret}'})
+print('ok')" | grep -qx ok || fail "seeding"
+
+echo "==> Backup"
+dc exec -T forail-task bash /etc/forail/backup.sh
+
+echo "==> Writing ${after} after the backup"
+manage "from forail.main.models import Organization; Organization.objects.create(name='${after}'); print('ok')" | grep -qx ok || fail "second marker"
+
+echo "==> Restore refuses while the stack is running"
+if dc run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes >/dev/null 2>&1; then
+    fail "restore ran with forail-web/forail-task still connected"
+fi
+
+echo "==> Restore"
+dc stop forail-web forail-task
+dc run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes
+dc up -d --wait forail-web forail-task
+
+echo "==> Verifying"
+result=$(manage "
+from forail.main.models import Organization, Credential
+print(Organization.objects.filter(name='${before}').exists(),
+      Organization.objects.filter(name='${after}').exists(),
+      Credential.objects.get(name='${before}').get_input('password') == '${secret}',
+      bool(Organization.objects.create(name='bkp-new-${stamp}').pk))")
+[ "$result" = "True False True True" ] || fail "expected 'True False True True' (before restored, after gone, secret decrypts, new writes work), got '${result}'"
+
+echo "PASS: backup and restore round-trip"