From fd94113984de1b3b05bb069535ff6957b0587572 Mon Sep 17 00:00:00 2001 From: Krstan Vjestica Date: Sun, 4 Oct 2026 18:44:02 +0200 Subject: [PATCH 1/2] fix(backup): a restore that restores, and a backup that fails loudly Tested on a compose stack (2026.07.2-rc1), the documented procedure did not work at any step: - `docker compose exec postgres /scripts/backup.sh` fails: the scripts are mounted into forail-task at /etc/forail/, not into postgres. - backup.sh piped pg_dump into gzip without pipefail, so a failed dump still produced a "backup" and exit 0. - restore.sh psql-loaded a plain dump into the live database: 1,868 "already exists" errors, nothing replaced (data written after the backup survived), "Restore complete." and exit 0. Worse, the dump's setval() calls did run, moving sequences back below existing ids, so the next inserts failed with duplicate keys and forail-init could no longer start. - Nothing recorded which FORAIL_SECRET_KEY a backup belongs to; restored under another key, every credential is unreadable. backup.sh: pg_dump custom format, written under a temporary name and renamed only when complete, pipefail, mode 0600, plus a .meta file with the Forail version, last migration and a fingerprint of the secret key. restore.sh: refuses without --yes, refuses while other sessions are connected (forail-web/forail-task must be stopped; it runs in a one-off `docker compose run --rm --no-deps forail-task` container), refuses on a secret-key mismatch unless told otherwise. It restores into a new database and swaps it in by renaming, so a failure changes nothing and the previous database stays available until dropped. pg_restore --clean is not an option: it cannot drop the partitioned event tables. Old .sql.gz backups still restore. scripts/test-backup-restore.sh runs the round trip against a running stack: marker data and an encrypted credential, backup, a later write, restore refused while running, restore, then checks the marker is back, the later write is gone, the secret decrypts and new writes succeed. It passes on a fresh 2026.07.2-rc1 stack. Also verified by hand: backup on 2026.07.2-rc1, upgrade to an image built from current backend develop work; forail-init applied 0211 and 0212, all data and job history present, credentials decrypt. --- scripts/backup.sh | 47 +++++++++++---- scripts/restore.sh | 106 ++++++++++++++++++++++++++++----- scripts/test-backup-restore.sh | 58 ++++++++++++++++++ 3 files changed, 183 insertions(+), 28 deletions(-) create mode 100644 scripts/test-backup-restore.sh diff --git a/scripts/backup.sh b/scripts/backup.sh index c0c5a6f..dd0e1f5 100644 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -1,27 +1,50 @@ #!/bin/bash -set -e +# Back up the Forail database. +# +# docker compose exec -T forail-task bash /etc/forail/backup.sh +# +# Writes, in $BACKUP_DIR (the forail_backups volume): +# forail_backup_.dump pg_dump custom format (compressed) +# forail_backup_.meta what restore.sh checks before it touches anything +# +# The database is not the whole backup. Credentials, tokens and other secrets +# in it are encrypted with FORAIL_SECRET_KEY; without that key a restored +# database cannot decrypt any of them. Keep .env (or at least +# FORAIL_SECRET_KEY) with the backups, somewhere else than this host. +set -euo pipefail BACKUP_DIR="${BACKUP_DIR:-/var/lib/awx/backups}" RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-7}" -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -BACKUP_FILE="${BACKUP_DIR}/forail_backup_${TIMESTAMP}.sql.gz" +STAMP=$(date +%Y%m%d_%H%M%S) +BASE="${BACKUP_DIR}/forail_backup_${STAMP}" + +export PGPASSWORD="${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set}" +PGARGS=(-h "${POSTGRES_HOST:-postgres}" -p "${POSTGRES_PORT:-5432}" -U "${POSTGRES_USER:-forail}" -d "${POSTGRES_DB:-forail}") mkdir -p "${BACKUP_DIR}" +umask 077 # the dump holds encrypted secrets and every user's data echo "==> Starting backup..." -PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \ - -h "${POSTGRES_HOST:-postgres}" \ - -p "${POSTGRES_PORT:-5432}" \ - -U "${POSTGRES_USER:-forail}" \ - -d "${POSTGRES_DB:-forail}" \ - | gzip > "${BACKUP_FILE}" +# Write to a temporary name first: an interrupted dump must never look like +# the latest good backup to restore.sh. +pg_dump "${PGARGS[@]}" --format=custom --compress=6 --file="${BASE}.dump.partial" +mv "${BASE}.dump.partial" "${BASE}.dump" + +last_migration=$(psql "${PGARGS[@]}" -At -c "SELECT max(name) FROM django_migrations WHERE app = 'main'") +key_fingerprint=$(printf '%s' "${FORAIL_SECRET_KEY:-${AWX_SECRET_KEY:-}}" | sha256sum | cut -c1-16) +{ + echo "created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "forail_version=$(forail-manage version 2>/dev/null || echo unknown)" + echo "last_main_migration=${last_migration}" + echo "secret_key_sha256_16=${key_fingerprint}" +} > "${BASE}.meta" -echo "==> Backup saved to ${BACKUP_FILE}" +echo "==> Backup saved to ${BASE}.dump" echo "==> Removing backups older than ${RETENTION_DAYS} days..." -find "${BACKUP_DIR}" -name "forail_backup_*.sql.gz" -mtime "+${RETENTION_DAYS}" -delete +find "${BACKUP_DIR}" \( -name 'forail_backup_*.dump' -o -name 'forail_backup_*.meta' -o -name 'forail_backup_*.sql.gz' \) -mtime "+${RETENTION_DAYS}" -delete echo "==> Current backups:" -ls -lh "${BACKUP_DIR}"/forail_backup_*.sql.gz 2>/dev/null || echo " (none)" +ls -lh "${BACKUP_DIR}"/forail_backup_*.dump 2>/dev/null || echo " (none)" echo "==> Backup complete." diff --git a/scripts/restore.sh b/scripts/restore.sh index 1adb21a..3c4d012 100644 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -1,28 +1,102 @@ #!/bin/bash -set -e +# Restore the Forail database from a backup made by backup.sh. +# +# Stop everything that writes to the database first, then run this in a +# one-off container -- the forail-task container is one of the writers: +# +# docker compose stop forail-web forail-task +# docker compose run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes [FILE] +# docker compose up -d +# +# FILE defaults to the newest backup in $BACKUP_DIR. The current database is +# replaced, not merged, and kept under another name until you drop it. Exits +# non-zero, and changes nothing, if any step fails. +set -euo pipefail BACKUP_DIR="${BACKUP_DIR:-/var/lib/awx/backups}" -BACKUP_FILE="${1}" +CONFIRMED=no +IGNORE_KEY=no +FILE="" +for arg in "$@"; do + case "$arg" in + --yes) CONFIRMED=yes ;; + --ignore-secret-key-mismatch) IGNORE_KEY=yes ;; + -*) echo "Unknown option: $arg" >&2; exit 2 ;; + *) FILE="$arg" ;; + esac +done -if [ -z "${BACKUP_FILE}" ]; then - BACKUP_FILE=$(ls -t "${BACKUP_DIR}"/forail_backup_*.sql.gz 2>/dev/null | head -1) - if [ -z "${BACKUP_FILE}" ]; then - echo "ERROR: No backup file specified and no backups found in ${BACKUP_DIR}" +if [ -z "$FILE" ]; then + FILE=$(ls -t "${BACKUP_DIR}"/forail_backup_*.dump "${BACKUP_DIR}"/forail_backup_*.sql.gz 2>/dev/null | head -1 || true) + [ -n "$FILE" ] || { echo "ERROR: no backup given and none found in ${BACKUP_DIR}" >&2; exit 1; } + echo "==> No file given, using the newest: ${FILE}" +fi +[ -f "$FILE" ] || { echo "ERROR: backup not found: ${FILE}" >&2; exit 1; } + +export PGPASSWORD="${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set}" +PGARGS=(-h "${POSTGRES_HOST:-postgres}" -p "${POSTGRES_PORT:-5432}" -U "${POSTGRES_USER:-forail}") +DB="${POSTGRES_DB:-forail}" + +# A database restored under a different FORAIL_SECRET_KEY comes up fine and +# then fails on every credential, token and encrypted setting. +META="${FILE%.*}.meta" +if [ -f "$META" ]; then + want=$(sed -n 's/^secret_key_sha256_16=//p' "$META") + have=$(printf '%s' "${FORAIL_SECRET_KEY:-${AWX_SECRET_KEY:-}}" | sha256sum | cut -c1-16) + if [ -n "$want" ] && [ "$want" != "$have" ] && [ "$IGNORE_KEY" != yes ]; then + echo "ERROR: this backup was taken with a different FORAIL_SECRET_KEY." >&2 + echo " Set the original key in .env before restoring, or every stored credential" >&2 + echo " will be unreadable. --ignore-secret-key-mismatch restores anyway." >&2 exit 1 fi - echo "==> No file specified, using latest: ${BACKUP_FILE}" + echo "==> Backup metadata:"; sed 's/^/ /' "$META" +else + echo "==> No metadata next to ${FILE}; cannot check FORAIL_SECRET_KEY." +fi + +active=$(psql "${PGARGS[@]}" -d "$DB" -At -c "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid()") +if [ "$active" != 0 ]; then + echo "ERROR: ${active} other connection(s) to ${DB}. Stop forail-web and forail-task first:" >&2 + echo " docker compose stop forail-web forail-task" >&2 + exit 1 fi -if [ ! -f "${BACKUP_FILE}" ]; then - echo "ERROR: Backup file not found: ${BACKUP_FILE}" +if [ "$CONFIRMED" != yes ]; then + echo "This REPLACES the database ${DB} with ${FILE}. Re-run with --yes to proceed." >&2 exit 1 fi -echo "==> Restoring from ${BACKUP_FILE}..." -gunzip -c "${BACKUP_FILE}" | PGPASSWORD="${POSTGRES_PASSWORD}" psql \ - -h "${POSTGRES_HOST:-postgres}" \ - -p "${POSTGRES_PORT:-5432}" \ - -U "${POSTGRES_USER:-forail}" \ - -d "${POSTGRES_DB:-forail}" +# Restore into a fresh database and swap it in by renaming. pg_restore +# --clean cannot drop Forail's partitioned event tables, and a failed load +# must not leave a half-restored database behind. The current database is +# kept, renamed, until you drop it. +STAMP=$(date +%Y%m%d_%H%M%S) +NEW="${DB}_restoring" +OLD="${DB}_before_restore_${STAMP}" +admin() { psql "${PGARGS[@]}" -d postgres -v ON_ERROR_STOP=1 -q -c "$1"; } + +echo "==> Restoring ${FILE} into a new database ${NEW}..." +admin "DROP DATABASE IF EXISTS \"${NEW}\"" +admin "CREATE DATABASE \"${NEW}\"" +cleanup() { admin "DROP DATABASE IF EXISTS \"${NEW}\"" || true; } +trap 'echo "ERROR: restore failed; ${DB} is unchanged." >&2; cleanup' ERR +case "$FILE" in + *.dump) + pg_restore "${PGARGS[@]}" -d "$NEW" --no-owner --exit-on-error "$FILE" + ;; + *.sql.gz) + # Plain dumps from the old backup.sh. + gunzip -c "$FILE" | psql "${PGARGS[@]}" -d "$NEW" -v ON_ERROR_STOP=1 -q + ;; + *) echo "ERROR: unknown backup format: ${FILE}" >&2; false ;; +esac +trap - ERR + +echo "==> Swapping: ${DB} -> ${OLD}, ${NEW} -> ${DB}" +admin "ALTER DATABASE \"${DB}\" RENAME TO \"${OLD}\"" +admin "ALTER DATABASE \"${NEW}\" RENAME TO \"${DB}\"" -echo "==> Restore complete." +echo "==> The previous database is kept as ${OLD}. Once the restore is verified:" +echo " docker compose exec postgres psql -U ${POSTGRES_USER:-forail} -d postgres -c 'DROP DATABASE \"${OLD}\"'" +echo "==> Restore complete. Start the stack: docker compose up -d" +echo " If the backup is from an older Forail, forail-init applies the migrations on start." diff --git a/scripts/test-backup-restore.sh b/scripts/test-backup-restore.sh new file mode 100644 index 0000000..0da5b27 --- /dev/null +++ b/scripts/test-backup-restore.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# End-to-end check of backup.sh and restore.sh against a running stack. +# +# docker compose up -d && bash scripts/test-backup-restore.sh +# +# Creates marker data -- an organization and a credential with a secret -- +# backs up, adds a second marker, restores, and checks that the first marker +# is back, the second is gone, the secret still decrypts, and the stack +# accepts new writes (restored sequences). Leaves the pre-restore database +# behind under its kept name; drop it afterwards if you like. +# +# Destructive: run it against a throwaway stack, never production. +set -euo pipefail +cd "$(dirname "$0")/.." + +dc() { docker compose "$@"; } +manage() { dc exec -T forail-task forail-manage shell -c "$1" 2>/dev/null | tail -1; } +fail() { echo "FAIL: $*" >&2; exit 1; } + +stamp=$(date +%s) +before="bkp-before-${stamp}" +after="bkp-after-${stamp}" +secret="s3cret-${stamp}" + +echo "==> Seeding ${before}" +manage " +from forail.main.models import Organization, Credential, CredentialType +o = Organization.objects.create(name='${before}') +ct = CredentialType.objects.get(namespace='ssh', managed=True) +Credential.objects.create(name='${before}', organization=o, credential_type=ct, inputs={'username': 'u', 'password': '${secret}'}) +print('ok')" | grep -qx ok || fail "seeding" + +echo "==> Backup" +dc exec -T forail-task bash /etc/forail/backup.sh + +echo "==> Writing ${after} after the backup" +manage "from forail.main.models import Organization; Organization.objects.create(name='${after}'); print('ok')" | grep -qx ok || fail "second marker" + +echo "==> Restore refuses while the stack is running" +if dc run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes >/dev/null 2>&1; then + fail "restore ran with forail-web/forail-task still connected" +fi + +echo "==> Restore" +dc stop forail-web forail-task +dc run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes +dc up -d --wait forail-web forail-task + +echo "==> Verifying" +result=$(manage " +from forail.main.models import Organization, Credential +print(Organization.objects.filter(name='${before}').exists(), + Organization.objects.filter(name='${after}').exists(), + Credential.objects.get(name='${before}').get_input('password') == '${secret}', + bool(Organization.objects.create(name='bkp-new-${stamp}').pk))") +[ "$result" = "True False True True" ] || fail "expected 'True False True True' (before restored, after gone, secret decrypts, new writes work), got '${result}'" + +echo "PASS: backup and restore round-trip" From 66f014eebaa748b47d3e20541ba9e0143f442135 Mon Sep 17 00:00:00 2001 From: Krstan Vjestica Date: Mon, 5 Oct 2026 11:38:05 +0200 Subject: [PATCH 2/2] docs: backup and restore commands that exist The README told operators to run ./scripts/backup.sh and restore.sh on the host, where neither pg_dump nor the database credentials are. Give the commands that actually work -- backup in forail-task, restore in a one-off container with the writers stopped -- and say plainly that the database backup is useless without FORAIL_SECRET_KEY. The full procedure, upgrade and rollback are in the Administrator Handbook. --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f25d0a5..18a3d3d 100644 --- a/README.md +++ b/README.md @@ -134,13 +134,17 @@ outside this repository: ## Backup & Restore ```bash -# Backup -./scripts/backup.sh +# Backup (writes to the forail_backups volume) +docker compose exec -T forail-task bash /etc/forail/backup.sh -# Restore -./scripts/restore.sh /path/to/backup.sql.gz +# Restore: stop the writers, restore in a one-off container, start again +docker compose stop forail-web forail-task +docker compose run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes +docker compose up -d ``` +Keep `.env` -- at least `FORAIL_SECRET_KEY` -- with your backups: without it, a restored database cannot decrypt any stored credential. Details, upgrade and rollback: the [Administrator Handbook](https://forail-platform.github.io/docs/admin-handbook.html#backup). + ## Related Repositories - [forail-backend](https://github.com/forail-platform/forail-backend) — Django API + Task Engine