diff --git a/README.md b/README.md index f25d0a5..18a3d3d 100644 --- a/README.md +++ b/README.md @@ -134,13 +134,17 @@ outside this repository: ## Backup & Restore ```bash -# Backup -./scripts/backup.sh +# Backup (writes to the forail_backups volume) +docker compose exec -T forail-task bash /etc/forail/backup.sh -# Restore -./scripts/restore.sh /path/to/backup.sql.gz +# Restore: stop the writers, restore in a one-off container, start again +docker compose stop forail-web forail-task +docker compose run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes +docker compose up -d ``` +Keep `.env` -- at least `FORAIL_SECRET_KEY` -- with your backups: without it, a restored database cannot decrypt any stored credential. Details, upgrade and rollback: the [Administrator Handbook](https://forail-platform.github.io/docs/admin-handbook.html#backup). + ## Related Repositories - [forail-backend](https://github.com/forail-platform/forail-backend) — Django API + Task Engine diff --git a/scripts/backup.sh b/scripts/backup.sh index c0c5a6f..dd0e1f5 100644 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -1,27 +1,50 @@ #!/bin/bash -set -e +# Back up the Forail database. +# +# docker compose exec -T forail-task bash /etc/forail/backup.sh +# +# Writes, in $BACKUP_DIR (the forail_backups volume): +# forail_backup_.dump pg_dump custom format (compressed) +# forail_backup_.meta what restore.sh checks before it touches anything +# +# The database is not the whole backup. Credentials, tokens and other secrets +# in it are encrypted with FORAIL_SECRET_KEY; without that key a restored +# database cannot decrypt any of them. Keep .env (or at least +# FORAIL_SECRET_KEY) with the backups, somewhere else than this host. +set -euo pipefail BACKUP_DIR="${BACKUP_DIR:-/var/lib/awx/backups}" RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-7}" -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -BACKUP_FILE="${BACKUP_DIR}/forail_backup_${TIMESTAMP}.sql.gz" +STAMP=$(date +%Y%m%d_%H%M%S) +BASE="${BACKUP_DIR}/forail_backup_${STAMP}" + +export PGPASSWORD="${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set}" +PGARGS=(-h "${POSTGRES_HOST:-postgres}" -p "${POSTGRES_PORT:-5432}" -U "${POSTGRES_USER:-forail}" -d "${POSTGRES_DB:-forail}") mkdir -p "${BACKUP_DIR}" +umask 077 # the dump holds encrypted secrets and every user's data echo "==> Starting backup..." -PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \ - -h "${POSTGRES_HOST:-postgres}" \ - -p "${POSTGRES_PORT:-5432}" \ - -U "${POSTGRES_USER:-forail}" \ - -d "${POSTGRES_DB:-forail}" \ - | gzip > "${BACKUP_FILE}" +# Write to a temporary name first: an interrupted dump must never look like +# the latest good backup to restore.sh. +pg_dump "${PGARGS[@]}" --format=custom --compress=6 --file="${BASE}.dump.partial" +mv "${BASE}.dump.partial" "${BASE}.dump" + +last_migration=$(psql "${PGARGS[@]}" -At -c "SELECT max(name) FROM django_migrations WHERE app = 'main'") +key_fingerprint=$(printf '%s' "${FORAIL_SECRET_KEY:-${AWX_SECRET_KEY:-}}" | sha256sum | cut -c1-16) +{ + echo "created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "forail_version=$(forail-manage version 2>/dev/null || echo unknown)" + echo "last_main_migration=${last_migration}" + echo "secret_key_sha256_16=${key_fingerprint}" +} > "${BASE}.meta" -echo "==> Backup saved to ${BACKUP_FILE}" +echo "==> Backup saved to ${BASE}.dump" echo "==> Removing backups older than ${RETENTION_DAYS} days..." -find "${BACKUP_DIR}" -name "forail_backup_*.sql.gz" -mtime "+${RETENTION_DAYS}" -delete +find "${BACKUP_DIR}" \( -name 'forail_backup_*.dump' -o -name 'forail_backup_*.meta' -o -name 'forail_backup_*.sql.gz' \) -mtime "+${RETENTION_DAYS}" -delete echo "==> Current backups:" -ls -lh "${BACKUP_DIR}"/forail_backup_*.sql.gz 2>/dev/null || echo " (none)" +ls -lh "${BACKUP_DIR}"/forail_backup_*.dump 2>/dev/null || echo " (none)" echo "==> Backup complete." diff --git a/scripts/restore.sh b/scripts/restore.sh index 1adb21a..3c4d012 100644 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -1,28 +1,102 @@ #!/bin/bash -set -e +# Restore the Forail database from a backup made by backup.sh. +# +# Stop everything that writes to the database first, then run this in a +# one-off container -- the forail-task container is one of the writers: +# +# docker compose stop forail-web forail-task +# docker compose run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes [FILE] +# docker compose up -d +# +# FILE defaults to the newest backup in $BACKUP_DIR. The current database is +# replaced, not merged, and kept under another name until you drop it. Exits +# non-zero, and changes nothing, if any step fails. +set -euo pipefail BACKUP_DIR="${BACKUP_DIR:-/var/lib/awx/backups}" -BACKUP_FILE="${1}" +CONFIRMED=no +IGNORE_KEY=no +FILE="" +for arg in "$@"; do + case "$arg" in + --yes) CONFIRMED=yes ;; + --ignore-secret-key-mismatch) IGNORE_KEY=yes ;; + -*) echo "Unknown option: $arg" >&2; exit 2 ;; + *) FILE="$arg" ;; + esac +done -if [ -z "${BACKUP_FILE}" ]; then - BACKUP_FILE=$(ls -t "${BACKUP_DIR}"/forail_backup_*.sql.gz 2>/dev/null | head -1) - if [ -z "${BACKUP_FILE}" ]; then - echo "ERROR: No backup file specified and no backups found in ${BACKUP_DIR}" +if [ -z "$FILE" ]; then + FILE=$(ls -t "${BACKUP_DIR}"/forail_backup_*.dump "${BACKUP_DIR}"/forail_backup_*.sql.gz 2>/dev/null | head -1 || true) + [ -n "$FILE" ] || { echo "ERROR: no backup given and none found in ${BACKUP_DIR}" >&2; exit 1; } + echo "==> No file given, using the newest: ${FILE}" +fi +[ -f "$FILE" ] || { echo "ERROR: backup not found: ${FILE}" >&2; exit 1; } + +export PGPASSWORD="${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set}" +PGARGS=(-h "${POSTGRES_HOST:-postgres}" -p "${POSTGRES_PORT:-5432}" -U "${POSTGRES_USER:-forail}") +DB="${POSTGRES_DB:-forail}" + +# A database restored under a different FORAIL_SECRET_KEY comes up fine and +# then fails on every credential, token and encrypted setting. +META="${FILE%.*}.meta" +if [ -f "$META" ]; then + want=$(sed -n 's/^secret_key_sha256_16=//p' "$META") + have=$(printf '%s' "${FORAIL_SECRET_KEY:-${AWX_SECRET_KEY:-}}" | sha256sum | cut -c1-16) + if [ -n "$want" ] && [ "$want" != "$have" ] && [ "$IGNORE_KEY" != yes ]; then + echo "ERROR: this backup was taken with a different FORAIL_SECRET_KEY." >&2 + echo " Set the original key in .env before restoring, or every stored credential" >&2 + echo " will be unreadable. --ignore-secret-key-mismatch restores anyway." >&2 exit 1 fi - echo "==> No file specified, using latest: ${BACKUP_FILE}" + echo "==> Backup metadata:"; sed 's/^/ /' "$META" +else + echo "==> No metadata next to ${FILE}; cannot check FORAIL_SECRET_KEY." +fi + +active=$(psql "${PGARGS[@]}" -d "$DB" -At -c "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid()") +if [ "$active" != 0 ]; then + echo "ERROR: ${active} other connection(s) to ${DB}. Stop forail-web and forail-task first:" >&2 + echo " docker compose stop forail-web forail-task" >&2 + exit 1 fi -if [ ! -f "${BACKUP_FILE}" ]; then - echo "ERROR: Backup file not found: ${BACKUP_FILE}" +if [ "$CONFIRMED" != yes ]; then + echo "This REPLACES the database ${DB} with ${FILE}. Re-run with --yes to proceed." >&2 exit 1 fi -echo "==> Restoring from ${BACKUP_FILE}..." -gunzip -c "${BACKUP_FILE}" | PGPASSWORD="${POSTGRES_PASSWORD}" psql \ - -h "${POSTGRES_HOST:-postgres}" \ - -p "${POSTGRES_PORT:-5432}" \ - -U "${POSTGRES_USER:-forail}" \ - -d "${POSTGRES_DB:-forail}" +# Restore into a fresh database and swap it in by renaming. pg_restore +# --clean cannot drop Forail's partitioned event tables, and a failed load +# must not leave a half-restored database behind. The current database is +# kept, renamed, until you drop it. +STAMP=$(date +%Y%m%d_%H%M%S) +NEW="${DB}_restoring" +OLD="${DB}_before_restore_${STAMP}" +admin() { psql "${PGARGS[@]}" -d postgres -v ON_ERROR_STOP=1 -q -c "$1"; } + +echo "==> Restoring ${FILE} into a new database ${NEW}..." +admin "DROP DATABASE IF EXISTS \"${NEW}\"" +admin "CREATE DATABASE \"${NEW}\"" +cleanup() { admin "DROP DATABASE IF EXISTS \"${NEW}\"" || true; } +trap 'echo "ERROR: restore failed; ${DB} is unchanged." >&2; cleanup' ERR +case "$FILE" in + *.dump) + pg_restore "${PGARGS[@]}" -d "$NEW" --no-owner --exit-on-error "$FILE" + ;; + *.sql.gz) + # Plain dumps from the old backup.sh. + gunzip -c "$FILE" | psql "${PGARGS[@]}" -d "$NEW" -v ON_ERROR_STOP=1 -q + ;; + *) echo "ERROR: unknown backup format: ${FILE}" >&2; false ;; +esac +trap - ERR + +echo "==> Swapping: ${DB} -> ${OLD}, ${NEW} -> ${DB}" +admin "ALTER DATABASE \"${DB}\" RENAME TO \"${OLD}\"" +admin "ALTER DATABASE \"${NEW}\" RENAME TO \"${DB}\"" -echo "==> Restore complete." +echo "==> The previous database is kept as ${OLD}. Once the restore is verified:" +echo " docker compose exec postgres psql -U ${POSTGRES_USER:-forail} -d postgres -c 'DROP DATABASE \"${OLD}\"'" +echo "==> Restore complete. Start the stack: docker compose up -d" +echo " If the backup is from an older Forail, forail-init applies the migrations on start." diff --git a/scripts/test-backup-restore.sh b/scripts/test-backup-restore.sh new file mode 100644 index 0000000..0da5b27 --- /dev/null +++ b/scripts/test-backup-restore.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# End-to-end check of backup.sh and restore.sh against a running stack. +# +# docker compose up -d && bash scripts/test-backup-restore.sh +# +# Creates marker data -- an organization and a credential with a secret -- +# backs up, adds a second marker, restores, and checks that the first marker +# is back, the second is gone, the secret still decrypts, and the stack +# accepts new writes (restored sequences). Leaves the pre-restore database +# behind under its kept name; drop it afterwards if you like. +# +# Destructive: run it against a throwaway stack, never production. +set -euo pipefail +cd "$(dirname "$0")/.." + +dc() { docker compose "$@"; } +manage() { dc exec -T forail-task forail-manage shell -c "$1" 2>/dev/null | tail -1; } +fail() { echo "FAIL: $*" >&2; exit 1; } + +stamp=$(date +%s) +before="bkp-before-${stamp}" +after="bkp-after-${stamp}" +secret="s3cret-${stamp}" + +echo "==> Seeding ${before}" +manage " +from forail.main.models import Organization, Credential, CredentialType +o = Organization.objects.create(name='${before}') +ct = CredentialType.objects.get(namespace='ssh', managed=True) +Credential.objects.create(name='${before}', organization=o, credential_type=ct, inputs={'username': 'u', 'password': '${secret}'}) +print('ok')" | grep -qx ok || fail "seeding" + +echo "==> Backup" +dc exec -T forail-task bash /etc/forail/backup.sh + +echo "==> Writing ${after} after the backup" +manage "from forail.main.models import Organization; Organization.objects.create(name='${after}'); print('ok')" | grep -qx ok || fail "second marker" + +echo "==> Restore refuses while the stack is running" +if dc run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes >/dev/null 2>&1; then + fail "restore ran with forail-web/forail-task still connected" +fi + +echo "==> Restore" +dc stop forail-web forail-task +dc run --rm --no-deps forail-task bash /etc/forail/restore.sh --yes +dc up -d --wait forail-web forail-task + +echo "==> Verifying" +result=$(manage " +from forail.main.models import Organization, Credential +print(Organization.objects.filter(name='${before}').exists(), + Organization.objects.filter(name='${after}').exists(), + Credential.objects.get(name='${before}').get_input('password') == '${secret}', + bool(Organization.objects.create(name='bkp-new-${stamp}').pk))") +[ "$result" = "True False True True" ] || fail "expected 'True False True True' (before restored, after gone, secret decrypts, new writes work), got '${result}'" + +echo "PASS: backup and restore round-trip"