diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml new file mode 100644 index 0000000..73434bd --- /dev/null +++ b/.github/workflows/db-backup.yml @@ -0,0 +1,124 @@ +name: Production DB logical backup + +# $0 recovery point for the hosted control plane (AUDIT.md finding b: +# pitr_enabled=false, zero provider physical backups as of 2026-07-25). +# Daily `supabase db dump` (roles + schema + data, the exact triple the +# cloud data-safety runbook prescribes), encrypted to an age public key +# committed in ops/backup/age-recipients.txt, uploaded as a workflow +# artifact with 90-day retention. +# +# The private key exists ONLY with the founder (see +# ops/backup/RESTORE_DRILL.md). This repo is PUBLIC: artifacts are +# downloadable by any logged-in GitHub user, so the age encryption is +# load-bearing, not defense-in-depth. Never upload plaintext. +# +# Cost: the repo is public, so Actions minutes and artifact storage are +# free. Daily cadence => RPO up to 24h (logical only). Anything better +# (RPO minutes) requires the paid Supabase PITR add-on — see the +# tradeoff table in ops/backup/RESTORE_DRILL.md. +# +# FAIL-CLOSED: this workflow fails loudly when the DB secret or the age +# recipient is missing, and refuses to upload an empty or schema-only +# dump. A backup job that silently succeeds without a usable backup is +# worse than a red run. + +on: + workflow_dispatch: + schedule: + - cron: '23 7 * * *' # daily 07:23 UTC (off-peak; odd minute to avoid the top-of-hour scheduler crush) + +permissions: + contents: read + +concurrency: + group: db-backup + cancel-in-progress: false + +jobs: + dump: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Require configuration (fail closed) + env: + SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} + run: | + fail=0 + if [ -z "$SUPABASE_DB_URL" ]; then + echo '::error::Secret SUPABASE_DB_URL is not set. Founder action: gh secret set SUPABASE_DB_URL --repo OpenAdaptAI/openadapt-ops (value: the production Postgres connection string from Supabase -> Project Settings -> Database).' + fail=1 + fi + if ! grep -Eq '^age1[0-9a-z]+$' ops/backup/age-recipients.txt; then + echo '::error::ops/backup/age-recipients.txt contains no age recipient. Founder action: generate the keypair per ops/backup/RESTORE_DRILL.md section 1 and commit the PUBLIC key line.' + fail=1 + fi + exit "$fail" + + - uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0 + with: + version: latest # tracks server-version support in `supabase db dump`; the dump SQL itself is what we archive + + - name: Install age + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + age --version + + - name: Dump roles, schema, and data (cloud runbook triple) + env: + SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} + run: | + set -euo pipefail + mkdir -p dump + # Same commands as openadapt-cloud docs/RUNBOOK_DATA_SAFETY.md + # section 2 step 5: a single unflagged dump is schema-only and + # is NOT a recoverable backup. + supabase db dump --db-url "$SUPABASE_DB_URL" -f dump/roles.sql --role-only + supabase db dump --db-url "$SUPABASE_DB_URL" -f dump/schema.sql + supabase db dump --db-url "$SUPABASE_DB_URL" -f dump/data.sql --use-copy --data-only \ + -x 'storage.buckets_vectors' -x 'storage.vector_indexes' + + - name: Refuse an empty or schema-only dump (fail closed) + run: | + set -euo pipefail + test -s dump/roles.sql || { echo '::error::roles.sql is empty'; exit 1; } + grep -q 'CREATE' dump/schema.sql || { echo '::error::schema.sql has no CREATE statements'; exit 1; } + copies=$(grep -c '^COPY ' dump/data.sql || true) + bytes=$(wc -c < dump/data.sql) + echo "data.sql: ${copies} COPY blocks, ${bytes} bytes" + if [ "$copies" -lt 1 ]; then + echo '::error::data.sql contains no COPY blocks — this is schema-only or empty; refusing to upload a worthless artifact.' + exit 1 + fi + + - name: Encrypt to the committed age public key + run: | + set -euo pipefail + stamp=$(date -u +%Y%m%dT%H%M%SZ) + tar -czf "db-backup-${stamp}.tar.gz" -C dump roles.sql schema.sql data.sql + plain_sha=$(sha256sum "db-backup-${stamp}.tar.gz" | cut -d' ' -f1) + age -R ops/backup/age-recipients.txt -o "db-backup-${stamp}.tar.gz.age" "db-backup-${stamp}.tar.gz" + rm -f "db-backup-${stamp}.tar.gz" dump/roles.sql dump/schema.sql dump/data.sql + cipher_sha=$(sha256sum "db-backup-${stamp}.tar.gz.age" | cut -d' ' -f1) + { + echo "created_at_utc=${stamp}" + echo "workflow_run=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "repo_commit=${GITHUB_SHA}" + echo "plaintext_tar_sha256=${plain_sha}" + echo "ciphertext_sha256=${cipher_sha}" + echo "recipients_file_sha256=$(sha256sum ops/backup/age-recipients.txt | cut -d' ' -f1)" + } > manifest.txt + cat manifest.txt + echo "STAMP=${stamp}" >> "$GITHUB_ENV" + + - name: Upload encrypted artifact (90-day retention) + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: db-backup-${{ env.STAMP }} + path: | + db-backup-*.tar.gz.age + manifest.txt + retention-days: 90 + if-no-files-found: error diff --git a/.github/workflows/prod-health-alert.yml b/.github/workflows/prod-health-alert.yml new file mode 100644 index 0000000..9028eff --- /dev/null +++ b/.github/workflows/prod-health-alert.yml @@ -0,0 +1,82 @@ +name: Production health alert + +# $0 pager for the hosted control plane. Probes the real dependency +# probe (openadapt-cloud src/lib/readiness.ts — returns {"ready":true} +# with HTTP 200, or 503 with per-component detail). When the endpoint +# is unhealthy or unreachable after 3 attempts spread over ~2 minutes, +# the run FAILS; GitHub emails the actor who last modified this +# workflow file on a failed scheduled run. Optional Telegram alert +# fires when the crier bot secrets exist in this repo. +# +# Cadence: every 30 minutes = 48 runs/day of a <1-minute job. The repo +# is public, so Actions minutes are free; even metered this would be +# under ~50 min/day. 30 min bounds detection latency without turning +# one transient blip into a page (the in-run retry absorbs blips). +# +# Caveats (documented, not hidden): +# - GitHub auto-disables scheduled workflows after 60 days WITHOUT +# repo activity. sync.yml commits docs daily, which counts as +# activity; if that sync ever stops, these schedules die silently +# ~60 days later. See ops/backup/RESTORE_DRILL.md section 6. +# - Failure emails go to the last committer of this file; keep that a +# monitored account. + +on: + workflow_dispatch: + schedule: + - cron: '*/30 * * * *' + +permissions: + contents: read + +concurrency: + group: prod-health + cancel-in-progress: false + +jobs: + probe: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Probe app.openadapt.ai/api/health/ready (3 attempts over ~2 min) + run: | + set -u + url='https://app.openadapt.ai/api/health/ready' + attempts=3 + for i in $(seq 1 "$attempts"); do + if body=$(curl -sS -m 20 -w '\n%{http_code}' "$url" 2>&1); then + code=$(printf '%s\n' "$body" | tail -n1) + json=$(printf '%s\n' "$body" | sed '$d') + ready=$(printf '%s' "$json" | jq -r '.ready // false' 2>/dev/null || echo 'parse-error') + echo "attempt ${i}/${attempts}: http=${code} ready=${ready}" + if [ "$code" = '200' ] && [ "$ready" = 'true' ]; then + exit 0 + fi + # The endpoint is public and returns metadata-only detail; + # print which components are not ready to make the failure + # email actionable. + printf '%s' "$json" | jq -r '.components[]? | select(.state != "ready") | "NOT READY: \(.name): \(.detail)"' 2>/dev/null || true + else + echo "attempt ${i}/${attempts}: request failed (network error or timeout)" + fi + if [ "$i" -lt "$attempts" ]; then sleep 60; fi + done + echo "::error::${url} is unhealthy or unreachable after ${attempts} attempts over ~2 minutes." + exit 1 + + - name: Telegram alert (optional — runs only when crier secrets exist) + if: ${{ failure() }} + env: + TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} + TELEGRAM_OWNER_ID: ${{ secrets.TELEGRAM_OWNER_ID }} + run: | + set -u + if [ -z "${TELEGRAM_BOT_TOKEN}" ] || [ -z "${TELEGRAM_OWNER_ID}" ]; then + echo 'Telegram secrets not configured; skipping. GitHub failure email is the pager.' + echo 'To enable: gh secret set TELEGRAM_BOT_TOKEN --repo OpenAdaptAI/openadapt-ops && gh secret set TELEGRAM_OWNER_ID --repo OpenAdaptAI/openadapt-ops (same values the crier bot uses).' + exit 0 + fi + curl -sS -m 20 -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ + --data-urlencode "chat_id=${TELEGRAM_OWNER_ID}" \ + --data-urlencode "text=ALERT: app.openadapt.ai /api/health/ready is unhealthy or unreachable. Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + > /dev/null && echo 'Telegram alert sent.' diff --git a/ops/backup/RESTORE_DRILL.md b/ops/backup/RESTORE_DRILL.md new file mode 100644 index 0000000..9caf73d --- /dev/null +++ b/ops/backup/RESTORE_DRILL.md @@ -0,0 +1,141 @@ +# Production DB backup: keypair, restore drill, and paid-vs-free tradeoff + +Companion to `.github/workflows/db-backup.yml` (daily encrypted logical backup) +and `.github/workflows/prod-health-alert.yml` (30-minute health pager). +Authoritative context: `openadapt-cloud` `AUDIT.md` finding (b) and +`docs/RUNBOOK_DATA_SAFETY.md` (the provider reported `pitr_enabled=false` and +zero physical backups on 2026-07-25). + +**Threat model note — this repo is PUBLIC.** Workflow artifacts on a public +repo are downloadable by any logged-in GitHub user. The backup artifact is +therefore ciphertext only (`age`), encrypted to the public key committed in +`ops/backup/age-recipients.txt`. The private key exists only offline with the +founder. If the private key is ever exposed, rotate (section 5) and treat all +retained artifacts as exposed. + +## 1. One-time founder setup: generate the age keypair + +On a trusted machine (not CI): + +```sh +brew install age # or: apt-get install age +umask 077 +age-keygen -o ~/openadapt-db-backup.agekey +``` + +`age-keygen` prints one line `# public key: age1...`. + +1. Store `~/openadapt-db-backup.agekey` (the PRIVATE key) in the team vault + (e.g. 1Password) AND on an offline copy. It never goes into this repo, + GitHub secrets, or any CI log. Without it, every backup artifact is + permanently unreadable. +2. Commit ONLY the public key: append the `age1...` line (just the key, no + `# public key:` prefix) to `ops/backup/age-recipients.txt` on a branch and + open a PR. +3. Set the database secret (value: the production Postgres connection string, + Supabase dashboard -> Project Settings -> Database -> Connection string, + direct or session-pooler URI): + +```sh +gh secret set SUPABASE_DB_URL --repo OpenAdaptAI/openadapt-ops +``` + +The backup workflow fails loudly until both steps are done. That is +intentional: a red daily run is the reminder that production still has no +recovery point. + +## 2. What each daily artifact contains + +Artifact `db-backup-` (retention 90 days): + +- `db-backup-.tar.gz.age` — age-encrypted tarball of the runbook + triple: `roles.sql`, `schema.sql`, `data.sql` (`--use-copy --data-only`, + excluding `storage.buckets_vectors` / `storage.vector_indexes`). +- `manifest.txt` — timestamps, run URL, plaintext and ciphertext SHA-256. + +Storage buckets (bundles/reports/recordings) are NOT in this backup; database +PITR would not cover them either. Bucket export/restore stays with the cloud +runbook's `retention` drill tooling. + +## 3. Restore drill (run on a scratch database, never prod) + +Quarterly, or before any risky migration. Mirrors `RUNBOOK_DATA_SAFETY.md` +section 2 steps 6-8; the scratch project must be new, disposable, and in the +same regional/data-handling boundary as production. + +```sh +umask 077 +mkdir -p /secure/offline/openadapt-restore && cd /secure/offline/openadapt-restore + +# 1. Download the newest artifact (list runs, then download by run id): +gh run list --repo OpenAdaptAI/openadapt-ops --workflow db-backup.yml --limit 5 +gh run download --repo OpenAdaptAI/openadapt-ops + +# 2. Verify + decrypt with the founder-held private key: +cd db-backup- +sha256sum db-backup-.tar.gz.age # must equal ciphertext_sha256 in manifest.txt +age -d -i ~/openadapt-db-backup.agekey \ + -o db-backup-.tar.gz db-backup-.tar.gz.age +sha256sum db-backup-.tar.gz # must equal plaintext_tar_sha256 in manifest.txt +tar -xzf db-backup-.tar.gz # -> roles.sql schema.sql data.sql + +# 3. Create a brand-new scratch Supabase project (dashboard). Then restore +# in one transaction, same order and flags as the cloud runbook: +psql --single-transaction --variable ON_ERROR_STOP=1 \ + --file roles.sql \ + --file schema.sql \ + --command 'SET session_replication_role = replica' \ + --file data.sql \ + --dbname "$SCRATCH_DB_URL" + +# 4. Validate: row counts on runs/orgs/usage vs production expectations; +# spot-check one recent run row. Record the wall-clock restore time (RTO). + +# 5. Decommission the scratch project and delete the local plaintext: +rm -f roles.sql schema.sql data.sql db-backup-.tar.gz +``` + +Restore caveats (same as PITR): after any real restore, reconcile Stripe by +replaying webhooks from the Stripe dashboard (`stripe_events` is idempotent), +and expect up to 24h of lost writes (see RPO below). + +## 4. Can we get to a recovery point for $0 — and what does paying buy? + +**Yes.** Once section 1 is done, this workflow gives a daily, encrypted, +off-provider logical recovery point at $0 (public repo: Actions minutes and +artifact storage are free; the Supabase Free plan itself includes no backups). + +| Option | Monthly cost (Supabase public pricing, checked 2026-08-02) | RPO | Restore | Covers | +|---|---|---|---|---| +| This workflow (`pg_dump` daily, encrypted artifact, 90d retention) | $0 | up to 24h | manual, logical (`psql`), drill above; hours | DB only (roles+schema+data) | +| Supabase Pro (daily physical backups, 7-day retention) | $25 | up to 24h | dashboard-driven physical restore; also unlocks restore-to-new-project | DB only | +| Supabase Pro + PITR add-on | $25 + $100 per 7 days of PITR retention (+ compute add-on if the project is below the required tier, ~$10-15 net of Pro's $10 compute credit) | minutes | dashboard PITR to timestamp | DB only (WAL) | + +**Recommendation:** buy Supabase Pro ($25/mo) now — it is the cheapest change +that adds provider-side physical backups and makes `provider-status +--require-recovery` pass, and hosted retention/deletion gating in +openadapt-cloud wants provider recovery evidence. Defer the PITR add-on +(~$110-140/mo all-in) until there is at least one paying pilot; at N=0 +customers, a 24h RPO with a drilled restore path is a defensible posture, and +this workflow keeps an independent, off-provider copy either way. Revisit the +moment real customer billing/usage state is at stake. + +## 5. Key rotation / compromise + +1. Generate a new keypair (section 1); commit the new public key line. +2. Keep the old private key in the vault until every artifact encrypted to it + has aged out (90 days), then destroy it. +3. If the private key was exposed: rotate immediately AND rotate the database + credentials in `SUPABASE_DB_URL` (the artifacts contain full table data). + +## 6. Alerting caveats (prod-health-alert.yml) + +- GitHub emails the actor who last modified the workflow file when a scheduled + run fails. Keep that account monitored. +- GitHub auto-disables scheduled workflows after 60 days with no repo + activity. `sync.yml` commits docs daily, which keeps schedules alive; if the + doc sync ever stops, both schedules here die silently ~60 days later — + check Actions -> enabled state during any quarterly drill. +- Optional Telegram page: set the crier bot's `TELEGRAM_BOT_TOKEN` and + `TELEGRAM_OWNER_ID` as repo secrets (commands in the workflow log) and the + failure step will also message the founder directly. diff --git a/ops/backup/age-recipients.txt b/ops/backup/age-recipients.txt new file mode 100644 index 0000000..f78e4b0 --- /dev/null +++ b/ops/backup/age-recipients.txt @@ -0,0 +1,9 @@ +# age recipients for production DB backups (.github/workflows/db-backup.yml). +# +# This file must contain exactly the PUBLIC key line(s) of the founder's +# age keypair — one `age1...` recipient per line. The PRIVATE key never +# enters this repo, GitHub secrets, or CI. Generation procedure: +# ops/backup/RESTORE_DRILL.md section 1. +# +# The backup workflow FAILS (on purpose) until a real recipient line +# exists below. Do not put anything secret in this file.