ops: encrypted backup sidecar for the deletion ledger

.github/workflows/docker.yml

@@ -721,9 +721,10 @@ jobs:
           # TC-959: main's N0 compose (#349 c33c2302…) with the 95m ingress body
           # cap (raw d95bfb70…), the TC-1274 + TC-1317 + TC-1482 + c_index + TC-1840 + TC-1843 Vector config and the
           # optional TC-1482 NODE_STATS_READ_TOKEN env line, the TC-950 pool of 10 and the TC-1777 named-keys gate
-          # env lines + deletion-ledger volume (raw 298ea356…; 192dfb7e… before TC-1777),
+          # env lines + deletion-ledger volume, and the TC-887 ledger-backup sidecar pinned by digest
+          # (raw 81412971…; 298ea356… before TC-887; 192dfb7e… before TC-1777),
           # its single node digest normalized to 64 zeros.
-          n0_template_hash = 'c9bcbea9dac90fc2b74d8439146596ba802796a211d1b6d65aeda2286d7ed028'
+          n0_template_hash = '0d57efc103654fe3ec361680bdd0f264d4a9db111b1a48e970f28980f4805bbc'
           node_pin = re.compile(rb'(?m)^    image: ghcr\.io/tinycloudlabs/tinycloud-node@sha256:([0-9a-f]{64})$')
           def n0_template(raw):
               pins = list(node_pin.finditer(raw))
@@ -1000,6 +1001,11 @@ jobs:
           PHALA_ENV_BETTERSTACK_SOURCE_TOKEN: ${{ secrets.PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN }}
           PHALA_ENV_SQL_IDENTITY_FENCE: ${{ inputs.sql_identity_fence }}
           PHALA_ENV_NAMED_KEYS_V1: ${{ inputs.named_keys_v1 }}
+          # TC-887: the deletion-ledger backup sidecar (deploy/ledger-backup/README.md).
+          PHALA_ENV_NODE_BACKUP_R2_ENDPOINT: ${{ secrets.PROD_TINYCLOUD_NODE_BACKUP_R2_ENDPOINT }}
+          PHALA_ENV_NODE_BACKUP_R2_ACCESS_KEY_ID: ${{ secrets.PROD_TINYCLOUD_NODE_BACKUP_R2_ACCESS_KEY_ID }}
+          PHALA_ENV_NODE_BACKUP_R2_SECRET_ACCESS_KEY: ${{ secrets.PROD_TINYCLOUD_NODE_BACKUP_R2_SECRET_ACCESS_KEY }}
+          PHALA_ENV_NODE_BACKUP_AGE_RECIPIENT: ${{ secrets.PROD_TINYCLOUD_NODE_BACKUP_AGE_RECIPIENT }}
         # Update the existing prod CVM (name: tinycloud-node) in the
         # Tiny Cloud workspace. The PHALA_CLOUD_API_KEY secret is
         # workspace-scoped and must be issued from that workspace.
@@ -1026,6 +1032,10 @@ jobs:
             PHALA_ENV_BETTERSTACK_SOURCE_TOKEN
             PHALA_ENV_SQL_IDENTITY_FENCE
             PHALA_ENV_NAMED_KEYS_V1
+            PHALA_ENV_NODE_BACKUP_R2_ENDPOINT
+            PHALA_ENV_NODE_BACKUP_R2_ACCESS_KEY_ID
+            PHALA_ENV_NODE_BACKUP_R2_SECRET_ACCESS_KEY
+            PHALA_ENV_NODE_BACKUP_AGE_RECIPIENT
           )
 
           for var in "${required_vars[@]}"; do
@@ -1046,6 +1056,20 @@ jobs:
             exit 1
           fi
 
+          # TC-887: the CVM may hold only the age PUBLIC recipient. Refuse an
+          # identity (private key) or anything else before it is written.
+          case "${PHALA_ENV_NODE_BACKUP_AGE_RECIPIENT}" in
+            AGE-SECRET-KEY-*)
+              echo "::error::NODE_BACKUP_AGE_RECIPIENT is a private key; only the age1... public recipient may be sealed"
+              exit 1
+              ;;
+            age1*) ;;
+            *)
+              echo "::error::NODE_BACKUP_AGE_RECIPIENT must be an age1... public recipient"
+              exit 1
+              ;;
+          esac
+
           # Optional (TC-1482): the same single-line rule, but absence is allowed.
           if [[ "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" == *$'\n'* || "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" == *$'\r'* ]]; then
             echo "::error::Phala deploy inputs must be single-line values"
@@ -1091,6 +1115,10 @@ jobs:
             printf 'BETTERSTACK_SOURCE_TOKEN=%s\n' "$PHALA_ENV_BETTERSTACK_SOURCE_TOKEN"
             printf 'SQL_IDENTITY_FENCE=%s\n' "$PHALA_ENV_SQL_IDENTITY_FENCE"
             printf 'NAMED_KEYS_V1=%s\n' "$PHALA_ENV_NAMED_KEYS_V1"
+            printf 'NODE_BACKUP_R2_ENDPOINT=%s\n' "$PHALA_ENV_NODE_BACKUP_R2_ENDPOINT"
+            printf 'NODE_BACKUP_R2_ACCESS_KEY_ID=%s\n' "$PHALA_ENV_NODE_BACKUP_R2_ACCESS_KEY_ID"
+            printf 'NODE_BACKUP_R2_SECRET_ACCESS_KEY=%s\n' "$PHALA_ENV_NODE_BACKUP_R2_SECRET_ACCESS_KEY"
+            printf 'NODE_BACKUP_AGE_RECIPIENT=%s\n' "$PHALA_ENV_NODE_BACKUP_AGE_RECIPIENT"
             # TC-1482: optional. Unset keeps the node starting with
             # GET /admin/stats disabled (404); never written empty.
             if [ -n "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" ]; then

.github/workflows/ledger-backup.yml

@@ -0,0 +1,148 @@
+name: ledger-backup
+
+# TC-887: the deletion-ledger backup sidecar (deploy/ledger-backup/).
+# - scripts: the backup loop and the restore drill against an S3 stub,
+#   including a node appending while the backup copies.
+# - image: builds the sidecar from this revision; same-repository runs publish
+#   it to GHCR as :<revision>, labelled with the sha256 of its sources.
+# - pinned: the image docker-compose.dstack-postgres.yaml pins must carry this
+#   revision's sidecar sources, and passes backup + restore drill end to end in
+#   a container against MinIO.
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+    paths:
+      - 'deploy/ledger-backup/**'
+      - 'docker-compose.dstack-postgres.yaml'
+      - 'scripts/at-rest-harness/harness.sh'
+      - '.github/workflows/ledger-backup.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'deploy/ledger-backup/**'
+      - 'docker-compose.dstack-postgres.yaml'
+      - 'scripts/at-rest-harness/harness.sh'
+      - '.github/workflows/ledger-backup.yml'
+
+permissions:
+  contents: read
+
+env:
+  IMAGE: ghcr.io/tinycloudlabs/tinycloud-node-ledger-backup
+  # The files the image is built from; their sha256 is the image's source label.
+  SOURCES: deploy/ledger-backup/Dockerfile deploy/ledger-backup/ledger-backup.sh
+
+jobs:
+  scripts:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - name: Install age
+        run: sudo apt-get update -qq && sudo apt-get install -y -qq age
+      - name: Backup loop and restore drill against an S3 stub
+        env:
+          LEDGER_BACKUP_TEST_REQUIRE: '1'
+          LEDGER_BACKUP_TEST_TMP: ${{ runner.temp }}
+        run: node --test deploy/ledger-backup/ledger-backup.test.mjs
+
+  image:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - name: Source label
+        id: source
+        run: |
+          set -euo pipefail
+          echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          echo "sha256=$(cat $SOURCES | sha256sum | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT"
+      - uses: docker/setup-buildx-action@v3
+      - name: Log in to GHCR
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build (and publish for same-repository runs)
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: deploy/ledger-backup/Dockerfile
+          platforms: linux/amd64
+          provenance: false
+          push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
+          tags: ${{ env.IMAGE }}:${{ steps.source.outputs.revision }}
+          labels: |
+            org.opencontainers.image.revision=${{ steps.source.outputs.revision }}
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            xyz.tinycloud.ledger-backup.sources-sha256=${{ steps.source.outputs.sha256 }}
+      - name: Report the image
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+          SOURCES_SHA256: ${{ steps.source.outputs.sha256 }}
+        run: |
+          echo "Built ${IMAGE}@${DIGEST} (sources sha256 ${SOURCES_SHA256})" | tee -a "$GITHUB_STEP_SUMMARY"
+
+  pinned:
+    needs: scripts
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - name: Read the pinned image from the production compose
+        id: pin
+        run: |
+          set -euo pipefail
+          images=$(sed -n -E 's/^    image: (ghcr\.io\/tinycloudlabs\/tinycloud-node-ledger-backup@sha256:[0-9a-f]{64})$/\1/p' docker-compose.dstack-postgres.yaml)
+          [ "$(printf '%s\n' "$images" | awk 'NF { n += 1 } END { print n + 0 }')" = 1 ] \
+            || { echo "::error::docker-compose.dstack-postgres.yaml must pin exactly one ${IMAGE}@sha256 image"; exit 1; }
+          echo "image=$images" >> "$GITHUB_OUTPUT"
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: The pinned image is built from this revision's sidecar sources
+        env:
+          PINNED: ${{ steps.pin.outputs.image }}
+        run: |
+          set -euo pipefail
+          docker pull --quiet "$PINNED"
+          label=$(docker image inspect --format '{{ index .Config.Labels "xyz.tinycloud.ledger-backup.sources-sha256" }}' "$PINNED")
+          here=$(cat $SOURCES | sha256sum | cut -d ' ' -f 1)
+          [ "$label" = "$here" ] || { echo "::error::The pinned image was built from other sidecar sources (label ${label}, this revision ${here}). Pin the digest the image job publishes for this revision."; exit 1; }
+          echo "Pinned ${PINNED}: sources sha256 ${here}, built at revision $(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$PINNED")" | tee -a "$GITHUB_STEP_SUMMARY"
+      - name: Install age
+        run: sudo apt-get update -qq && sudo apt-get install -y -qq age
+      - name: Fetch the pinned MinIO (sha256-verified)
+        env:
+          TC887_BIN_DIR: ${{ runner.temp }}/bin
+        run: bash scripts/at-rest-harness/harness.sh bins
+      - name: Backup and restore drill with the pinned image against MinIO
+        env:
+          MINIO_BIN: ${{ runner.temp }}/bin/minio
+          PINNED: ${{ steps.pin.outputs.image }}
+        run: bash deploy/ledger-backup/ci-e2e.sh "$PINNED"

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## [Unreleased]
 
+### Changes
+- Production compose: a `ledger-backup` sidecar backs up the named-keys deletion ledger, the only data that exists solely on the CVM disk. It reads the `deletion-ledger` volume read-only and without a lock, keeps whole lines re-checked against the live file, encrypts them with `age` to a public recipient and uploads them to the R2 bucket `tinycloud-node-backups` every 300 s when the ledger changed and at least daily. `deploy/ledger-backup/ledger-restore-drill.sh` decrypts the newest backup into a scratch directory and checks it. This changes the compose hash (N0 template lock in `docker.yml` and the fixture move with it), and every `docker.yml` deploy now needs the `PROD_TINYCLOUD_NODE_BACKUP_*` secrets (`NODE_BACKUP_R2_ENDPOINT`, `NODE_BACKUP_R2_ACCESS_KEY_ID`, `NODE_BACKUP_R2_SECRET_ACCESS_KEY`, `NODE_BACKUP_AGE_RECIPIENT`); the node itself is unchanged. Details: TC-887
+
 ## [1.25.15] - 2026-11-01
 
 ### Changes

deploy/ledger-backup/Dockerfile

@@ -0,0 +1,16 @@
+# Sidecar image for the encrypted deletion-ledger backup loop (TC-887).
+# Alpine keeps only the current package revision per release; if a pin below
+# stops resolving, the build fails loudly. Bump the pins together with the base
+# digest (Alpine 3.24.3 at this digest).
+FROM alpine:3.24@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a
+# /var/lib/tinycloud/ledger gets the node image's owner and mode: if this
+# container is the first to mount a fresh `deletion-ledger` volume, Docker
+# seeds the volume from here and the node can still write it.
+RUN apk add --no-cache age=1.3.2-r1 curl=8.23.0-r0 \
+    && addgroup -g 1000 tinycloud && adduser -u 1000 -G tinycloud -s /sbin/nologin -D -H tinycloud \
+    && mkdir -p /var/lib/tinycloud/ledger && chown 1000:1000 /var/lib/tinycloud/ledger && chmod 700 /var/lib/tinycloud/ledger
+COPY --chmod=0755 deploy/ledger-backup/ledger-backup.sh /usr/local/bin/ledger-backup.sh
+USER 1000:1000
+ENTRYPOINT ["/usr/local/bin/ledger-backup.sh"]
+CMD []
+LABEL org.opencontainers.image.source=https://github.com/TinyCloudLabs/tinycloud-node

deploy/ledger-backup/README.md

@@ -0,0 +1,119 @@
+# Deletion-ledger backup (TC-887)
+
+The production node (`docker-compose.dstack-postgres.yaml`, Postgres + S3)
+keeps one thing that exists only on the CVM's disk: the named-keys deletion
+ledger, `/var/lib/tinycloud/ledger/deletions.jsonl` on the `deletion-ledger`
+volume. Phala keeps no copy of CVM disks. Everything else the node writes
+locally is a cache (SQL files rehydrated from `database_artifact`), temporary
+(upload staging), per process (`runtime/control.*`) or re-issuable (TLS
+certificates).
+
+## Why the ledger matters
+
+Every delete under `apps/` is appended to the ledger before the database
+commit (`docs/encryption-named-keys.md`, "Deletion ledger"). After a database
+restore, `tinycloud node deletion-ledger replay` (also run at every node start)
+sets each key it lists back to `deleted` with no seed. Losing the ledger alone
+changes nothing visible: the node starts a new, empty one. But if the database
+is later restored from a backup taken before some deletes, those rows carry
+the sealed seeds again, the deleted keys can be derived again, and data that
+was crypto-shredded can be decrypted. The ledger is the only record that
+cancels that.
+
+## What the sidecar does
+
+`ledger-backup` (`Dockerfile`, `ledger-backup.sh`) runs next to the node:
+
+- mounts `deletion-ledger` **read-only**, as the node's uid 1000, with a
+  read-only root filesystem, no capabilities and a 32 MiB tmpfs `/tmp`. The
+  plaintext copy exists only in that tmpfs and is deleted after encryption;
+- takes no lock (the node holds `deletions.jsonl.lock` while it runs). It
+  copies the file's current length, keeps whole lines only, waits 5 s and
+  accepts the copy only if the live file still starts with exactly those
+  bytes; otherwise it copies again (an append whose write or fsync fails is
+  cut back by the node);
+- encrypts with `age -r <NODE_BACKUP_AGE_RECIPIENT>` (an `age1…` public
+  recipient; it refuses to start with an `AGE-SECRET-KEY-…`) and PUTs
+  `ledger/deletions-<UTC timestamp>.jsonl.age` to the R2 bucket
+  `tinycloud-node-backups` with `If-None-Match: *` (never overwrites). The R2
+  credentials reach curl on stdin, never in argv;
+- checks every 300 s and uploads when the content changed, and at least once
+  every 24 h even if it did not, so a stopped sidecar shows as a gap in the
+  bucket. A failed attempt logs `backup: FAILED` and retries after 60 s.
+
+Every backup is the whole ledger (it is append-only), so the newest object is
+enough for a restore. The objects are small; no lifecycle rule is needed. If
+one is added, it must never delete the newest objects.
+
+Sealed env (one card): `NODE_BACKUP_R2_ENDPOINT`, `NODE_BACKUP_R2_ACCESS_KEY_ID`,
+`NODE_BACKUP_R2_SECRET_ACCESS_KEY`, `NODE_BACKUP_AGE_RECIPIENT`, delivered by
+`docker.yml` from the `production` secrets `PROD_TINYCLOUD_NODE_BACKUP_*`
+(`docs/releasing.md`, "Deletion-ledger backup sidecar").
+
+## What is in a backup
+
+One JSON line per delete operation and per outcome, exactly the node's format:
+`{"op","networkId","keys":[{"keyPath","keyVersion"}],"deletedAt"}`,
+`{"committed": op}` and `{"aborted": op}`. `op` is the CID of the owner's
+signed delete credential; `networkId` names the owner's DID; key paths name
+app ids and key names. No key material, seeds or user data. It is still user
+metadata, hence the encryption.
+
+## Restore drill
+
+Run by an operator, never on the CVM (it needs the identity):
+
+```sh
+umask 077
+export TMPDIR="$(mktemp -d -p /var/tmp)"          # private scratch area
+export AGE_IDENTITY_FILE=/path/to/node-ledger-backup.agekey
+export NODE_BACKUP_R2_ENDPOINT=... NODE_BACKUP_R2_BUCKET=tinycloud-node-backups
+export NODE_BACKUP_R2_ACCESS_KEY_ID=... NODE_BACKUP_R2_SECRET_ACCESS_KEY=...   # a read key
+sh deploy/ledger-backup/ledger-restore-drill.sh            # newest object
+sh deploy/ledger-backup/ledger-restore-drill.sh --expect known.jsonl
+```
+
+It decrypts into a fresh scratch directory, checks that every line is a whole
+record of exactly the node's schema, prints the line count, the record counts
+(intents, committed, aborted, still open) and the last entry, optionally
+compares with a known sample (`--expect` byte for byte, `--expect-prefix`),
+and removes the scratch directory (`RESTORE_KEEP=1` keeps it and prints its
+path). Needs `age`, `jq` and curl 8 or newer.
+
+To put a restored ledger back on a node: stop the node, place the file at
+`deletion_ledger_path` (uid 1000, mode 0600), and start the node; it replays
+the ledger before serving. After a database restore, run
+`tinycloud node deletion-ledger replay` before the node starts.
+
+## Key custody
+
+- The CVM holds only the public recipient (sealed `NODE_BACKUP_AGE_RECIPIENT`).
+  `docker.yml` and the sidecar both refuse an identity in that place.
+- The identity (`AGE-SECRET-KEY-1…`) is generated off the CVM by its holder,
+  with `age-keygen -o node-ledger-backup.agekey` on a trusted machine, and is
+  never sealed, committed, pasted into CI or sent through chat.
+- The holder, where the identity is stored, and its escrow copy (a second
+  person or place, so one loss does not end recovery) are approved by security
+  review (Naomi) and recorded on TC-887 before the first deploy. The recipient
+  sealed on the CVM must be the one derived from that identity
+  (`age-keygen -y node-ledger-backup.agekey`).
+- If every copy of the identity is lost, every backup made with it is
+  unreadable. Generate a new identity, reseal the recipient and redeploy: the
+  next backup is complete again, because each backup is the whole ledger.
+- The R2 key sealed on the CVM is scoped to the `tinycloud-node-backups` bucket
+  only; a leaked key exposes ciphertext only. R2 object tokens that can write
+  can also delete, so a bucket lock rule on `ledger/` (retention in days) is
+  what stops a compromised CVM from deleting earlier backups.
+
+## CI
+
+`.github/workflows/ledger-backup.yml`: `scripts` runs `ledger-backup.test.mjs`
+(an S3 stub; misconfiguration, failed uploads, a torn final line, an append cut
+back during the copy, appends during the copy, the loop's change/unchanged
+schedule, the drill's schema checks). `image` builds the sidecar and, for
+same-repository runs, publishes `ghcr.io/tinycloudlabs/tinycloud-node-ledger-backup:<revision>`
+labelled with the sha256 of its sources. `pinned` requires the compose's
+pinned digest to carry this revision's sources and runs `ci-e2e.sh` with it:
+MinIO for R2, the compose service's flags, a fresh volume first mounted by the
+sidecar, a refused private key, no write to the container filesystem, backups
+while a writer appends, and the restore drill.