From ad6c7c8128b4047d8d6617427ba3cdab06887569 Mon Sep 17 00:00:00 2001
From: "tinycloud-bot[bot]"
 <348376441+tinycloud-bot[bot]@users.noreply.github.com>
Date: Sun, 1 Nov 2026 21:41:05 +0000
Subject: [PATCH 1/3] ops: encrypted backup sidecar for the deletion ledger;
 Details: TC-887

---
 .github/workflows/ledger-backup.yml          | 148 +++++++++
 deploy/ledger-backup/Dockerfile              |  16 +
 deploy/ledger-backup/ci-e2e.sh               | 138 ++++++++
 deploy/ledger-backup/ledger-backup.sh        | 162 ++++++++++
 deploy/ledger-backup/ledger-backup.test.mjs  | 322 +++++++++++++++++++
 deploy/ledger-backup/ledger-restore-drill.sh | 150 +++++++++
 6 files changed, 936 insertions(+)
 create mode 100644 .github/workflows/ledger-backup.yml
 create mode 100644 deploy/ledger-backup/Dockerfile
 create mode 100755 deploy/ledger-backup/ci-e2e.sh
 create mode 100755 deploy/ledger-backup/ledger-backup.sh
 create mode 100644 deploy/ledger-backup/ledger-backup.test.mjs
 create mode 100755 deploy/ledger-backup/ledger-restore-drill.sh

diff --git a/.github/workflows/ledger-backup.yml b/.github/workflows/ledger-backup.yml
new file mode 100644
index 00000000..fbadc76c
--- /dev/null
+++ b/.github/workflows/ledger-backup.yml
@@ -0,0 +1,148 @@
+name: ledger-backup
+
+# TC-887: the deletion-ledger backup sidecar (deploy/ledger-backup/).
+# - scripts: the backup loop and the restore drill against an S3 stub,
+#   including a node appending while the backup copies.
+# - image: builds the sidecar from this revision; same-repository runs publish
+#   it to GHCR as :<revision>, labelled with the sha256 of its sources.
+# - pinned: the image docker-compose.dstack-postgres.yaml pins must carry this
+#   revision's sidecar sources, and passes backup + restore drill end to end in
+#   a container against MinIO.
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+    paths:
+      - 'deploy/ledger-backup/**'
+      - 'docker-compose.dstack-postgres.yaml'
+      - 'scripts/at-rest-harness/harness.sh'
+      - '.github/workflows/ledger-backup.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'deploy/ledger-backup/**'
+      - 'docker-compose.dstack-postgres.yaml'
+      - 'scripts/at-rest-harness/harness.sh'
+      - '.github/workflows/ledger-backup.yml'
+
+permissions:
+  contents: read
+
+env:
+  IMAGE: ghcr.io/tinycloudlabs/tinycloud-node-ledger-backup
+  # The files the image is built from; their sha256 is the image's source label.
+  SOURCES: deploy/ledger-backup/Dockerfile deploy/ledger-backup/ledger-backup.sh
+
+jobs:
+  scripts:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - name: Install age
+        run: sudo apt-get update -qq && sudo apt-get install -y -qq age
+      - name: Backup loop and restore drill against an S3 stub
+        env:
+          LEDGER_BACKUP_TEST_REQUIRE: '1'
+          LEDGER_BACKUP_TEST_TMP: ${{ runner.temp }}
+        run: node --test deploy/ledger-backup/ledger-backup.test.mjs
+
+  image:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - name: Source label
+        id: source
+        run: |
+          set -euo pipefail
+          echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          echo "sha256=$(cat $SOURCES | sha256sum | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT"
+      - uses: docker/setup-buildx-action@v3
+      - name: Log in to GHCR
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build (and publish for same-repository runs)
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: deploy/ledger-backup/Dockerfile
+          platforms: linux/amd64
+          provenance: false
+          push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
+          tags: ${{ env.IMAGE }}:${{ steps.source.outputs.revision }}
+          labels: |
+            org.opencontainers.image.revision=${{ steps.source.outputs.revision }}
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            xyz.tinycloud.ledger-backup.sources-sha256=${{ steps.source.outputs.sha256 }}
+      - name: Report the image
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+          SOURCES_SHA256: ${{ steps.source.outputs.sha256 }}
+        run: |
+          echo "Built ${IMAGE}@${DIGEST} (sources sha256 ${SOURCES_SHA256})" | tee -a "$GITHUB_STEP_SUMMARY"
+
+  pinned:
+    needs: scripts
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - name: Read the pinned image from the production compose
+        id: pin
+        run: |
+          set -euo pipefail
+          images=$(sed -n -E 's/^    image: (ghcr\.io\/tinycloudlabs\/tinycloud-node-ledger-backup@sha256:[0-9a-f]{64})$/\1/p' docker-compose.dstack-postgres.yaml)
+          [ "$(printf '%s\n' "$images" | awk 'NF { n += 1 } END { print n + 0 }')" = 1 ] \
+            || { echo "::error::docker-compose.dstack-postgres.yaml must pin exactly one ${IMAGE}@sha256 image"; exit 1; }
+          echo "image=$images" >> "$GITHUB_OUTPUT"
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: The pinned image is built from this revision's sidecar sources
+        env:
+          PINNED: ${{ steps.pin.outputs.image }}
+        run: |
+          set -euo pipefail
+          docker pull --quiet "$PINNED"
+          label=$(docker image inspect --format '{{ index .Config.Labels "xyz.tinycloud.ledger-backup.sources-sha256" }}' "$PINNED")
+          here=$(cat $SOURCES | sha256sum | cut -d ' ' -f 1)
+          [ "$label" = "$here" ] || { echo "::error::The pinned image was built from other sidecar sources (label ${label}, this revision ${here}). Pin the digest the image job publishes for this revision."; exit 1; }
+          echo "Pinned ${PINNED}: sources sha256 ${here}, built at revision $(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$PINNED")" | tee -a "$GITHUB_STEP_SUMMARY"
+      - name: Install age
+        run: sudo apt-get update -qq && sudo apt-get install -y -qq age
+      - name: Fetch the pinned MinIO (sha256-verified)
+        env:
+          TC887_BIN_DIR: ${{ runner.temp }}/bin
+        run: bash scripts/at-rest-harness/harness.sh bins
+      - name: Backup and restore drill with the pinned image against MinIO
+        env:
+          MINIO_BIN: ${{ runner.temp }}/bin/minio
+          PINNED: ${{ steps.pin.outputs.image }}
+        run: bash deploy/ledger-backup/ci-e2e.sh "$PINNED"
diff --git a/deploy/ledger-backup/Dockerfile b/deploy/ledger-backup/Dockerfile
new file mode 100644
index 00000000..5f2648ca
--- /dev/null
+++ b/deploy/ledger-backup/Dockerfile
@@ -0,0 +1,16 @@
+# Sidecar image for the encrypted deletion-ledger backup loop (TC-887).
+# Alpine keeps only the current package revision per release; if a pin below
+# stops resolving, the build fails loudly. Bump the pins together with the base
+# digest (Alpine 3.24.3 at this digest).
+FROM alpine:3.24@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a
+# /var/lib/tinycloud/ledger gets the node image's owner and mode: if this
+# container is the first to mount a fresh `deletion-ledger` volume, Docker
+# seeds the volume from here and the node can still write it.
+RUN apk add --no-cache age=1.3.2-r1 curl=8.23.0-r0 \
+    && addgroup -g 1000 tinycloud && adduser -u 1000 -G tinycloud -s /sbin/nologin -D -H tinycloud \
+    && mkdir -p /var/lib/tinycloud/ledger && chown 1000:1000 /var/lib/tinycloud/ledger && chmod 700 /var/lib/tinycloud/ledger
+COPY --chmod=0755 deploy/ledger-backup/ledger-backup.sh /usr/local/bin/ledger-backup.sh
+USER 1000:1000
+ENTRYPOINT ["/usr/local/bin/ledger-backup.sh"]
+CMD []
+LABEL org.opencontainers.image.source=https://github.com/TinyCloudLabs/tinycloud-node
diff --git a/deploy/ledger-backup/ci-e2e.sh b/deploy/ledger-backup/ci-e2e.sh
new file mode 100755
index 00000000..a56ab9e4
--- /dev/null
+++ b/deploy/ledger-backup/ci-e2e.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# TC-887 end-to-end check of a ledger-backup image on a disposable CI runner:
+# MinIO stands in for R2, a Docker volume for `deletion-ledger`, and the image
+# runs with the production compose service's flags. Then the restore drill
+# decrypts what it uploaded into a scratch directory and checks it.
+#
+#   MINIO_BIN=<path to minio> ci-e2e.sh <image>
+#
+# Needs docker, age, age-keygen, jq and curl >= 8 on the runner. Every value
+# here is synthetic; no deployment credential is read.
+set -euo pipefail
+umask 077
+
+image=${1:?usage: ci-e2e.sh <image>}
+: "${MINIO_BIN:?set MINIO_BIN to the minio binary}"
+here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+scratch=$(mktemp -d -p "${RUNNER_TEMP:-/var/tmp}")
+run_id="ledger-e2e-$$-$RANDOM"
+volume="$run_id"
+minio_pid=
+cleanup() {
+  docker rm -f "$run_id-writer" "$run_id-backup" >/dev/null 2>&1 || true
+  docker volume rm -f "$volume" >/dev/null 2>&1 || true
+  [ -z "$minio_pid" ] || kill "$minio_pid" 2>/dev/null || true
+  rm -rf "$scratch"
+}
+trap cleanup EXIT
+
+step() { printf '\n== %s\n' "$*"; }
+fail() { echo "ci-e2e: $*" >&2; exit 1; }
+
+step "MinIO (region auto, as R2 signs) with a bucket named like production's"
+export MINIO_ROOT_USER="e2e$(od -An -N6 -tx1 /dev/urandom | tr -d ' \n')"
+export MINIO_ROOT_PASSWORD="$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')"
+export MINIO_SITE_REGION=auto
+port=$(( 20000 + RANDOM % 20000 ))
+mkdir "$scratch/minio"
+"$MINIO_BIN" server "$scratch/minio" --address "127.0.0.1:$port" --console-address "127.0.0.1:0" >"$scratch/minio.log" 2>&1 &
+minio_pid=$!
+endpoint="http://127.0.0.1:$port"
+for _ in $(seq 100); do curl --silent --output /dev/null "$endpoint/minio/health/live" && break; sleep 0.2; done
+curl --silent --fail --output /dev/null "$endpoint/minio/health/live" || fail "MinIO did not start"
+export NODE_BACKUP_R2_ENDPOINT=$endpoint
+export NODE_BACKUP_R2_BUCKET=tinycloud-node-backups
+export NODE_BACKUP_R2_ACCESS_KEY_ID=$MINIO_ROOT_USER
+export NODE_BACKUP_R2_SECRET_ACCESS_KEY=$MINIO_ROOT_PASSWORD
+printf 'user = "%s:%s"\n' "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" \
+  | curl --config - --fail --silent --show-error --aws-sigv4 "aws:amz:auto:s3" -X PUT "$endpoint/$NODE_BACKUP_R2_BUCKET"
+
+step "Operator identity (off-CVM in production) and its public recipient"
+age-keygen -o "$scratch/identity.txt" 2>/dev/null
+export NODE_BACKUP_AGE_RECIPIENT
+NODE_BACKUP_AGE_RECIPIENT=$(age-keygen -y "$scratch/identity.txt")
+export AGE_IDENTITY_FILE=$scratch/identity.txt
+
+as_node() { # as_node <docker run args...> -- <sh script>; uid 1000 shell in the image on the ledger volume
+  docker run --rm --user 1000:1000 --entrypoint sh -v "$volume:/var/lib/tinycloud/ledger" "$@"
+}
+backup_args=(--network host --init --user 1000:1000 --read-only --cap-drop ALL
+  --security-opt no-new-privileges:true --memory 96m --memory-swap 96m --cpus 0.25
+  --tmpfs /tmp:rw,noexec,nosuid,size=32m,uid=1000,gid=1000,mode=0700
+  -v "$volume:/var/lib/tinycloud/ledger:ro"
+  -e NODE_BACKUP_R2_ENDPOINT -e NODE_BACKUP_R2_BUCKET -e NODE_BACKUP_R2_ACCESS_KEY_ID
+  -e NODE_BACKUP_R2_SECRET_ACCESS_KEY -e NODE_BACKUP_AGE_RECIPIENT -e BACKUP_ONCE=1 -e BACKUP_SETTLE_SECONDS=1)
+
+step "A fresh deletion-ledger volume first mounted by the sidecar image stays writable by the node (uid 1000, 0700)"
+docker volume create "$volume" >/dev/null
+mode=$(as_node "$image" -c 'stat -c "%u:%g %a" /var/lib/tinycloud/ledger')
+[ "$mode" = "1000:1000 700" ] || fail "fresh volume is $mode, not 1000:1000 700"
+echo "volume: $mode"
+
+step "A ledger in the node's format, mode 0600, ending in a torn line"
+line() { printf '{"op":"bafyop%d","networkId":"urn:tinycloud:encryption:did:key:z6MkE2eSample:mainnet","keys":[{"keyPath":"apps/e2e/k%d","keyVersion":1}],"deletedAt":"2026-11-01T00:00:00Z"}\n{"committed":"bafyop%d"}\n' "$1" "$1" "$1"; }
+for i in 1 2 3; do line "$i"; done > "$scratch/sample.jsonl"
+{ cat "$scratch/sample.jsonl"; printf '{"op":"bafyop4","netw'; } \
+  | as_node -i "$image" -c 'cat > /var/lib/tinycloud/ledger/deletions.jsonl && chmod 600 /var/lib/tinycloud/ledger/deletions.jsonl'
+
+step "The sidecar refuses a private key as its recipient"
+private_key=$(grep '^AGE-SECRET-KEY-' "$scratch/identity.txt")
+if NODE_BACKUP_AGE_RECIPIENT=$private_key docker run --rm "${backup_args[@]}" "$image" >"$scratch/refused.out" 2>&1; then
+  fail "a private key was accepted as the recipient"
+fi
+grep -q 'PRIVATE key; refusing to start' "$scratch/refused.out" || fail "unexpected refusal: $(cat "$scratch/refused.out")"
+if grep -q 'AGE-SECRET-KEY-' "$scratch/refused.out"; then fail "the refusal echoed the private key"; fi
+unset private_key
+echo "refused: $(cat "$scratch/refused.out")"
+
+step "Backup once, exactly as the compose service runs; nothing is written to the container's own filesystem"
+docker run --name "$run_id-backup" "${backup_args[@]}" "$image"
+changes=$(docker diff "$run_id-backup")
+[ -z "$changes" ] || fail "the container filesystem changed: $changes"
+docker rm "$run_id-backup" >/dev/null
+echo "docker diff: (no changes)"
+
+step "Restore drill: newest object -> scratch dir, schema, counts, last entry, byte-for-byte sample"
+sh "$here/ledger-restore-drill.sh" --expect "$scratch/sample.jsonl"
+
+step "Backups taken while the node appends (each record in two writes) are whole-line prefixes of the final ledger"
+docker run -d --name "$run_id-writer" --user 1000:1000 --entrypoint sh -v "$volume:/var/lib/tinycloud/ledger" "$image" -c '
+  ledger=/var/lib/tinycloud/ledger/deletions.jsonl
+  # Finish the torn line first: the next node start would otherwise drop it.
+  printf "orkId\":\"urn:tinycloud:encryption:did:key:z6MkE2eSample:mainnet\",\"keys\":[{\"keyPath\":\"apps/e2e/k4\",\"keyVersion\":1}],\"deletedAt\":\"2026-11-01T00:00:00Z\"}\n" >> "$ledger"
+  i=100
+  while :; do
+    printf "{\"op\":\"bafyop%d\",\"networkId\":\"urn:tinycloud:encryption:did:key:z6MkE2eSample:mainnet\"," "$i" >> "$ledger"
+    sleep 0.003
+    printf "\"keys\":[{\"keyPath\":\"apps/e2e/k%d\",\"keyVersion\":1}],\"deletedAt\":\"2026-11-01T00:00:00Z\"}\n" "$i" >> "$ledger"
+    i=$((i + 1))
+  done' >/dev/null
+sleep 1
+for round in 1 2 3; do
+  sleep 1.1  # object keys have one-second resolution
+  docker run --rm "${backup_args[@]}" "$image"
+done
+docker rm -f "$run_id-writer" >/dev/null
+as_node "$image" -c 'cat /var/lib/tinycloud/ledger/deletions.jsonl' > "$scratch/final.jsonl"
+listing=$(printf 'user = "%s:%s"\n' "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" \
+  | curl --config - --fail --silent --show-error --aws-sigv4 "aws:amz:auto:s3" -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+    "$endpoint/$NODE_BACKUP_R2_BUCKET/?list-type=2&prefix=ledger/")
+mapfile -t keys < <(grep -o '<Key>[^<]*</Key>' <<<"$listing" | sed 's/<[^>]*>//g' | sort)
+[ "${#keys[@]}" -eq 4 ] || fail "expected 4 backup objects, found ${#keys[@]}"
+previous=0
+for key in "${keys[@]:1}"; do
+  printf 'user = "%s:%s"\n' "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" \
+    | curl --config - --fail --silent --show-error --aws-sigv4 "aws:amz:auto:s3" -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+      -o "$scratch/object.age" "$endpoint/$NODE_BACKUP_R2_BUCKET/$key"
+  age -d -i "$AGE_IDENTITY_FILE" -o "$scratch/restored.jsonl" "$scratch/object.age"
+  size=$(wc -c < "$scratch/restored.jsonl")
+  [ "$(tail -c 1 "$scratch/restored.jsonl" | od -An -c | tr -d ' ')" = '\n' ] || fail "$key does not end with a whole line"
+  head -c "$size" "$scratch/final.jsonl" | cmp -s - "$scratch/restored.jsonl" || fail "$key is not a prefix of the final ledger"
+  [ "$size" -gt "$previous" ] || fail "$key holds no more than the backup before it"
+  echo "$key: $(tr -cd '\n' < "$scratch/restored.jsonl" | wc -c) whole lines, a prefix of the final ledger"
+  previous=$size
+  rm -f "$scratch/restored.jsonl" "$scratch/object.age"
+done
+sh "$here/ledger-restore-drill.sh" --expect-prefix "$scratch/sample.jsonl"
+echo
+echo "ci-e2e: OK"
diff --git a/deploy/ledger-backup/ledger-backup.sh b/deploy/ledger-backup/ledger-backup.sh
new file mode 100755
index 00000000..92244a7c
--- /dev/null
+++ b/deploy/ledger-backup/ledger-backup.sh
@@ -0,0 +1,162 @@
+#!/bin/sh
+# Encrypted backup loop for the node's named-keys deletion ledger (TC-887).
+#
+# Entrypoint of the `ledger-backup` sidecar image (deploy/ledger-backup/Dockerfile).
+# The compose service mounts the `deletion-ledger` volume READ-ONLY, runs as the
+# node's uid 1000 with a read-only root filesystem, and gives this script a
+# RAM-backed tmpfs /tmp. Nothing else in the container is writable, so the
+# plaintext copy never reaches the overlay disk.
+#
+# snapshot -> age -r <public recipient> -> S3 PUT to Cloudflare R2.
+# The CVM holds only the age PUBLIC recipient; the identity (private key) is
+# held by operators off the CVM (deploy/ledger-backup/README.md, "Key custody").
+#
+# Consistency: the node holds an exclusive flock on `<ledger>.lock` for as long
+# as it runs, so the copy is lock-free. The ledger is append-only, but an
+# append whose write or fsync fails is cut back to the length before it. A
+# snapshot therefore (1) copies the file's current length, (2) keeps only whole
+# lines, (3) waits BACKUP_SETTLE_SECONDS and (4) accepts the copy only if the
+# live file still starts with exactly those bytes; otherwise it starts again.
+#
+# Env (sealed): NODE_BACKUP_R2_ENDPOINT, NODE_BACKUP_R2_ACCESS_KEY_ID,
+#   NODE_BACKUP_R2_SECRET_ACCESS_KEY, NODE_BACKUP_AGE_RECIPIENT (age1... only).
+# Env (plain): NODE_BACKUP_R2_BUCKET,
+#   NODE_BACKUP_LEDGER_PATH (default /var/lib/tinycloud/ledger/deletions.jsonl),
+#   BACKUP_INTERVAL_SECONDS (default 300: how often the ledger is checked),
+#   BACKUP_UNCHANGED_UPLOAD_SECONDS (default 86400: upload at least this often
+#     even if the ledger did not change, so a dead sidecar shows as a gap),
+#   BACKUP_RETRY_SECONDS (default 60), BACKUP_SETTLE_SECONDS (default 5),
+#   BACKUP_SNAPSHOT_TRIES (default 10),
+#   BACKUP_ONCE=1 (one snapshot and upload; the exit status reflects it; CI).
+set -eu
+umask 077
+
+die() { echo "backup: $*" >&2; exit 1; }
+
+for name in NODE_BACKUP_R2_ENDPOINT NODE_BACKUP_R2_BUCKET NODE_BACKUP_R2_ACCESS_KEY_ID NODE_BACKUP_R2_SECRET_ACCESS_KEY NODE_BACKUP_AGE_RECIPIENT; do
+  eval "value=\${$name:-}"
+  [ -n "$value" ] || die "$name is required"
+done
+unset value
+case "$NODE_BACKUP_AGE_RECIPIENT" in
+  AGE-SECRET-KEY-*) die "NODE_BACKUP_AGE_RECIPIENT is a PRIVATE key; refusing to start" ;;
+  age1*) ;;
+  *) die "NODE_BACKUP_AGE_RECIPIENT must be an age1... public recipient" ;;
+esac
+# The credentials reach curl through its stdin config, never argv, inside one
+# double-quoted value: refuse anything that would need escaping there.
+for name in NODE_BACKUP_R2_ACCESS_KEY_ID NODE_BACKUP_R2_SECRET_ACCESS_KEY; do
+  eval "value=\$$name"
+  case "$value" in
+    *'"'* | *'\'* | *'
+'* | *"$(printf '\r')"*) die "$name contains a character that is not allowed" ;;
+  esac
+done
+unset value
+
+ledger=${NODE_BACKUP_LEDGER_PATH:-/var/lib/tinycloud/ledger/deletions.jsonl}
+interval=${BACKUP_INTERVAL_SECONDS:-300}
+unchanged_every=${BACKUP_UNCHANGED_UPLOAD_SECONDS:-86400}
+retry=${BACKUP_RETRY_SECONDS:-60}
+settle=${BACKUP_SETTLE_SECONDS:-5}
+tries=${BACKUP_SNAPSHOT_TRIES:-10}
+for value in "$interval" "$unchanged_every" "$retry" "$settle" "$tries"; do
+  case "$value" in '' | *[!0-9]*) die "BACKUP_* intervals and counts must be whole numbers" ;; esac
+done
+[ "$tries" -ge 1 ] || die "BACKUP_SNAPSHOT_TRIES must be at least 1"
+
+# Fail at start, not at the first upload, if age cannot use the recipient.
+printf '' | age -r "$NODE_BACKUP_AGE_RECIPIENT" > /dev/null 2>&1 \
+  || die "NODE_BACKUP_AGE_RECIPIENT is not a valid age recipient"
+
+# mktemp honours TMPDIR and falls back to /tmp, the compose service's tmpfs.
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+# Exit promptly on `docker stop`; EXIT then removes $work.
+trap 'exit 143' TERM INT
+
+# Interruptible sleep: a trap only runs between commands.
+nap() { sleep "$1" & wait $!; }
+
+bytes() { n=$(wc -c < "$1") && echo $((n)); }
+lines() { n=$(tr -cd '\n' < "$1" | wc -c) && echo $((n)); }
+
+# Writes a consistent whole-line copy of the ledger to $work/snap.
+snapshot() {
+  attempt=0
+  while [ "$attempt" -lt "$tries" ]; do
+    attempt=$((attempt + 1))
+    rm -f "$work/raw" "$work/snap"
+    [ -f "$ledger" ] || { echo "backup: the ledger file does not exist" >&2; return 1; }
+    size=$(bytes "$ledger") || return 1
+    head -c "$size" "$ledger" > "$work/raw" || return 1
+    if [ "$(bytes "$work/raw")" -eq "$size" ]; then
+      whole=$(lines "$work/raw")
+      if [ "$whole" -gt 0 ]; then
+        head -n "$whole" "$work/raw" > "$work/snap" || return 1
+      else
+        : > "$work/snap"
+      fi
+      rm -f "$work/raw"
+      length=$(bytes "$work/snap")
+      nap "$settle"
+      # The live file must still begin with exactly the bytes copied.
+      if head -c "$length" "$ledger" | cmp -s - "$work/snap"; then
+        return 0
+      fi
+    fi
+    echo "backup: the ledger changed under the copy (attempt $attempt of $tries); copying again" >&2
+  done
+  rm -f "$work/raw" "$work/snap"
+  return 1
+}
+
+put() { # put <file> <key>
+  printf 'user = "%s:%s"\n' "$NODE_BACKUP_R2_ACCESS_KEY_ID" "$NODE_BACKUP_R2_SECRET_ACCESS_KEY" \
+    | curl --config - --fail --silent --show-error --max-time 120 \
+      --aws-sigv4 "aws:amz:auto:s3" \
+      -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+      -H "If-None-Match: *" \
+      -T "$1" \
+      "${NODE_BACKUP_R2_ENDPOINT%/}/${NODE_BACKUP_R2_BUCKET}/$2"
+}
+
+last_hash=
+last_upload=0
+
+# `|| return 1`: set -e is ignored inside `if backup_once`, so fail each step explicitly.
+backup_once() {
+  snapshot || return 1
+  hash=$(sha256sum "$work/snap" | cut -d ' ' -f 1) || return 1
+  now=$(date +%s)
+  if [ "$hash" = "$last_hash" ] && [ $((now - last_upload)) -lt "$unchanged_every" ]; then
+    rm -f "$work/snap"
+    return 0
+  fi
+  count=$(lines "$work/snap")
+  stamp=$(date -u +%Y%m%dT%H%M%SZ)
+  key="ledger/deletions-${stamp}.jsonl.age"
+  age -r "$NODE_BACKUP_AGE_RECIPIENT" -o "$work/snap.age" "$work/snap" || return 1
+  rm -f "$work/snap"
+  put "$work/snap.age" "$key" || return 1
+  size=$(bytes "$work/snap.age")
+  rm -f "$work/snap.age"
+  last_hash=$hash
+  last_upload=$now
+  echo "backup: uploaded ${key} (${count} lines, ${size} bytes encrypted)"
+}
+
+if [ "${BACKUP_ONCE:-}" = 1 ]; then
+  backup_once || die "FAILED"
+  exit 0
+fi
+
+while true; do
+  if backup_once; then
+    nap "$interval"
+  else
+    echo "backup: FAILED; retrying in ${retry}s" >&2
+    rm -f "$work/raw" "$work/snap" "$work/snap.age"
+    nap "$retry"
+  fi
+done
diff --git a/deploy/ledger-backup/ledger-backup.test.mjs b/deploy/ledger-backup/ledger-backup.test.mjs
new file mode 100644
index 00000000..fe87544c
--- /dev/null
+++ b/deploy/ledger-backup/ledger-backup.test.mjs
@@ -0,0 +1,322 @@
+// TC-887: the deletion-ledger backup loop and its restore drill, against an
+// in-process S3 stub. CI runs the same scripts again inside the pinned image
+// against MinIO (.github/workflows/ledger-backup.yml).
+import assert from "node:assert/strict";
+import { spawn, spawnSync } from "node:child_process";
+import { appendFileSync, existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, truncateSync, writeFileSync } from "node:fs";
+import { createServer } from "node:http";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { setTimeout as sleep } from "node:timers/promises";
+
+const dir = new URL(".", import.meta.url).pathname;
+const backupScript = join(dir, "ledger-backup.sh");
+const drillScript = join(dir, "ledger-restore-drill.sh");
+const BUCKET = "tinycloud-node-backups";
+const KEY_ID = "test-key-id";
+const SECRET = "test-secret-value";
+
+const tools = ["age", "age-keygen", "curl", "jq"];
+const missing = tools.filter((tool) => spawnSync("sh", ["-c", `command -v ${tool}`]).status !== 0);
+// curl 7.x does not sign x-amz-content-sha256 with --aws-sigv4; the image ships curl 8.
+const curlMajor = Number(/^curl (\d+)\./.exec(spawnSync("curl", ["--version"], { encoding: "utf8" }).stdout ?? "")?.[1]);
+if (!missing.includes("curl") && !(curlMajor >= 8)) missing.push("curl >= 8");
+if (missing.length && process.env.LEDGER_BACKUP_TEST_REQUIRE === "1") throw new Error(`missing tools: ${missing.join(", ")}`);
+const skip = missing.length ? `needs ${missing.join(", ")}` : false;
+
+// The node's own line shapes (tinycloud-core deletion_ledger.rs).
+const MARKER = "PlaintextMarkerNeverInTheClear";
+const intent = (i) =>
+  JSON.stringify({
+    op: `bafyop${i}`,
+    networkId: `urn:tinycloud:encryption:did:key:z6Mk${MARKER}${i}:mainnet`,
+    keys: [{ keyPath: `apps/app${i}/key`, keyVersion: 1 }],
+    deletedAt: "2026-11-01T00:00:00Z",
+  }) + "\n";
+const committed = (i) => JSON.stringify({ committed: `bafyop${i}` }) + "\n";
+const operation = (i) => intent(i) + committed(i);
+
+function mockR2() {
+  const objects = new Map();
+  const seen = [];
+  let failUploads = false;
+  const server = createServer((req, res) => {
+    const chunks = [];
+    req.on("data", (chunk) => chunks.push(chunk));
+    req.on("end", () => {
+      const url = new URL(req.url, "http://mock");
+      seen.push({ method: req.method, path: url.pathname, authorization: req.headers.authorization, sha: req.headers["x-amz-content-sha256"], ifNoneMatch: req.headers["if-none-match"] });
+      if (req.method === "PUT") {
+        if (failUploads) return res.writeHead(500).end();
+        if (req.headers["if-none-match"] === "*" && objects.has(url.pathname)) return res.writeHead(412).end();
+        objects.set(url.pathname, Buffer.concat(chunks));
+        return res.writeHead(200).end();
+      }
+      if (req.method === "GET" && url.searchParams.get("list-type") === "2") {
+        const prefix = `/${BUCKET}/${url.searchParams.get("prefix")}`;
+        const keys = [...objects.keys()].filter((key) => key.startsWith(prefix)).map((key) => key.slice(BUCKET.length + 2));
+        return res.writeHead(200).end(`<ListBucketResult><IsTruncated>false</IsTruncated>${keys.map((key) => `<Contents><Key>${key}</Key></Contents>`).join("")}</ListBucketResult>`);
+      }
+      if (req.method === "GET" && objects.has(url.pathname)) return res.writeHead(200).end(objects.get(url.pathname));
+      res.writeHead(404).end();
+    });
+  });
+  return {
+    objects,
+    seen,
+    failUploads: (value) => (failUploads = value),
+    listen: () => new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(`http://127.0.0.1:${server.address().port}`))),
+    close: () => server.close(),
+  };
+}
+
+function start(command, args, env) {
+  const child = spawn(command, args, { env, stdio: ["ignore", "pipe", "pipe"] });
+  const result = { child, out: "", err: "", status: undefined };
+  child.stdout.on("data", (chunk) => (result.out += chunk));
+  child.stderr.on("data", (chunk) => (result.err += chunk));
+  result.done = new Promise((resolve) => child.on("close", (status) => resolve((result.status = status))));
+  return result;
+}
+
+async function run(command, args, env) {
+  const result = start(command, args, env);
+  await result.done;
+  return result;
+}
+
+async function waitFor(condition, ms, what) {
+  const deadline = Date.now() + ms;
+  while (!condition()) {
+    if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`);
+    await sleep(25);
+  }
+}
+
+// One scratch area per test: the ledger, the age identity and the scripts' TMPDIR.
+async function fixture(t) {
+  const root = mkdtempSync(join(process.env.LEDGER_BACKUP_TEST_TMP ?? tmpdir(), "ledger-backup-test-"));
+  const scratch = join(root, "tmp");
+  spawnSync("mkdir", ["-m", "700", scratch]);
+  const r2 = mockR2();
+  const endpoint = await r2.listen();
+  t.after(() => {
+    r2.close();
+    rmSync(root, { recursive: true, force: true });
+  });
+  const keygen = spawnSync("age-keygen", ["-o", join(root, "identity.txt")], { encoding: "utf8" });
+  assert.equal(keygen.status, 0, keygen.stderr);
+  const recipient = /[Pp]ublic key: (age1\S+)/.exec(keygen.stderr + keygen.stdout)?.[1];
+  assert.ok(recipient, "age-keygen must print a public recipient");
+  const ledger = join(root, "deletions.jsonl");
+  const base = { PATH: process.env.PATH, TMPDIR: scratch, LC_ALL: "C" };
+  const backupEnv = {
+    ...base,
+    NODE_BACKUP_LEDGER_PATH: ledger,
+    NODE_BACKUP_R2_ENDPOINT: endpoint,
+    NODE_BACKUP_R2_BUCKET: BUCKET,
+    NODE_BACKUP_R2_ACCESS_KEY_ID: KEY_ID,
+    NODE_BACKUP_R2_SECRET_ACCESS_KEY: SECRET,
+    NODE_BACKUP_AGE_RECIPIENT: recipient,
+    BACKUP_SETTLE_SECONDS: "0",
+    BACKUP_ONCE: "1",
+  };
+  const drillEnv = {
+    ...base,
+    AGE_IDENTITY_FILE: join(root, "identity.txt"),
+    NODE_BACKUP_R2_ENDPOINT: endpoint,
+    NODE_BACKUP_R2_BUCKET: BUCKET,
+    NODE_BACKUP_R2_ACCESS_KEY_ID: KEY_ID,
+    NODE_BACKUP_R2_SECRET_ACCESS_KEY: SECRET,
+  };
+  const decrypt = (body) => {
+    const encrypted = join(root, "object.age");
+    writeFileSync(encrypted, body);
+    const result = spawnSync("age", ["-d", "-i", join(root, "identity.txt"), encrypted]);
+    assert.equal(result.status, 0, String(result.stderr));
+    return result.stdout.toString();
+  };
+  const file = (name, content) => {
+    const path = join(root, name);
+    writeFileSync(path, content);
+    return path;
+  };
+  return { root, scratch, r2, ledger, recipient, backupEnv, drillEnv, decrypt, file, identityText: () => readFileSync(join(root, "identity.txt"), "utf8") };
+}
+
+test("misconfiguration is refused before the ledger is read or anything is uploaded", { skip }, async (t) => {
+  const f = await fixture(t);
+  writeFileSync(f.ledger, operation(1));
+  const privateKey = /AGE-SECRET-KEY-\S+/.exec(f.identityText())[0];
+  const cases = [
+    [{ NODE_BACKUP_AGE_RECIPIENT: privateKey }, /PRIVATE key; refusing to start/],
+    [{ NODE_BACKUP_AGE_RECIPIENT: "ssh-ed25519 AAAA" }, /must be an age1\.\.\. public recipient/],
+    [{ NODE_BACKUP_AGE_RECIPIENT: "age1notarealrecipient" }, /not a valid age recipient/],
+    [{ NODE_BACKUP_R2_BUCKET: "" }, /NODE_BACKUP_R2_BUCKET is required/],
+    [{ NODE_BACKUP_R2_SECRET_ACCESS_KEY: 'a"b' }, /NODE_BACKUP_R2_SECRET_ACCESS_KEY contains a character/],
+    [{ BACKUP_INTERVAL_SECONDS: "5m" }, /whole numbers/],
+  ];
+  for (const [change, message] of cases) {
+    const result = await run("sh", [backupScript], { ...f.backupEnv, ...change });
+    assert.notEqual(result.status, 0, JSON.stringify(change));
+    assert.match(result.err, message);
+    assert.ok(!result.err.includes(privateKey), "a refused private key is never echoed");
+  }
+  assert.equal(f.r2.seen.length, 0);
+});
+
+test("a failed upload or a missing ledger fails the run", { skip }, async (t) => {
+  const f = await fixture(t);
+  const absent = await run("sh", [backupScript], f.backupEnv);
+  assert.notEqual(absent.status, 0);
+  assert.match(absent.err, /does not exist/);
+  writeFileSync(f.ledger, operation(1));
+  f.r2.failUploads(true);
+  const failed = await run("sh", [backupScript], f.backupEnv);
+  assert.notEqual(failed.status, 0);
+  assert.match(failed.err, /FAILED/);
+  assert.equal(f.r2.objects.size, 0);
+  assert.deepEqual(readdirSync(f.scratch), [], "the plaintext copy is removed on failure");
+});
+
+test("one backup holds the whole lines, encrypted, and the drill restores them exactly", { skip }, async (t) => {
+  const f = await fixture(t);
+  const whole = operation(1) + operation(2) + intent(3);
+  // A crash can leave an unterminated final line; the node drops it at its next start.
+  writeFileSync(f.ledger, whole + intent(4).slice(0, 40));
+  const backup = await run("sh", [backupScript], f.backupEnv);
+  assert.equal(backup.status, 0, backup.err);
+  assert.match(backup.out, /uploaded ledger\/deletions-\d{8}T\d{6}Z\.jsonl\.age \(5 lines, \d+ bytes encrypted\)/);
+  assert.deepEqual(readdirSync(f.scratch), [], "no plaintext is left behind");
+
+  assert.equal(f.r2.objects.size, 1);
+  const [[path, body]] = [...f.r2.objects];
+  assert.match(path, new RegExp(`^/${BUCKET}/ledger/deletions-\\d{8}T\\d{6}Z\\.jsonl\\.age$`));
+  assert.ok(body.subarray(0, 21).toString() === "age-encryption.org/v1", "the object is an age file");
+  assert.ok(!body.includes(MARKER), "the object carries no plaintext");
+  assert.equal(f.decrypt(body), whole);
+  const put = f.r2.seen.find((entry) => entry.method === "PUT");
+  assert.match(put.authorization, new RegExp(`^AWS4-HMAC-SHA256 Credential=${KEY_ID}/\\d{8}/auto/s3/aws4_request`));
+  assert.equal(put.sha, "UNSIGNED-PAYLOAD");
+  assert.equal(put.ifNoneMatch, "*", "a backup object is never overwritten");
+
+  const expected = f.file("expected.jsonl", whole);
+  const drill = await run("sh", [drillScript, "--expect", expected], f.drillEnv);
+  assert.equal(drill.status, 0, drill.err);
+  assert.match(drill.out, /latest backup is ledger\/deletions-/);
+  assert.match(drill.out, /decrypted \d+ bytes, 5 lines/);
+  assert.match(drill.out, /records intents=3 committed=2 aborted=0 open=1 keys=3/);
+  assert.match(drill.out, /last entry intent op=bafyop3 deletedAt=2026-11-01T00:00:00Z keys=1/);
+  assert.match(drill.out, /matches the expected ledger byte for byte/);
+  assert.match(drill.out, /restore-drill: OK/);
+
+  const different = f.file("different.jsonl", whole + committed(3));
+  const mismatch = await run("sh", [drillScript, "--expect", different], f.drillEnv);
+  assert.notEqual(mismatch.status, 0);
+  assert.match(mismatch.err, /differs from/);
+  const other = await run("sh", ["-c", `age-keygen -o "$1" 2>/dev/null`, "sh", join(f.root, "other.txt")], f.drillEnv);
+  assert.equal(other.status, 0);
+  const wrong = await run("sh", [drillScript], { ...f.drillEnv, AGE_IDENTITY_FILE: join(f.root, "other.txt") });
+  assert.notEqual(wrong.status, 0);
+  assert.match(wrong.err, /decryption failed/);
+  assert.deepEqual(readdirSync(f.scratch), [], "the drill removes its scratch directory");
+});
+
+test("the drill refuses a backup that is not exactly the node's ledger format", { skip }, async (t) => {
+  const f = await fixture(t);
+  const bad = [
+    ["extra field", intent(1).replace('"op"', '"extra":1,"op"'), /INVALID line 1/],
+    ["not JSON", operation(1) + "not a ledger line\n", /INVALID line 3/],
+    ["fractional keyVersion", intent(1).replace('"keyVersion":1', '"keyVersion":1.5'), /INVALID line 1/],
+    ["unterminated", operation(1) + intent(2).trimEnd(), /last line is not terminated/],
+  ];
+  for (const [name, content, message] of bad) {
+    const plain = f.file("bad.jsonl", content);
+    const encrypted = join(f.root, "bad.jsonl.age");
+    assert.equal(spawnSync("age", ["-r", f.recipient, "-o", encrypted, plain]).status, 0);
+    const drill = await run("sh", [drillScript, "--file", encrypted], f.drillEnv);
+    assert.notEqual(drill.status, 0, name);
+    assert.match(drill.err, message, name);
+  }
+});
+
+test("an append the node cuts back during the copy is never backed up", { skip }, async (t) => {
+  const f = await fixture(t);
+  const kept = operation(1) + operation(2);
+  writeFileSync(f.ledger, kept + intent(3));
+  // The node writes intent 3, its fsync fails, and it truncates the line away:
+  // here, as soon as the script has its first copy.
+  const backup = start("sh", [backupScript], { ...f.backupEnv, BACKUP_SETTLE_SECONDS: "2" });
+  const snapshotTaken = () => readdirSync(f.scratch).some((work) => existsSync(join(f.scratch, work, "snap")));
+  await waitFor(snapshotTaken, 5000, "the first copy");
+  truncateSync(f.ledger, Buffer.byteLength(kept));
+  await backup.done;
+  assert.equal(backup.status, 0, backup.err);
+  assert.match(backup.err, /the ledger changed under the copy \(attempt 1 of 10\)/);
+  assert.equal(f.r2.objects.size, 1);
+  assert.equal(f.decrypt([...f.r2.objects.values()][0]), kept);
+});
+
+test("a backup taken while the node appends is a whole-line prefix of the final ledger", { skip }, async (t) => {
+  const f = await fixture(t);
+  writeFileSync(f.ledger, operation(0));
+  let writing = true;
+  let i = 1;
+  // Each record arrives in two writes, so the copy often sees a partial line.
+  const writer = (async () => {
+    while (writing) {
+      const line = operation(i++);
+      const cut = Math.floor(line.length / 2);
+      appendFileSync(f.ledger, line.slice(0, cut));
+      await sleep(3);
+      appendFileSync(f.ledger, line.slice(cut));
+      await sleep(2);
+    }
+  })();
+  for (let round = 0; round < 5; round++) {
+    const backup = await run("sh", [backupScript], { ...f.backupEnv, BACKUP_SETTLE_SECONDS: "1" });
+    assert.equal(backup.status, 0, backup.err);
+  }
+  writing = false;
+  await writer;
+  const final = readFileSync(f.ledger, "utf8");
+  assert.equal(f.r2.objects.size, 5);
+  let previous = "";
+  for (const key of [...f.r2.objects.keys()].sort()) {
+    const restored = f.decrypt(f.r2.objects.get(key));
+    assert.ok(restored.endsWith("\n"), "whole lines only");
+    assert.ok(final.startsWith(restored), "a prefix of what the node wrote");
+    assert.ok(restored.length > previous.length, "later backups hold more");
+    for (const line of restored.trimEnd().split("\n")) JSON.parse(line);
+    previous = restored;
+  }
+  const drill = await run("sh", [drillScript, "--expect-prefix", f.file("first.jsonl", operation(0))], f.drillEnv);
+  assert.equal(drill.status, 0, drill.err);
+  assert.match(drill.out, /starts with the expected \d+ bytes/);
+});
+
+test("the loop uploads on change, skips unchanged checks, re-uploads an unchanged ledger on schedule and stops on TERM", { skip }, async (t) => {
+  const f = await fixture(t);
+  writeFileSync(f.ledger, operation(1));
+  const env = { ...f.backupEnv, BACKUP_ONCE: "", BACKUP_INTERVAL_SECONDS: "1", BACKUP_UNCHANGED_UPLOAD_SECONDS: "5", BACKUP_RETRY_SECONDS: "1" };
+  const loop = start("sh", [backupScript], env);
+  t.after(() => loop.child.kill("SIGKILL"));
+  const uploads = () => f.r2.objects.size;
+  await waitFor(() => uploads() === 1, 5000, "the first upload");
+  const first = Date.now();
+  await sleep(2500);
+  assert.equal(uploads(), 1, "unchanged checks upload nothing");
+  appendFileSync(f.ledger, operation(2));
+  await waitFor(() => uploads() === 2, 5000, "the upload of the change");
+  await waitFor(() => uploads() === 3, 9000, "the scheduled upload of the unchanged ledger");
+  assert.ok(Date.now() - first >= 5000, "not before BACKUP_UNCHANGED_UPLOAD_SECONDS");
+  const keys = [...f.r2.objects.keys()].sort();
+  assert.equal(f.decrypt(f.r2.objects.get(keys[2])), f.decrypt(f.r2.objects.get(keys[1])));
+  const stopped = Date.now();
+  loop.child.kill("SIGTERM");
+  await loop.done;
+  assert.ok(Date.now() - stopped < 3000, "TERM stops the loop promptly");
+  assert.equal(loop.status, 143);
+  assert.deepEqual(readdirSync(f.scratch), [], "the work directory is removed on exit");
+});
diff --git a/deploy/ledger-backup/ledger-restore-drill.sh b/deploy/ledger-backup/ledger-restore-drill.sh
new file mode 100755
index 00000000..db014a20
--- /dev/null
+++ b/deploy/ledger-backup/ledger-restore-drill.sh
@@ -0,0 +1,150 @@
+#!/bin/sh
+# Restore drill for the node's deletion-ledger backups (TC-887).
+# Run by an operator, never on the CVM: it needs the age PRIVATE key.
+#
+#   ledger-restore-drill.sh [--file <backup.jsonl.age>] [--expect <ledger>] [--expect-prefix <ledger>]
+#
+# 1. Without --file, find the newest ledger backup in the bucket and download it.
+# 2. Decrypt it with the identity in AGE_IDENTITY_FILE into a fresh scratch
+#    directory (mktemp -d; TMPDIR decides where; use a private one).
+# 3. Check it: every line is a whole ledger record of exactly the node's schema
+#    (`{op, networkId, keys:[{keyPath, keyVersion}], deletedAt}`, `{committed}`
+#    or `{aborted}`), print the line count, the record counts and the last entry.
+# 4. --expect: the restored ledger must equal that file byte for byte.
+#    --expect-prefix: the restored ledger must start with exactly that file.
+# The scratch directory is removed on exit unless RESTORE_KEEP=1, which prints
+# its path (the decrypted ledger is `deletions.jsonl` in it).
+#
+# Env:
+#   AGE_IDENTITY_FILE                       age identity (private key) file. Required.
+#   NODE_BACKUP_R2_ENDPOINT / _BUCKET / _ACCESS_KEY_ID / _SECRET_ACCESS_KEY
+#                                           Read credentials. Not needed with --file.
+# Needs: age, jq, and curl >= 8 without --file.
+set -eu
+umask 077
+
+die() { echo "restore-drill: $*" >&2; exit 1; }
+
+file=
+expect=
+expect_prefix=
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --file) [ $# -ge 2 ] || die "--file needs a path"; file=$2; shift 2 ;;
+    --expect) [ $# -ge 2 ] || die "--expect needs a path"; expect=$2; shift 2 ;;
+    --expect-prefix) [ $# -ge 2 ] || die "--expect-prefix needs a path"; expect_prefix=$2; shift 2 ;;
+    *) die "unknown argument: $1" ;;
+  esac
+done
+
+[ -n "${AGE_IDENTITY_FILE:-}" ] && [ -r "$AGE_IDENTITY_FILE" ] || die "AGE_IDENTITY_FILE must point to a readable age identity file"
+for tool in age jq; do command -v "$tool" >/dev/null 2>&1 || die "$tool is required"; done
+[ -z "$expect" ] || [ -r "$expect" ] || die "cannot read $expect"
+[ -z "$expect_prefix" ] || [ -r "$expect_prefix" ] || die "cannot read $expect_prefix"
+
+work=$(mktemp -d)
+cleanup() {
+  if [ "${RESTORE_KEEP:-}" = 1 ]; then
+    echo "restore-drill: kept scratch directory $work"
+  else
+    rm -rf "$work"
+  fi
+}
+trap cleanup EXIT
+trap 'exit 143' TERM INT
+
+if [ -z "$file" ]; then
+  for name in NODE_BACKUP_R2_ENDPOINT NODE_BACKUP_R2_BUCKET NODE_BACKUP_R2_ACCESS_KEY_ID NODE_BACKUP_R2_SECRET_ACCESS_KEY; do
+    eval "value=\${$name:-}"
+    [ -n "$value" ] || die "$name is required (or pass --file)"
+  done
+  command -v curl >/dev/null 2>&1 || die "curl is required"
+  # curl 7.x does not sign x-amz-content-sha256 with --aws-sigv4, so R2 rejects it.
+  curl_major=$(curl --version | sed -n '1s/^curl \([0-9][0-9]*\)\..*/\1/p')
+  case "$curl_major" in '' | *[!0-9]*) die "cannot parse the curl version" ;; esac
+  [ "$curl_major" -ge 8 ] || die "curl $curl_major is too old; version 8 or newer is required"
+  s3() { # s3 <path-and-query> [curl args...]; credentials on stdin, never argv
+    path=$1; shift
+    printf 'user = "%s:%s"\n' "$NODE_BACKUP_R2_ACCESS_KEY_ID" "$NODE_BACKUP_R2_SECRET_ACCESS_KEY" \
+      | curl --config - --fail --silent --show-error --max-time 120 --aws-sigv4 "aws:amz:auto:s3" \
+        -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" "$@" \
+        "${NODE_BACKUP_R2_ENDPOINT%/}/${NODE_BACKUP_R2_BUCKET}${path}"
+  }
+  # Keys embed a UTC timestamp, so lexical order is chronological. Page through
+  # ListObjectsV2 (1000 keys a page) and keep the greatest key.
+  latest=
+  token=
+  while :; do
+    if [ -n "$token" ]; then
+      s3 "/?list-type=2&prefix=ledger/" -G --data-urlencode "continuation-token=$token" > "$work/list.xml"
+    else
+      s3 "/?list-type=2&prefix=ledger/" > "$work/list.xml"
+    fi
+    page=$(grep -o '<Key>[^<]*</Key>' "$work/list.xml" | sed 's/<[^>]*>//g' | grep '^ledger/deletions-[0-9TZ]*\.jsonl\.age$' | sort | tail -n 1 || true)
+    if [ -n "$page" ] && { [ -z "$latest" ] || [ "$page" \> "$latest" ]; }; then latest=$page; fi
+    grep -q '<IsTruncated>true</IsTruncated>' "$work/list.xml" || break
+    token=$(sed -n 's/.*<NextContinuationToken>\([^<]*\)<\/NextContinuationToken>.*/\1/p' "$work/list.xml")
+    [ -n "$token" ] || die "truncated listing without a continuation token"
+  done
+  rm -f "$work/list.xml"
+  [ -n "$latest" ] || die "no ledger backups found in the bucket"
+  echo "restore-drill: latest backup is ${latest}"
+  s3 "/${latest}" -o "$work/backup.age"
+  file=$work/backup.age
+fi
+[ -r "$file" ] || die "cannot read $file"
+
+ledger=$work/deletions.jsonl
+age -d -i "$AGE_IDENTITY_FILE" -o "$ledger" "$file" || die "decryption failed (wrong identity?)"
+size=$(wc -c < "$ledger")
+size=$((size))
+count=$(tr -cd '\n' < "$ledger" | wc -c)
+count=$((count))
+echo "restore-drill: decrypted ${size} bytes, ${count} lines"
+
+if [ "$size" -gt 0 ] && [ "$(tail -c 1 "$ledger" | od -An -c | tr -d ' ')" != '\n' ]; then
+  die "the last line is not terminated: not a whole-line snapshot"
+fi
+
+# Exactly the node's serde shapes (deny_unknown_fields): an intent, or a
+# committed / aborted marker naming an operation.
+summary=$(jq -R -n -r '
+  def str: type == "string";
+  def intent: type == "object" and (keys == ["deletedAt", "keys", "networkId", "op"])
+    and (.op | str) and (.networkId | str) and (.deletedAt | str)
+    and (.keys | type == "array" and all(.[]; type == "object" and keys == ["keyPath", "keyVersion"]
+      and (.keyPath | str) and (.keyVersion | type == "number" and . == floor)));
+  def marker($k): type == "object" and keys == [$k] and (.[$k] | str);
+  [inputs | (try fromjson catch {invalid: true})] as $rows
+  | ($rows | map(intent or marker("committed") or marker("aborted"))) as $valid
+  | if ($valid | all | not) then
+      "INVALID line \(1 + ($valid | index(false)))"
+    else
+      ($rows | map(select(intent))) as $intents
+      | ($rows | map(select(marker("committed")) | .committed)) as $committed
+      | ($rows | map(select(marker("aborted")) | .aborted)) as $aborted
+      | ($intents | map(.op) - $committed - $aborted) as $open
+      | "OK intents=\($intents | length) committed=\($committed | length) aborted=\($aborted | length) open=\($open | length) keys=\($intents | map(.keys | length) | add // 0)"
+      + "\nLAST " + (if ($rows | length) == 0 then "(empty ledger)"
+          else ($rows[-1] | if intent then "intent op=\(.op) deletedAt=\(.deletedAt) keys=\(.keys | length)"
+            elif has("committed") then "committed op=\(.committed)" else "aborted op=\(.aborted)" end) end)
+    end' < "$ledger") || die "jq could not read the ledger"
+case "$summary" in
+  OK*) ;;
+  *) die "schema check failed: ${summary}" ;;
+esac
+echo "restore-drill: records $(printf '%s\n' "$summary" | sed -n 's/^OK //p')"
+echo "restore-drill: last entry $(printf '%s\n' "$summary" | sed -n 's/^LAST //p')"
+
+if [ -n "$expect" ]; then
+  cmp -s "$expect" "$ledger" || die "the restored ledger differs from $expect"
+  echo "restore-drill: matches the expected ledger byte for byte"
+fi
+if [ -n "$expect_prefix" ]; then
+  plen=$(wc -c < "$expect_prefix")
+  plen=$((plen))
+  [ "$size" -ge "$plen" ] && head -c "$plen" "$ledger" | cmp -s - "$expect_prefix" \
+    || die "the restored ledger does not start with $expect_prefix"
+  echo "restore-drill: starts with the expected ${plen} bytes"
+fi
+echo "restore-drill: OK"

From 40165c655906cd71ef220bad50347174d3e71068 Mon Sep 17 00:00:00 2001
From: "tinycloud-bot[bot]"
 <348376441+tinycloud-bot[bot]@users.noreply.github.com>
Date: Sun, 1 Nov 2026 21:46:35 +0000
Subject: [PATCH 2/3] ops: pin the ledger-backup sidecar in the production
 compose and seal its env; Details: TC-887

---
 .github/workflows/docker.yml              |  32 +++++-
 CHANGELOG.md                              |   3 +
 deploy/ledger-backup/README.md            | 119 ++++++++++++++++++++++
 docker-compose.dstack-postgres.yaml       |  40 ++++++++
 docs/encryption-named-keys.md             |   4 +-
 docs/releasing.md                         |  36 ++++++-
 scripts/ci/fixtures/tc959-n0-compose.yaml |  40 ++++++++
 scripts/ci/test-deploy-recovery.py        |  23 ++++-
 scripts/tc959-docker.mjs                  |   1 +
 9 files changed, 290 insertions(+), 8 deletions(-)
 create mode 100644 deploy/ledger-backup/README.md

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 367b36a6..7784f3f9 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -721,9 +721,10 @@ jobs:
           # TC-959: main's N0 compose (#349 c33c2302…) with the 95m ingress body
           # cap (raw d95bfb70…), the TC-1274 + TC-1317 + TC-1482 + c_index + TC-1840 + TC-1843 Vector config and the
           # optional TC-1482 NODE_STATS_READ_TOKEN env line, the TC-950 pool of 10 and the TC-1777 named-keys gate
-          # env lines + deletion-ledger volume (raw 298ea356…; 192dfb7e… before TC-1777),
+          # env lines + deletion-ledger volume, and the TC-887 ledger-backup sidecar pinned by digest
+          # (raw 81412971…; 298ea356… before TC-887; 192dfb7e… before TC-1777),
           # its single node digest normalized to 64 zeros.
-          n0_template_hash = 'c9bcbea9dac90fc2b74d8439146596ba802796a211d1b6d65aeda2286d7ed028'
+          n0_template_hash = '0d57efc103654fe3ec361680bdd0f264d4a9db111b1a48e970f28980f4805bbc'
           node_pin = re.compile(rb'(?m)^    image: ghcr\.io/tinycloudlabs/tinycloud-node@sha256:([0-9a-f]{64})$')
           def n0_template(raw):
               pins = list(node_pin.finditer(raw))
@@ -1000,6 +1001,11 @@ jobs:
           PHALA_ENV_BETTERSTACK_SOURCE_TOKEN: ${{ secrets.PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN }}
           PHALA_ENV_SQL_IDENTITY_FENCE: ${{ inputs.sql_identity_fence }}
           PHALA_ENV_NAMED_KEYS_V1: ${{ inputs.named_keys_v1 }}
+          # TC-887: the deletion-ledger backup sidecar (deploy/ledger-backup/README.md).
+          PHALA_ENV_NODE_BACKUP_R2_ENDPOINT: ${{ secrets.PROD_TINYCLOUD_NODE_BACKUP_R2_ENDPOINT }}
+          PHALA_ENV_NODE_BACKUP_R2_ACCESS_KEY_ID: ${{ secrets.PROD_TINYCLOUD_NODE_BACKUP_R2_ACCESS_KEY_ID }}
+          PHALA_ENV_NODE_BACKUP_R2_SECRET_ACCESS_KEY: ${{ secrets.PROD_TINYCLOUD_NODE_BACKUP_R2_SECRET_ACCESS_KEY }}
+          PHALA_ENV_NODE_BACKUP_AGE_RECIPIENT: ${{ secrets.PROD_TINYCLOUD_NODE_BACKUP_AGE_RECIPIENT }}
         # Update the existing prod CVM (name: tinycloud-node) in the
         # Tiny Cloud workspace. The PHALA_CLOUD_API_KEY secret is
         # workspace-scoped and must be issued from that workspace.
@@ -1026,6 +1032,10 @@ jobs:
             PHALA_ENV_BETTERSTACK_SOURCE_TOKEN
             PHALA_ENV_SQL_IDENTITY_FENCE
             PHALA_ENV_NAMED_KEYS_V1
+            PHALA_ENV_NODE_BACKUP_R2_ENDPOINT
+            PHALA_ENV_NODE_BACKUP_R2_ACCESS_KEY_ID
+            PHALA_ENV_NODE_BACKUP_R2_SECRET_ACCESS_KEY
+            PHALA_ENV_NODE_BACKUP_AGE_RECIPIENT
           )
 
           for var in "${required_vars[@]}"; do
@@ -1046,6 +1056,20 @@ jobs:
             exit 1
           fi
 
+          # TC-887: the CVM may hold only the age PUBLIC recipient. Refuse an
+          # identity (private key) or anything else before it is written.
+          case "${PHALA_ENV_NODE_BACKUP_AGE_RECIPIENT}" in
+            AGE-SECRET-KEY-*)
+              echo "::error::NODE_BACKUP_AGE_RECIPIENT is a private key; only the age1... public recipient may be sealed"
+              exit 1
+              ;;
+            age1*) ;;
+            *)
+              echo "::error::NODE_BACKUP_AGE_RECIPIENT must be an age1... public recipient"
+              exit 1
+              ;;
+          esac
+
           # Optional (TC-1482): the same single-line rule, but absence is allowed.
           if [[ "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" == *$'\n'* || "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" == *$'\r'* ]]; then
             echo "::error::Phala deploy inputs must be single-line values"
@@ -1091,6 +1115,10 @@ jobs:
             printf 'BETTERSTACK_SOURCE_TOKEN=%s\n' "$PHALA_ENV_BETTERSTACK_SOURCE_TOKEN"
             printf 'SQL_IDENTITY_FENCE=%s\n' "$PHALA_ENV_SQL_IDENTITY_FENCE"
             printf 'NAMED_KEYS_V1=%s\n' "$PHALA_ENV_NAMED_KEYS_V1"
+            printf 'NODE_BACKUP_R2_ENDPOINT=%s\n' "$PHALA_ENV_NODE_BACKUP_R2_ENDPOINT"
+            printf 'NODE_BACKUP_R2_ACCESS_KEY_ID=%s\n' "$PHALA_ENV_NODE_BACKUP_R2_ACCESS_KEY_ID"
+            printf 'NODE_BACKUP_R2_SECRET_ACCESS_KEY=%s\n' "$PHALA_ENV_NODE_BACKUP_R2_SECRET_ACCESS_KEY"
+            printf 'NODE_BACKUP_AGE_RECIPIENT=%s\n' "$PHALA_ENV_NODE_BACKUP_AGE_RECIPIENT"
             # TC-1482: optional. Unset keeps the node starting with
             # GET /admin/stats disabled (404); never written empty.
             if [ -n "${PHALA_ENV_NODE_STATS_READ_TOKEN:-}" ]; then
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6f1273e..e02d0e45 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,9 @@
 
 ## [Unreleased]
 
+### Changes
+- Production compose: a `ledger-backup` sidecar backs up the named-keys deletion ledger, the only data that exists solely on the CVM disk. It reads the `deletion-ledger` volume read-only and without a lock, keeps whole lines re-checked against the live file, encrypts them with `age` to a public recipient and uploads them to the R2 bucket `tinycloud-node-backups` every 300 s when the ledger changed and at least daily. `deploy/ledger-backup/ledger-restore-drill.sh` decrypts the newest backup into a scratch directory and checks it. This changes the compose hash (N0 template lock in `docker.yml` and the fixture move with it), and every `docker.yml` deploy now needs the `PROD_TINYCLOUD_NODE_BACKUP_*` secrets (`NODE_BACKUP_R2_ENDPOINT`, `NODE_BACKUP_R2_ACCESS_KEY_ID`, `NODE_BACKUP_R2_SECRET_ACCESS_KEY`, `NODE_BACKUP_AGE_RECIPIENT`); the node itself is unchanged. Details: TC-887
+
 ## [1.25.15] - 2026-11-01
 
 ### Changes
diff --git a/deploy/ledger-backup/README.md b/deploy/ledger-backup/README.md
new file mode 100644
index 00000000..618beeac
--- /dev/null
+++ b/deploy/ledger-backup/README.md
@@ -0,0 +1,119 @@
+# Deletion-ledger backup (TC-887)
+
+The production node (`docker-compose.dstack-postgres.yaml`, Postgres + S3)
+keeps one thing that exists only on the CVM's disk: the named-keys deletion
+ledger, `/var/lib/tinycloud/ledger/deletions.jsonl` on the `deletion-ledger`
+volume. Phala keeps no copy of CVM disks. Everything else the node writes
+locally is a cache (SQL files rehydrated from `database_artifact`), temporary
+(upload staging), per process (`runtime/control.*`) or re-issuable (TLS
+certificates).
+
+## Why the ledger matters
+
+Every delete under `apps/` is appended to the ledger before the database
+commit (`docs/encryption-named-keys.md`, "Deletion ledger"). After a database
+restore, `tinycloud node deletion-ledger replay` (also run at every node start)
+sets each key it lists back to `deleted` with no seed. Losing the ledger alone
+changes nothing visible: the node starts a new, empty one. But if the database
+is later restored from a backup taken before some deletes, those rows carry
+the sealed seeds again, the deleted keys can be derived again, and data that
+was crypto-shredded can be decrypted. The ledger is the only record that
+cancels that.
+
+## What the sidecar does
+
+`ledger-backup` (`Dockerfile`, `ledger-backup.sh`) runs next to the node:
+
+- mounts `deletion-ledger` **read-only**, as the node's uid 1000, with a
+  read-only root filesystem, no capabilities and a 32 MiB tmpfs `/tmp`. The
+  plaintext copy exists only in that tmpfs and is deleted after encryption;
+- takes no lock (the node holds `deletions.jsonl.lock` while it runs). It
+  copies the file's current length, keeps whole lines only, waits 5 s and
+  accepts the copy only if the live file still starts with exactly those
+  bytes; otherwise it copies again (an append whose write or fsync fails is
+  cut back by the node);
+- encrypts with `age -r <NODE_BACKUP_AGE_RECIPIENT>` (an `age1…` public
+  recipient; it refuses to start with an `AGE-SECRET-KEY-…`) and PUTs
+  `ledger/deletions-<UTC timestamp>.jsonl.age` to the R2 bucket
+  `tinycloud-node-backups` with `If-None-Match: *` (never overwrites). The R2
+  credentials reach curl on stdin, never in argv;
+- checks every 300 s and uploads when the content changed, and at least once
+  every 24 h even if it did not, so a stopped sidecar shows as a gap in the
+  bucket. A failed attempt logs `backup: FAILED` and retries after 60 s.
+
+Every backup is the whole ledger (it is append-only), so the newest object is
+enough for a restore. The objects are small; no lifecycle rule is needed. If
+one is added, it must never delete the newest objects.
+
+Sealed env (one card): `NODE_BACKUP_R2_ENDPOINT`, `NODE_BACKUP_R2_ACCESS_KEY_ID`,
+`NODE_BACKUP_R2_SECRET_ACCESS_KEY`, `NODE_BACKUP_AGE_RECIPIENT`, delivered by
+`docker.yml` from the `production` secrets `PROD_TINYCLOUD_NODE_BACKUP_*`
+(`docs/releasing.md`, "Deletion-ledger backup sidecar").
+
+## What is in a backup
+
+One JSON line per delete operation and per outcome, exactly the node's format:
+`{"op","networkId","keys":[{"keyPath","keyVersion"}],"deletedAt"}`,
+`{"committed": op}` and `{"aborted": op}`. `op` is the CID of the owner's
+signed delete credential; `networkId` names the owner's DID; key paths name
+app ids and key names. No key material, seeds or user data. It is still user
+metadata, hence the encryption.
+
+## Restore drill
+
+Run by an operator, never on the CVM (it needs the identity):
+
+```sh
+umask 077
+export TMPDIR="$(mktemp -d -p /var/tmp)"          # private scratch area
+export AGE_IDENTITY_FILE=/path/to/node-ledger-backup.agekey
+export NODE_BACKUP_R2_ENDPOINT=... NODE_BACKUP_R2_BUCKET=tinycloud-node-backups
+export NODE_BACKUP_R2_ACCESS_KEY_ID=... NODE_BACKUP_R2_SECRET_ACCESS_KEY=...   # a read key
+sh deploy/ledger-backup/ledger-restore-drill.sh            # newest object
+sh deploy/ledger-backup/ledger-restore-drill.sh --expect known.jsonl
+```
+
+It decrypts into a fresh scratch directory, checks that every line is a whole
+record of exactly the node's schema, prints the line count, the record counts
+(intents, committed, aborted, still open) and the last entry, optionally
+compares with a known sample (`--expect` byte for byte, `--expect-prefix`),
+and removes the scratch directory (`RESTORE_KEEP=1` keeps it and prints its
+path). Needs `age`, `jq` and curl 8 or newer.
+
+To put a restored ledger back on a node: stop the node, place the file at
+`deletion_ledger_path` (uid 1000, mode 0600), and start the node; it replays
+the ledger before serving. After a database restore, run
+`tinycloud node deletion-ledger replay` before the node starts.
+
+## Key custody
+
+- The CVM holds only the public recipient (sealed `NODE_BACKUP_AGE_RECIPIENT`).
+  `docker.yml` and the sidecar both refuse an identity in that place.
+- The identity (`AGE-SECRET-KEY-1…`) is generated off the CVM by its holder,
+  with `age-keygen -o node-ledger-backup.agekey` on a trusted machine, and is
+  never sealed, committed, pasted into CI or sent through chat.
+- The holder, where the identity is stored, and its escrow copy (a second
+  person or place, so one loss does not end recovery) are approved by security
+  review (Naomi) and recorded on TC-887 before the first deploy. The recipient
+  sealed on the CVM must be the one derived from that identity
+  (`age-keygen -y node-ledger-backup.agekey`).
+- If every copy of the identity is lost, every backup made with it is
+  unreadable. Generate a new identity, reseal the recipient and redeploy: the
+  next backup is complete again, because each backup is the whole ledger.
+- The R2 key sealed on the CVM is scoped to the `tinycloud-node-backups` bucket
+  only; a leaked key exposes ciphertext only. R2 object tokens that can write
+  can also delete, so a bucket lock rule on `ledger/` (retention in days) is
+  what stops a compromised CVM from deleting earlier backups.
+
+## CI
+
+`.github/workflows/ledger-backup.yml`: `scripts` runs `ledger-backup.test.mjs`
+(an S3 stub; misconfiguration, failed uploads, a torn final line, an append cut
+back during the copy, appends during the copy, the loop's change/unchanged
+schedule, the drill's schema checks). `image` builds the sidecar and, for
+same-repository runs, publishes `ghcr.io/tinycloudlabs/tinycloud-node-ledger-backup:<revision>`
+labelled with the sha256 of its sources. `pinned` requires the compose's
+pinned digest to carry this revision's sources and runs `ci-e2e.sh` with it:
+MinIO for R2, the compose service's flags, a fresh volume first mounted by the
+sidecar, a refused private key, no write to the container filesystem, backups
+while a writer appends, and the restore drill.
diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml
index 1b35326a..a89a389d 100644
--- a/docker-compose.dstack-postgres.yaml
+++ b/docker-compose.dstack-postgres.yaml
@@ -185,6 +185,46 @@ services:
         max-size: 10m
         max-file: '3'
 
+  # TC-887: the deletion ledger is the only data that exists solely on this
+  # CVM's disk (Phala keeps no copy of it). This sidecar snapshots it without
+  # a lock (whole lines, re-checked against the live file), encrypts it to an
+  # age PUBLIC recipient and uploads it to R2 every 300 s when it changed, and
+  # at least daily. The plaintext copy lives only in the tmpfs; the volume is
+  # read-only here. See deploy/ledger-backup/README.md.
+  ledger-backup:
+    image: ghcr.io/tinycloudlabs/tinycloud-node-ledger-backup@sha256:596961bb2a07034885b51bc78df324a08825c251a57341eae7ac2213aba70320
+    restart: unless-stopped
+    init: true
+    user: 1000:1000
+    read_only: true
+    cap_drop:
+    - ALL
+    security_opt:
+    - no-new-privileges:true
+    mem_limit: 96m
+    memswap_limit: 96m
+    cpus: 0.25
+    depends_on:
+    - tinycloud
+    volumes:
+    - deletion-ledger:/var/lib/tinycloud/ledger:ro
+    tmpfs:
+    - /tmp:rw,noexec,nosuid,size=32m,uid=1000,gid=1000,mode=0700
+    environment:
+      NODE_BACKUP_R2_ENDPOINT: "${NODE_BACKUP_R2_ENDPOINT:?seal the R2 endpoint of the ledger backup bucket}"
+      NODE_BACKUP_R2_BUCKET: tinycloud-node-backups
+      NODE_BACKUP_R2_ACCESS_KEY_ID: "${NODE_BACKUP_R2_ACCESS_KEY_ID:?seal the bucket-only R2 access key id}"
+      NODE_BACKUP_R2_SECRET_ACCESS_KEY: "${NODE_BACKUP_R2_SECRET_ACCESS_KEY:?seal the bucket-only R2 secret access key}"
+      NODE_BACKUP_AGE_RECIPIENT: "${NODE_BACKUP_AGE_RECIPIENT:?seal the age1... PUBLIC recipient; never the identity}"
+      BACKUP_INTERVAL_SECONDS: "300"
+      BACKUP_UNCHANGED_UPLOAD_SECONDS: "86400"
+      BACKUP_RETRY_SECONDS: "60"
+    logging:
+      driver: json-file
+      options:
+        max-size: 10m
+        max-file: '3'
+
 volumes:
   cert-data:
   deletion-ledger:
diff --git a/docs/encryption-named-keys.md b/docs/encryption-named-keys.md
index 15e319b0..3df67bd7 100644
--- a/docs/encryption-named-keys.md
+++ b/docs/encryption-named-keys.md
@@ -163,7 +163,9 @@ the node serves. A failed append that could not be cut back logs
 
 The node image contains the directory `/var/lib/tinycloud/ledger`, owned by
 uid 1000 (the node's user) with mode 0700; a volume mounted there is writable
-by the node.
+by the node. In the production compose a `ledger-backup` sidecar reads that
+volume (read-only, without the lock) and keeps encrypted copies of the ledger
+off the CVM (`deploy/ledger-backup/README.md`).
 
 At every start, before any route is served, the node replays the ledger onto
 the database (as `tinycloud node deletion-ledger replay` does) and closes each
diff --git a/docs/releasing.md b/docs/releasing.md
index c79e2f82..4ec7e3cb 100644
--- a/docs/releasing.md
+++ b/docs/releasing.md
@@ -583,6 +583,34 @@ its branch policy before any workflow that names it runs.
    line).
    Details: `test/probe/README.md`.
 
+### Deletion-ledger backup sidecar (TC-887): compose hash change
+
+The `ledger-backup` service in `docker-compose.dstack-postgres.yaml` backs up
+the named-keys deletion ledger, the only data that exists solely on the CVM
+disk (`deploy/ledger-backup/README.md`). It **changes the compose hash**, so
+it reaches production with the next release promotion, like any compose
+change. It goes live in its own booked window, never as a side effect of an
+unrelated release:
+
+- Merge it only once the window is booked. From the merge on, every deploy
+  through `docker.yml` (release promotion, same-image recovery and the pre-N0
+  rollback, which share the deploy step) needs the four `production`
+  environment secrets below. Without them the deploy step stops with
+  `Missing required GitHub secret for Phala deploy: NODE_BACKUP_…` before any
+  CVM change.
+- `PROD_TINYCLOUD_NODE_BACKUP_R2_ENDPOINT`, `PROD_TINYCLOUD_NODE_BACKUP_R2_ACCESS_KEY_ID`,
+  `PROD_TINYCLOUD_NODE_BACKUP_R2_SECRET_ACCESS_KEY` (a key scoped to the
+  `tinycloud-node-backups` bucket only) and `PROD_TINYCLOUD_NODE_BACKUP_AGE_RECIPIENT`
+  map to the sealed `NODE_BACKUP_R2_ENDPOINT`, `NODE_BACKUP_R2_ACCESS_KEY_ID`,
+  `NODE_BACKUP_R2_SECRET_ACCESS_KEY` and `NODE_BACKUP_AGE_RECIPIENT`. The
+  bucket name is in the compose.
+- The deploy step refuses, with a constant message, a recipient that is an age
+  identity (`AGE-SECRET-KEY-…`) or anything but an `age1…` public recipient,
+  before it writes the sealed file.
+- The sidecar image is pinned by digest in the compose. Changing the sidecar
+  means a new digest from the `ledger-backup` workflow's `image` job, a new
+  template hash here and in `docker.yml`, and the fixture copy.
+
 ### Main release with N0 log shipping (one dispatch)
 
 Ships a main release on #309's log sidecar compose. Before dispatching,
@@ -610,8 +638,8 @@ Before any CVM mutation the workflow requires:
 
 - the main compose, with its one node digest replaced by 64 zeros, to hash
   to the reviewed N0 template
-  **`c9bcbea9dac90fc2b74d8439146596ba802796a211d1b6d65aeda2286d7ed028`**
-  (raw `298ea356adf8d5e562557ab18bd3f5d8cdb816de82b1eaf6e6e50bf888e22765` normalized; before the TC-1777 named-keys gate env lines (`TINYCLOUD_ENCRYPTION_NETWORK__NAMED_KEYS_V1`, `…__DELETION_LEDGER_PATH`) and the `deletion-ledger` volume the template was `8df7e0f2…`, raw `192dfb7e407fa591d4f0232b95e43a41a85dd18add1d4976e70ebdbff5ddebca`; before the TC-1843 per-minute caps on 4xx and 5xx request events and on the 5xx diagnostic, and the `errors_dropped` event (the dropped count), in the Vector config the template was `7c3456dc…`, raw `d0e72bde…`; before the TC-1840 `db_failure_5xx` event (the node's 5xx diagnostic line shipped as an individual event) in the Vector config the template was `c91e5b7a…`, raw `99013c40…`; before the TC-950 pool of 10 (`TINYCLOUD_DATABASE__MAX_CONNECTIONS` 15 to 10) the template was `79469f97…`, raw `32a002ef…`; before the three `current_kv_c_index_*` warning codes in the Vector config (TC-1274 `current_kv` C-index migration) the template was `87b5d12a…`, raw `d1fcd665…`; before the optional TC-1482 `NODE_STATS_READ_TOKEN` env line and the TC-1482 route in the Vector config the template was `7fabba70…`, raw `1228d81927fc0a3fbba85745103e77b3112606f5d314d15974d400b222409350`: the 1.21.0 compose
+  **`0d57efc103654fe3ec361680bdd0f264d4a9db111b1a48e970f28980f4805bbc`**
+  (raw `81412971fdc38bca4f3f3b56d69d205a4455fc1ad716c062cee1450d6241f8e6` normalized; before the TC-887 `ledger-backup` sidecar (pinned by digest, with its `NODE_BACKUP_*` env lines) the template was `c9bcbea9…`, raw `298ea356adf8d5e562557ab18bd3f5d8cdb816de82b1eaf6e6e50bf888e22765`; before the TC-1777 named-keys gate env lines (`TINYCLOUD_ENCRYPTION_NETWORK__NAMED_KEYS_V1`, `…__DELETION_LEDGER_PATH`) and the `deletion-ledger` volume the template was `8df7e0f2…`, raw `192dfb7e407fa591d4f0232b95e43a41a85dd18add1d4976e70ebdbff5ddebca`; before the TC-1843 per-minute caps on 4xx and 5xx request events and on the 5xx diagnostic, and the `errors_dropped` event (the dropped count), in the Vector config the template was `7c3456dc…`, raw `d0e72bde…`; before the TC-1840 `db_failure_5xx` event (the node's 5xx diagnostic line shipped as an individual event) in the Vector config the template was `c91e5b7a…`, raw `99013c40…`; before the TC-950 pool of 10 (`TINYCLOUD_DATABASE__MAX_CONNECTIONS` 15 to 10) the template was `79469f97…`, raw `32a002ef…`; before the three `current_kv_c_index_*` warning codes in the Vector config (TC-1274 `current_kv` C-index migration) the template was `87b5d12a…`, raw `d1fcd665…`; before the optional TC-1482 `NODE_STATS_READ_TOKEN` env line and the TC-1482 route in the Vector config the template was `7fabba70…`, raw `1228d81927fc0a3fbba85745103e77b3112606f5d314d15974d400b222409350`: the 1.21.0 compose
   `d95bfb707871c1d626c4a8f2b8d32d354103edbb5077847c6a21fe5f7f6cfa50`, template
   `51f8e9e9…`, with only the embedded Vector config regenerated for TC-1274's,
   TC-1317's and TC-1482's route and warning codes (TC-1274 and TC-1317 were template `7fabba70…`, raw `1228d819…`); TC-1274 alone was template `00938226…`, raw
@@ -705,8 +733,8 @@ gh workflow run docker.yml -R TinyCloudLabs/tinycloud-node --ref main \
 
 The served/OCI identity remains **1.20.6-dstack / e18578ff9162986ddd813259c60f73866e3e60d1**.
 With the template lock on main this redeploys exactly main's raw compose
-`298ea356adf8d5e562557ab18bd3f5d8cdb816de82b1eaf6e6e50bf888e22765`
-(the 1.21.0 compose `d95bfb70…` with the TC-1274, TC-1317, TC-1482, `current_kv_c_index_*`, `db_failure_5xx`, the 4xx and 5xx caps and `errors_dropped` Vector config, the optional TC-1482 `NODE_STATS_READ_TOKEN` env line, the TC-950 pool of 10, and the TC-1777 `NAMED_KEYS_V1` gate env lines with the `deletion-ledger` volume), whatever digest
+`81412971fdc38bca4f3f3b56d69d205a4455fc1ad716c062cee1450d6241f8e6`
+(the 1.21.0 compose `d95bfb70…` with the TC-1274, TC-1317, TC-1482, `current_kv_c_index_*`, `db_failure_5xx`, the 4xx and 5xx caps and `errors_dropped` Vector config, the optional TC-1482 `NODE_STATS_READ_TOKEN` env line, the TC-950 pool of 10, the TC-1777 `NAMED_KEYS_V1` gate env lines with the `deletion-ledger` volume, and the TC-887 `ledger-backup` sidecar), whatever digest
 main's template currently pins. It is not a rollback of a
 later release across a schema migration. Current pre-N0 tag/pinned
 equivalents are separately hash-checked; arbitrary same-image config
diff --git a/scripts/ci/fixtures/tc959-n0-compose.yaml b/scripts/ci/fixtures/tc959-n0-compose.yaml
index 1b35326a..a89a389d 100644
--- a/scripts/ci/fixtures/tc959-n0-compose.yaml
+++ b/scripts/ci/fixtures/tc959-n0-compose.yaml
@@ -185,6 +185,46 @@ services:
         max-size: 10m
         max-file: '3'
 
+  # TC-887: the deletion ledger is the only data that exists solely on this
+  # CVM's disk (Phala keeps no copy of it). This sidecar snapshots it without
+  # a lock (whole lines, re-checked against the live file), encrypts it to an
+  # age PUBLIC recipient and uploads it to R2 every 300 s when it changed, and
+  # at least daily. The plaintext copy lives only in the tmpfs; the volume is
+  # read-only here. See deploy/ledger-backup/README.md.
+  ledger-backup:
+    image: ghcr.io/tinycloudlabs/tinycloud-node-ledger-backup@sha256:596961bb2a07034885b51bc78df324a08825c251a57341eae7ac2213aba70320
+    restart: unless-stopped
+    init: true
+    user: 1000:1000
+    read_only: true
+    cap_drop:
+    - ALL
+    security_opt:
+    - no-new-privileges:true
+    mem_limit: 96m
+    memswap_limit: 96m
+    cpus: 0.25
+    depends_on:
+    - tinycloud
+    volumes:
+    - deletion-ledger:/var/lib/tinycloud/ledger:ro
+    tmpfs:
+    - /tmp:rw,noexec,nosuid,size=32m,uid=1000,gid=1000,mode=0700
+    environment:
+      NODE_BACKUP_R2_ENDPOINT: "${NODE_BACKUP_R2_ENDPOINT:?seal the R2 endpoint of the ledger backup bucket}"
+      NODE_BACKUP_R2_BUCKET: tinycloud-node-backups
+      NODE_BACKUP_R2_ACCESS_KEY_ID: "${NODE_BACKUP_R2_ACCESS_KEY_ID:?seal the bucket-only R2 access key id}"
+      NODE_BACKUP_R2_SECRET_ACCESS_KEY: "${NODE_BACKUP_R2_SECRET_ACCESS_KEY:?seal the bucket-only R2 secret access key}"
+      NODE_BACKUP_AGE_RECIPIENT: "${NODE_BACKUP_AGE_RECIPIENT:?seal the age1... PUBLIC recipient; never the identity}"
+      BACKUP_INTERVAL_SECONDS: "300"
+      BACKUP_UNCHANGED_UPLOAD_SECONDS: "86400"
+      BACKUP_RETRY_SECONDS: "60"
+    logging:
+      driver: json-file
+      options:
+        max-size: 10m
+        max-file: '3'
+
 volumes:
   cert-data:
   deletion-ledger:
diff --git a/scripts/ci/test-deploy-recovery.py b/scripts/ci/test-deploy-recovery.py
index febb33c6..2419aa73 100755
--- a/scripts/ci/test-deploy-recovery.py
+++ b/scripts/ci/test-deploy-recovery.py
@@ -217,7 +217,7 @@ def _exc_info_to_string(self, error, test):
 class RecoveryTests(unittest.TestCase):
     def run_deploy(self, compose=CURRENT, changes=None, *, fail_at="", artifact_error="", authority="node-override",
                    missing_billing=False, missing_authorization=False, approved_privacy="", event="workflow_dispatch", stats_token=None,
-                   allowed_stats=False,
+                   allowed_stats=False, backup_recipient=None,
                    ref="refs/heads/main", oci_label=None, oci_revision=None, build_digest="", releases=RELEASED, running=REVISION, order=None):
         inputs = {"deploy_phala": "true", "authorize_quota_values": "true", "image_version": "1.20.6",
                   "deploy_image_digest": DIGEST, "rollback_image_version": "1.20.6", "rollback_image_revision": REVISION,
@@ -287,6 +287,8 @@ def run_deploy(self, compose=CURRENT, changes=None, *, fail_at="", artifact_erro
             if missing_billing: env.pop("QUOTA_BILLING_ADMIN_SECRET")
             env["PHALA_ENV_NAMED_KEYS_V1"] = inputs.get("named_keys_v1", "false")
             env["TINYCLOUD_ENCRYPTION_NETWORK__NAMED_KEYS_V1"] = inputs.get("named_keys_v1", "false")
+            # TC-887: the sealed recipient must look like an age public key; the canary does not.
+            env["PHALA_ENV_NODE_BACKUP_AGE_RECIPIENT"] = backup_recipient if backup_recipient is not None else "age1" + "q" * 58
             context = {"github.event_name": event, "github.ref": ref, "job.success": True,
                        **{"inputs." + key: (value == "true" if value in ("true", "false") else value) for key, value in inputs.items()}}
             logs = ""
@@ -365,6 +367,25 @@ def test_stats_token_equal_to_the_admin_secret_or_too_short_is_refused_before_th
                 self.assertTrue(accepted["success"], "A token of at least 32 bytes must be accepted")
                 self.assertEqual(accepted["env_names"].count("NODE_STATS_READ_TOKEN"), 1)
 
+    def test_backup_recipient_must_be_a_public_age_key_and_is_checked_before_the_file(self):
+        # TC-887: the CVM may hold only the age public recipient, never the identity.
+        identity = "AGE-SECRET-KEY-1" + secrets.token_hex(29).upper()
+        CANARIES.append(identity)
+        for label, value, message in (
+                ("an identity", identity, "::error::NODE_BACKUP_AGE_RECIPIENT is a private key; only the age1... public recipient may be sealed"),
+                ("an ssh key", "ssh-ed25519 AAAAC3Nza", "::error::NODE_BACKUP_AGE_RECIPIENT must be an age1... public recipient")):
+            with self.subTest(label):
+                refused = self.run_deploy(backup_recipient=value)
+                self.assertFalse(refused["success"])
+                self.assertIn(message, refused["logs"])
+                self.assertEqual(refused["events"].count("cvm-deploy"), 0, "A refused recipient must fail before the CVM mutation")
+                self.assertEqual(refused["env_names"], [], "The sealed file must not be written first")
+                self.assertNotIn(identity, refused["logs"] + refused["summary"], "The identity leaked")
+        sealed = self.run_deploy()
+        self.assertTrue(sealed["success"])
+        for name in ("NODE_BACKUP_R2_ENDPOINT", "NODE_BACKUP_R2_ACCESS_KEY_ID", "NODE_BACKUP_R2_SECRET_ACCESS_KEY", "NODE_BACKUP_AGE_RECIPIENT"):
+            self.assertEqual(sealed["env_names"].count(name), 1, name)
+
     def test_deleted_stats_token_secret_fails_closed_once_the_name_is_allowed(self):
         # TC-1482: after the first deploy with the token, `allowed_envs` on the CVM contains
         # NODE_STATS_READ_TOKEN. A later deploy (also rollback and same-image recovery,
diff --git a/scripts/tc959-docker.mjs b/scripts/tc959-docker.mjs
index f42257cd..ae8553b8 100755
--- a/scripts/tc959-docker.mjs
+++ b/scripts/tc959-docker.mjs
@@ -142,6 +142,7 @@ export async function main(args = process.argv.slice(2)) {
     DATABASE_URL: 'sqlite:/data/caps.db?mode=rwc', S3_BUCKET: 'tc959-synthetic', S3_ENDPOINT: 'http://127.0.0.1:9', AWS_KEY: 'TC959_SYNTHETIC_KEY', AWS_SECRET: 'TC959_SYNTHETIC_SECRET', AWS_REGION: 'us-east-1',
     SQL_IDENTITY_FENCE: 'false', SHARE_TRUST_BUNDLE_BASE64: Buffer.from('{}').toString('base64'), TINYCLOUD_QUOTA_URL: '', TINYCLOUD_ADMIN_SECRET: 'TC959_SYNTHETIC_ADMIN',
     CLOUDFLARE_API_TOKEN: 'TC959_SYNTHETIC_DNS', DSTACK_GATEWAY_DOMAIN: 'invalid.test', CERTBOT_EMAIL: 'email@example.test', BETTERSTACK_SOURCE_TOKEN: token, TINYCLOUD_LOGSHIP_PROFILE: 'errors_slow',
+    NODE_BACKUP_R2_ENDPOINT: 'http://127.0.0.1:9', NODE_BACKUP_R2_ACCESS_KEY_ID: 'TC959_SYNTHETIC_BACKUP_KEY', NODE_BACKUP_R2_SECRET_ACCESS_KEY: 'TC959_SYNTHETIC_BACKUP_SECRET', NODE_BACKUP_AGE_RECIPIENT: 'age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq',
   };
   const command = (binary, args, { final = false, optional = false, timeout = 15000 } = {}) => {
     if (!final) checkTime();

From d044ccd9f71c7b7074be2505917de3e4e0387648 Mon Sep 17 00:00:00 2001
From: "tinycloud-bot[bot]"
 <348376441+tinycloud-bot[bot]@users.noreply.github.com>
Date: Sun, 1 Nov 2026 21:49:58 +0000
Subject: [PATCH 3/3] ci: allow only Docker's init mount in the sidecar
 filesystem diff; Details: TC-887

---
 deploy/ledger-backup/ci-e2e.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/deploy/ledger-backup/ci-e2e.sh b/deploy/ledger-backup/ci-e2e.sh
index a56ab9e4..0a7986b6 100755
--- a/deploy/ledger-backup/ci-e2e.sh
+++ b/deploy/ledger-backup/ci-e2e.sh
@@ -87,10 +87,11 @@ echo "refused: $(cat "$scratch/refused.out")"
 
 step "Backup once, exactly as the compose service runs; nothing is written to the container's own filesystem"
 docker run --name "$run_id-backup" "${backup_args[@]}" "$image"
-changes=$(docker diff "$run_id-backup")
+# `--init` (compose `init: true`) bind-mounts Docker's own /sbin/docker-init; that is the only change allowed.
+changes=$(docker diff "$run_id-backup" | grep -v -x -e 'C /sbin' -e 'A /sbin/docker-init' || true)
 [ -z "$changes" ] || fail "the container filesystem changed: $changes"
 docker rm "$run_id-backup" >/dev/null
-echo "docker diff: (no changes)"
+echo "docker diff: no changes besides Docker's /sbin/docker-init mount"
 
 step "Restore drill: newest object -> scratch dir, schema, counts, last entry, byte-for-byte sample"
 sh "$here/ledger-restore-drill.sh" --expect "$scratch/sample.jsonl"