AP-SPEC-066 Epic 4: gateway readiness, outage controls and recovery operations

.github/CODEOWNERS

@@ -7,3 +7,5 @@
 /product/ @auths-dev/product-maintainers @auths-dev/security
 /bindings/ @auths-dev/binding-maintainers @auths-dev/security
 /demos/ @auths-dev/product-maintainers
+
+/deployment/ @auths-dev/product-maintainers @auths-dev/security

.github/ci/phase-ownership.toml

@@ -155,6 +155,14 @@ kind = "prefix"
 value = "qualification/"
 phases = ["authoritative", "compliance", "secrets"]
 
+# Gateway deployment and packaged operator assets include the PostgreSQL
+# recovery boundary and release handoff, without changing other providers.
+[[rules]]
+id = "gateway-deployment"
+kind = "prefix"
+value = "deployment/gateway/"
+phases = ["authoritative", "compliance", "secrets", "postgresql_live", "release"]
+
 [[rules]]
 id = "interoperability-fixtures"
 kind = "prefix"

.github/workflows/gateway-operator-package.yml

@@ -0,0 +1,126 @@
+name: Gateway operator package
+
+on:
+  pull_request:
+    paths:
+      - 'deployment/gateway/**'
+      - 'product/runtime/auths-gateway/**'
+      - 'product/qualification/**'
+      - 'docs/operations/GATEWAY_PRODUCTION_RUNBOOK.md'
+      - '.github/workflows/gateway-operator-package.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  package:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+      - uses: ./.github/actions/setup-rust-cache
+        with:
+          toolchain: 1.97.1
+      - name: Build shipped operator binaries without test features
+        run: cargo build --locked --release -p auths-gateway -p auths-recipe-qualification-issuance --bin auths-gateway --bin auths-qualification
+      - name: Make the source-free handoff
+        run: |
+          python3 deployment/gateway/tools/package.py \
+            --gateway target/release/auths-gateway \
+            --qualification target/release/auths-qualification \
+            --commit "$(git rev-parse HEAD)" \
+            --output target/gateway-operator-package.tgz
+      - name: Verify the packaged payload and command surfaces
+        run: |
+          mkdir -p "$RUNNER_TEMP/operator"
+          tar xzf target/gateway-operator-package.tgz -C "$RUNNER_TEMP/operator"
+          cd "$RUNNER_TEMP/operator/auths-gateway-operator"
+          python3 - <<'PY'
+          import hashlib, json
+          from pathlib import Path
+          manifest = json.loads(Path('manifest.json').read_text())
+          assert manifest['schema'] == 'auths.gateway-operator-package/1'
+          for item in manifest['files']:
+              assert hashlib.sha256(Path(item['path']).read_bytes()).hexdigest() == item['sha256']
+          assert not list(Path('.').rglob('*.rs'))
+          assert (Path('reference') / '../docs/GATEWAY_PRODUCTION_RUNBOOK.md').is_file()
+          assert '../docs/GATEWAY_PRODUCTION_RUNBOOK.md' in Path('reference/README.md').read_text()
+          PY
+          bin/auths-gateway disable --help | grep -- --store-only
+          bin/auths-gateway rotate-prepare --help | grep -- --operator-process
+          bin/auths-gateway doctor --help
+          bin/auths-qualification --help
+      - name: Verify systemd unit syntax
+        run: |
+          sudo install -D "$RUNNER_TEMP/operator/auths-gateway-operator/bin/auths-gateway" /opt/auths/bin/auths-gateway
+          sudo install -D "$RUNNER_TEMP/operator/auths-gateway-operator/bin/run-with-postgres-url" /opt/auths/bin/run-with-postgres-url
+          systemd-analyze verify deployment/gateway/systemd/*.service deployment/gateway/systemd/*.timer
+      - name: Exercise the extracted release binary with disposable installations
+        env:
+          AUTHS_GATEWAY_OPERATOR_TEST_BINARY: ${{ runner.temp }}/operator/auths-gateway-operator/bin/auths-gateway
+        run: |
+          cargo test --locked -p auths-gateway --test operator_plane --test support_bundle
+          cargo test --locked -p auths-gateway --test isolated_process -- --ignored
+      - name: Prepare public inputs for the operator simulation
+        run: |
+          mkdir -p target/operator-simulation-kit
+          cp deployment/gateway/tools/operator-simulation.py target/operator-simulation-kit/
+          cp bindings/fixtures/gateway/airtable/recipe.json target/operator-simulation-kit/
+          cp bindings/fixtures/gateway/airtable/profile.lock.json target/operator-simulation-kit/
+          cp core/fixtures/v1/denied/untrusted-root.context.cbor target/operator-simulation-kit/trusted.context.cbor
+          cp bindings/fixtures/gateway/custody-hostile.json target/operator-simulation-kit/
+          cd target/operator-simulation-kit
+          sha256sum operator-simulation.py recipe.json profile.lock.json trusted.context.cbor custody-hostile.json > SHA256SUMS
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: gateway-operator-simulation-kit
+          path: target/operator-simulation-kit/
+          if-no-files-found: error
+          retention-days: 30
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: gateway-operator-package
+          path: |
+            target/gateway-operator-package.tgz
+            target/gateway-operator-package.tgz.sha256
+          if-no-files-found: error
+          retention-days: 30
+
+  operator-simulation:
+    name: source-free operator simulation
+    needs: package
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      # No checkout, source build, repository import, or provider secret in this job.
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: gateway-operator-package
+          path: package
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: gateway-operator-simulation-kit
+          path: kit
+      - name: Verify the simulation kit
+        working-directory: kit
+        run: sha256sum -c SHA256SUMS
+      - name: Rehearse the documented operator commands and measure friction
+        env:
+          AUTHS_SIMULATION_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: |
+          sudo python3 kit/operator-simulation.py \
+            --archive "$GITHUB_WORKSPACE/package/gateway-operator-package.tgz" \
+            --inputs "$GITHUB_WORKSPACE/kit" \
+            --expected-commit "$AUTHS_SIMULATION_COMMIT" \
+            --report "$GITHUB_WORKSPACE/reports/operator-simulation.json"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        if: always()
+        with:
+          name: gateway-operator-simulation-report
+          path: reports/operator-simulation.json
+          if-no-files-found: warn
+          retention-days: 30

.github/workflows/postgres-lifecycle.yml

@@ -8,6 +8,7 @@ on:
       - "product/stores/auths-stores/**"
       - "bindings/fixtures/gateway/**"
       - ".github/workflows/postgres-lifecycle.yml"
+      - "deployment/gateway/**"
   workflow_dispatch:
 
 permissions:
@@ -40,6 +41,8 @@ jobs:
         run: |
           cargo test -p auths-gateway --lib -- --ignored postgres
           cargo test -p auths-gateway --features testkit-production-plaintext,testkit-production-unqualified --test operator_plane -- --ignored postgres
+      - name: Rehearse a physical backup and point-in-time restore
+        run: deployment/gateway/tools/exercise-postgres-restore.sh
       - name: Stop fixture
         if: always()
         run: docker compose -f product/stores/auths-stores/tests/postgres_tls/compose.yaml down -v

bindings/fixtures/gateway/production-codes.json

@@ -1,6 +1,14 @@
 {
   "schema": "auths.gateway-production-codes/1",
   "codes": [
+    {
+      "code": "gateway.admin.credential-journal-unavailable",
+      "owner": "operator",
+      "stage": "before-custody",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
+    },
     {
       "code": "gateway.attempt.replay",
       "owner": "engine",
@@ -23,6 +31,14 @@
         "id": "lease-generation-not-held"
       }
     },
+    {
+      "code": "gateway.connection.restore-rollback",
+      "owner": "connection",
+      "stage": "before-lease",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
+    },
     {
       "code": "gateway.credential.adapter-unsupported",
       "owner": "credential-store",
@@ -156,7 +172,47 @@
       "code": "gateway.readiness.connection-disabled",
       "owner": "readiness",
       "stage": "doctor",
-      "status": "new",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
+    },
+    {
+      "code": "gateway.readiness.observer-unavailable",
+      "owner": "readiness",
+      "stage": "doctor",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
+    },
+    {
+      "code": "gateway.readiness.recipe-drift",
+      "owner": "readiness",
+      "stage": "doctor",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
+    },
+    {
+      "code": "gateway.readiness.store-unavailable",
+      "owner": "readiness",
+      "stage": "doctor",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
+    },
+    {
+      "code": "gateway.readiness.transport-unavailable",
+      "owner": "readiness",
+      "stage": "doctor",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
+    },
+    {
+      "code": "gateway.readiness.trust-unavailable",
+      "owner": "readiness",
+      "stage": "doctor",
+      "status": "implemented",
       "epic": 4,
       "case": null
     },
@@ -167,6 +223,30 @@
       "status": "existing",
       "epic": null,
       "case": null
+    },
+    {
+      "code": "gateway.support.attempts-unavailable",
+      "owner": "support",
+      "stage": "support-bundle",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
+    },
+    {
+      "code": "gateway.support.connection-unavailable",
+      "owner": "support",
+      "stage": "support-bundle",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
+    },
+    {
+      "code": "gateway.support.unavailable",
+      "owner": "support",
+      "stage": "support-bundle",
+      "status": "implemented",
+      "epic": 4,
+      "case": null
     }
   ]
 }

bindings/python/docs/INTEGRATION_RECIPES.md

@@ -37,3 +37,21 @@ Identity, custody, reservation, and bounded-transport extension contracts have
 their own conformance suites. Passing one of those suites qualifies only the
 named mechanism. It does not qualify a new provider domain, domain errors,
 reconciliation behavior, or receipt semantics.
+
+
+## Production diagnostics and recovery
+
+The operator's `auths-gateway doctor` reports `auths.gateway-readiness/1`,
+with every required check and the optional observer state; any failed or
+unmade check gives a nonzero exit. Keep this output on the operator plane.
+The application receives `not-entered` with the gateway's exact stable code
+when qualification or a restore floor prevents a lease. It needs operator
+repair of the connection or signed inputs before another authorized operation
+can proceed. An `unknown` write retains its operation identity and is
+reconciled by read-only `reobserve`, without issuing another provider write.
+
+Signed observations require a configured observer. Production may be ready
+with no observer; that deployment reports signed outcomes unavailable.
+The [production runbook](../../../docs/operations/GATEWAY_PRODUCTION_RUNBOOK.md)
+covers disable/revoke during custody outages, exact-generation collection,
+rotation, qualification expiry/revocation and restore-floor recovery.

bindings/python/tests/test_gateway_client.py

@@ -21,6 +21,7 @@
     GatewayClient,
     GatewayEndpoint,
     GatewayObservationRefused,
+    GatewayNotEntered,
     GatewayObserved,
     GatewayObservedByProvider,
     GatewayPreEntryObservations,
@@ -105,6 +106,25 @@ def test_client_rejects_invalid_endpoint_and_oversized_input(socket_dir) -> None
         asyncio.run(client.submit(proof=b"", action=b"action"))
 
 
+@pytest.mark.parametrize("code", [
+    "gateway.qualification.missing",
+    "gateway.qualification.expired",
+    "gateway.qualification.revoked",
+    "gateway.qualification.digest-mismatch",
+    "gateway.qualification.target-mismatch",
+    "gateway.qualification.unavailable",
+    "gateway.qualification.revocation-stale",
+    "gateway.qualification.clock-untrusted",
+    "gateway.connection.restore-rollback",
+])
+def test_operator_gate_refusals_remain_not_entered_with_the_native_code(code) -> None:
+    from auths.gateway import _parse_result
+
+    result = _parse_result(json.dumps({"outcome": "not-entered", "code": code}).encode())
+    assert result == GatewayNotEntered(code)
+    assert result.outcome == "not-entered"
+
+
 def test_result_parser_does_not_infer_effect() -> None:
     from auths.gateway import _parse_result
 

bindings/typescript/README.md

@@ -50,6 +50,19 @@ The [Stripe refund example](../../examples/stripe-refund-approval/README.md)
 runs the whole journey: a grant with limits, a 2-of-3 approval quorum, gateway
 submission, and an offline audit.
 
+Keep the exact `code` on a `not-entered` result. Qualification refusals and
+`gateway.connection.restore-rollback` mean this submission stopped before
+provider entry; they do not establish the outcome of any earlier attempt.
+An `unknown` result means the write may have happened. Retain its operation
+identifier and ask the operator to reconcile it with `reobserve`; do not
+resubmit it as a new operation. Signed observations require a separately
+configured observer. An absent observer does not establish a signed outcome.
+
+The operator's `doctor` report lists each required production check and
+returns nonzero if any check fails. Use the
+[production operations runbook](../../docs/operations/GATEWAY_PRODUCTION_RUNBOOK.md)
+for diagnosis, emergency stop, rotation and recovery.
+
 ## Author the proof
 
 `@auths-dev/sdk/self-hosted` generates an exact MCP-shaped tool from a

bindings/typescript/test/unit/gateway-client.test.js

@@ -296,3 +296,16 @@ test("gateway client refuses unbounded observation requests before connecting",
     await assert.rejects(client.observePreEntry(operation), TypeError);
   }
 });
+
+
+test("operator gate refusals remain not-entered with the native code", { skip: process.platform === "win32" }, async () => {
+  for (const code of [
+    "gateway.qualification.missing", "gateway.qualification.expired",
+    "gateway.qualification.revoked", "gateway.qualification.digest-mismatch",
+    "gateway.qualification.target-mismatch", "gateway.qualification.unavailable",
+    "gateway.qualification.revocation-stale", "gateway.qualification.clock-untrusted",
+    "gateway.connection.restore-rollback",
+  ]) {
+    assert.deepEqual(await replyOnce(JSON.stringify({ outcome: "not-entered", code })), { outcome: "not-entered", code });
+  }
+});