Give the Simulation Entrypoint its own simulation schema, migration chain and database connection

.github/scripts/cloud-run-simulation-entry-smoke.sh

@@ -25,6 +25,12 @@ previous_revision="${3:-}"
 #   and model.
 # This is the identity the gateway reports from the routing registry; the
 # numerical release sentinel checks what the worker computes.
+# REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION=true (candidate checks only):
+# /ready must report the entrypoint's own database reachable and at the
+# migration revision the image expects, which shows that the runtime secret
+# reached the revision and that the migration job ran. Stable and rollback
+# checks leave it unset, because an older image may face a newer schema.
+require_database_revision="${REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION:-}"
 expected_identity="${EXPECTED_IDENTITY:-none}"
 expected_policyengine_version="${EXPECTED_POLICYENGINE_VERSION:-}"
 expected_app_name="${EXPECTED_APP_NAME:-}"
@@ -39,6 +45,14 @@ case "${expected_identity}" in
     exit 2
     ;;
 esac
+case "${require_database_revision}" in
+  "" | true) ;;
+  *)
+    printf 'REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION must be true or empty, not %s\n' \
+      "${require_database_revision}" >&2
+    exit 2
+    ;;
+esac
 case "${expected_latest}" in
   "" | true) ;;
   *)
@@ -63,6 +77,9 @@ if [ "${expected_identity}" = "required" ] ||
 fi
 routing_max_attempts="${SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS:-24}"
 routing_retry_delay_seconds="${SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS:-5}"
+# A cold candidate may not reach its database within /ready's three-second
+# wait; an unreachable database is asked about again this many times.
+database_ready_max_attempts="${SIMULATION_DATABASE_READY_MAX_ATTEMPTS:-6}"
 
 if ! [[ "${routing_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then
   printf 'SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS must be a positive integer\n' >&2
@@ -72,6 +89,10 @@ if ! [[ "${routing_retry_delay_seconds}" =~ ^[0-9]+$ ]]; then
   printf 'SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS must be a non-negative integer\n' >&2
   exit 2
 fi
+if ! [[ "${database_ready_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then
+  printf 'SIMULATION_DATABASE_READY_MAX_ATTEMPTS must be a positive integer\n' >&2
+  exit 2
+fi
 
 health_headers="$(mktemp)"
 health_body="$(mktemp)"
@@ -131,8 +152,29 @@ while true; do
   attempt=$((attempt + 1))
 done
 
-curl --fail --silent --show-error "${base_url}/ready" |
-  jq -e '.status == "ready"' >/dev/null
+ready_body="$(curl --fail --silent --show-error "${base_url}/ready")"
+jq -e '.status == "ready"' <<<"${ready_body}" >/dev/null
+if [ "${require_database_revision}" = "true" ]; then
+  database_attempt=1
+  while [ "${database_attempt}" -lt "${database_ready_max_attempts}" ] &&
+    jq -e '.simulation_database.reachable == false' <<<"${ready_body}" >/dev/null; do
+    printf '%s: the simulation database was not reached; asking again (%s/%s)\n' \
+      "${base_url}" "${database_attempt}" "${database_ready_max_attempts}" >&2
+    sleep "${routing_retry_delay_seconds}"
+    ready_body="$(curl --fail --silent --show-error "${base_url}/ready")"
+    database_attempt=$((database_attempt + 1))
+  done
+fi
+if [ "${require_database_revision}" = "true" ] &&
+  ! jq -e '
+    .simulation_database.reachable == true
+    and (.simulation_database.revision | type == "string")
+    and .simulation_database.revision == .simulation_database.expected_revision
+  ' <<<"${ready_body}" >/dev/null; then
+  printf '%s: the simulation database is not at the expected revision: %s\n' \
+    "${base_url}" "$(jq -c '.simulation_database' <<<"${ready_body}")" >&2
+  exit 1
+fi
 curl --fail --silent --show-error "${base_url}/versions" --output "${versions_body}"
 jq -e 'type == "object"' "${versions_body}" >/dev/null
 

.github/scripts/deploy-cloud-run-simulation-entry.sh

@@ -58,6 +58,28 @@ resolve_secret_reference() {
   printf '%s:%s' "${reference}" "${highest_enabled_version}"
 }
 
+runtime_secrets=()
+
+# Resolve one secret per assignment: an array assignment takes its exit status
+# from its last command substitution only, so resolving several secrets in
+# one would let an earlier failure pass unnoticed.
+add_runtime_secret() {
+  local variable_name="$1"
+  local resolved_reference
+  resolved_reference="$(resolve_secret_reference "$2")"
+  runtime_secrets+=("${variable_name}=${resolved_reference}")
+}
+
+# The Stage 12 comparison path's own wrapper, removed with it, so the main
+# path's wrapper can change without touching the Stage 12 block. Both use the
+# secret lookups above, which predate the split.
+add_stage12_runtime_secret() {
+  local variable_name="$1"
+  local resolved_reference
+  resolved_reference="$(resolve_secret_reference "$2")"
+  runtime_secrets+=("${variable_name}=${resolved_reference}")
+}
+
 required_variables=(
   PROJECT_ID
   REGION
@@ -77,6 +99,7 @@ required_variables=(
   OLD_GATEWAY_AUTH_AUDIENCE_VALUE
   OLD_GATEWAY_AUTH_CLIENT_ID_VALUE
   OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME
+  SIMULATION_DATABASE_URL_SECRET_NAME
   OBSERVABILITY_SERVICE_NAMESPACE
   OBSERVABILITY_TRACE_PROJECT_ID
   OTEL_EXPORTER_OTLP_ENDPOINT
@@ -115,11 +138,11 @@ export SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE="${SIMULATION_ENTRYPOINT_
 [[ "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "0" || "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "1" ]] \
   || fail "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE must be 0 or 1"
 
-runtime_secrets=(
-  "OLD_GATEWAY_AUTH_CLIENT_SECRET=$(
-    resolve_secret_reference "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}"
-  )"
-)
+add_runtime_secret OLD_GATEWAY_AUTH_CLIENT_SECRET \
+  "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}"
+# The runtime role's URL for the entrypoint's own simulation schema. The
+# migration job, which uses a separate migrator URL, runs before this deploy.
+add_runtime_secret SIMULATION_DATABASE_URL "${SIMULATION_DATABASE_URL_SECRET_NAME}"
 
 if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then
   stage12_variables=(
@@ -138,11 +161,9 @@ if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then
     || fail "STAGE12_ARTIFACT_BUCKET_VALUE is invalid"
   validate_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}"
 
-  runtime_secrets+=(
-    "MODAL_TOKEN_ID=$(resolve_secret_reference "${MODAL_TOKEN_ID_SECRET_NAME}")"
-    "MODAL_TOKEN_SECRET=$(resolve_secret_reference "${MODAL_TOKEN_SECRET_SECRET_NAME}")"
-    "STAGE12_DATABASE_URL=$(resolve_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}")"
-  )
+  add_stage12_runtime_secret MODAL_TOKEN_ID "${MODAL_TOKEN_ID_SECRET_NAME}"
+  add_stage12_runtime_secret MODAL_TOKEN_SECRET "${MODAL_TOKEN_SECRET_SECRET_NAME}"
+  add_stage12_runtime_secret STAGE12_DATABASE_URL "${STAGE12_DATABASE_URL_SECRET_NAME}"
 fi
 
 runtime_environment_file="${RUNNER_TEMP}/simulation-entry-runtime-environment.yaml"

.github/scripts/migrate-simulation-database.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Bring the simulation schema to the chain's head, or require it to be there.
+# Run from libs/policyengine-simulation-persistence, whose locked environment
+# (with the migrations extra) provides Alembic. migrations/env.py reads the
+# migrator URL from SIMULATION_MIGRATION_DATABASE_URL; this script never
+# prints it.
+#
+# SIMULATION_MIGRATION_MODE:
+#   upgrade  apply every pending revision, then require the database at the
+#            head (deployments from the default branch);
+#   verify   apply nothing and only require the database at this checkout's
+#            head, so a deployment dispatched from another branch can never
+#            apply that branch's revisions.
+
+set -euo pipefail
+set +x
+
+if [[ -z "${SIMULATION_MIGRATION_DATABASE_URL:-}" ]]; then
+  printf 'SIMULATION_MIGRATION_DATABASE_URL must name the simulation database\n' >&2
+  exit 1
+fi
+
+mode="${SIMULATION_MIGRATION_MODE:-}"
+case "${mode}" in
+  upgrade | verify) ;;
+  *)
+    printf 'SIMULATION_MIGRATION_MODE must be upgrade or verify, not %s\n' "${mode:-empty}" >&2
+    exit 1
+    ;;
+esac
+
+alembic_config="../../migrations/alembic.ini"
+
+if [[ "${mode}" == "upgrade" ]]; then
+  uv run --frozen --extra migrations alembic -c "${alembic_config}" upgrade head
+fi
+uv run --frozen --extra migrations alembic -c "${alembic_config}" current --check-heads

.github/workflows/pr.yml

@@ -16,7 +16,7 @@ jobs:
     # every job passed. See docs/engineering/skills/github-prs.md.
     name: CI complete
     if: ${{ always() }}
-    needs: [test, lint, pyright, quality-guards, check-changelog, docker-build, test-integration]
+    needs: [test, simulation-database, lint, pyright, quality-guards, check-changelog, docker-build, test-integration]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -51,6 +51,7 @@ jobs:
           - projects/policyengine-simulation-gateway
           - libs/policyengine-simulation-contract
           - libs/policyengine-simulation-observability
+          - libs/policyengine-simulation-persistence
           - libs/policyengine-stage12-persistence
           - libs/policyengine-fastapi
 
@@ -76,6 +77,55 @@ jobs:
       working-directory: ${{ matrix.project }}
       run: uv run --frozen pytest tests/ -v
 
+  simulation-database:
+    name: Simulation database
+    # Runs the simulation schema's migration chain (upgrade, alembic check,
+    # downgrade, upgrade) and the persistence tests that need a database, on a
+    # fresh PostgreSQL. Full CI runs only on the lowest unmerged pull request
+    # in a stack.
+    if: ${{ !github.event.pull_request.stack || github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref }}
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: simulation_ci
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd="pg_isready -U postgres"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=20
+    env:
+      SIMULATION_TEST_DATABASE_URL: postgresql://postgres:simulation_ci@127.0.0.1:5432/postgres
+      # A missing server fails the tests instead of skipping them.
+      SIMULATION_REQUIRE_DATABASE_TESTS: "1"
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.13'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v8.1.0
+      with:
+        version: "0.13.6"
+        enable-cache: true
+
+    - name: Install dependencies
+      working-directory: libs/policyengine-simulation-persistence
+      run: uv sync --extra test --frozen
+
+    - name: Run the migration chain and persistence tests on PostgreSQL
+      working-directory: libs/policyengine-simulation-persistence
+      run: uv run --frozen pytest tests/ -v -m postgres
+
   lint:
     name: Lint
     # Full CI runs only on the lowest unmerged pull request in a stack.
@@ -191,6 +241,7 @@ jobs:
         package:
           - libs/policyengine-simulation-contract
           - libs/policyengine-simulation-observability
+          - libs/policyengine-simulation-persistence
           - libs/policyengine-stage12-persistence
           - libs/policyengine-fastapi
           - projects/policyengine-simulation-entry

.github/workflows/simulation-deploy.reusable.yml

@@ -155,10 +155,42 @@ jobs:
           GATEWAY_AUTH_REQUIRED: ${{ vars.GATEWAY_AUTH_REQUIRED }}
         run: ../../.github/scripts/modal-sync-secrets.sh "${{ inputs.modal_environment }}"
 
+  migrate_simulation_database:
+    # Applies the simulation schema's Alembic chain before the entrypoint
+    # candidate deploys. The previous revision keeps serving meanwhile, so
+    # every migration must work with the release before it.
+    name: Migrate the simulation database schema
+    needs: prepare
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.release_environment }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+
+      - uses: astral-sh/setup-uv@v8.1.0
+        with:
+          version: "0.13.6"
+
+      - name: Install migration dependencies
+        working-directory: libs/policyengine-simulation-persistence
+        run: uv sync --frozen --extra migrations
+
+      - name: Apply simulation schema migrations
+        working-directory: libs/policyengine-simulation-persistence
+        env:
+          SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }}
+          # Only a deployment from main applies revisions. One dispatched from
+          # another branch requires the database to be at its chain's head.
+          SIMULATION_MIGRATION_MODE: ${{ github.ref == 'refs/heads/main' && 'upgrade' || 'verify' }}
+        run: ../../.github/scripts/migrate-simulation-database.sh
+
   deploy_entrypoint:
     name: Deploy and unit test Cloud Run entrypoint
-    if: ${{ always() && needs.prepare.result == 'success' && ((inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'success') || (!inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'skipped')) }}
-    needs: [prepare, configure_stage12_v2]
+    if: ${{ always() && needs.prepare.result == 'success' && needs.migrate_simulation_database.result == 'success' && ((inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'success') || (!inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'skipped')) }}
+    needs: [prepare, configure_stage12_v2, migrate_simulation_database]
     runs-on: ubuntu-latest
     environment: ${{ inputs.release_environment }}
     permissions:
@@ -241,6 +273,7 @@ jobs:
           OLD_GATEWAY_AUTH_AUDIENCE_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_AUDIENCE }}
           OLD_GATEWAY_AUTH_CLIENT_ID_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_CLIENT_ID }}
           OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME: ${{ vars.OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME }}
+          SIMULATION_DATABASE_URL_SECRET_NAME: ${{ vars.SIMULATION_DATABASE_URL_SECRET_NAME }}
           STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}
           STAGE12_ARTIFACT_BUCKET_VALUE: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
           STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }}
@@ -764,6 +797,8 @@ jobs:
           EXPECTED_UK_DATASET_SHA256: ${{ inputs.deploy_existing_stack && needs.prepare.outputs.uk_dataset_sha256 || '' }}
           # Latest must route to this release only when it should advance it.
           EXPECTED_LATEST: ${{ inputs.deploy_existing_stack && needs.update_routing.outputs.advances_latest == 'true' && 'true' || '' }}
+          # The candidate reaches its own database at the migrated revision.
+          REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION: "true"
         run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.deploy_entrypoint.outputs.revision }}"
 
       - name: Download the numerical release sentinel

AGENTS.md

@@ -128,7 +128,11 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`.
 
 ### Stage 12 persistence
 
-- Checked: `sim.stage12.deployment-scope`, `sim.stage12.deployment-uses-api-schema`, `sim.stage12.no-migration-chain`, `sim.stage12.no-production-user-associations`, `sim.stage12.no-schema-ddl`, `sim.stage12.persistence-typed-dml-only`, `sim.stage12.production-lock-has-db-runtime`
+- Checked: `sim.stage12.deployment-scope`, `sim.stage12.deployment-uses-api-schema`, `sim.stage12.no-production-user-associations`, `sim.stage12.no-schema-ddl`, `sim.stage12.persistence-typed-dml-only`, `sim.stage12.production-lock-has-db-runtime`
+
+### Simulation persistence
+
+- Checked: `sim.persistence.executor-independent`, `sim.persistence.migrate-before-entrypoint`, `sim.persistence.migration-cycle`, `sim.persistence.no-modal-database-credentials`, `sim.persistence.one-migration-chain`
 
 ### Deployment pipeline
 

Makefile

@@ -91,6 +91,7 @@ test:
 		projects/policyengine-simulation-gateway \
 		libs/policyengine-simulation-contract \
 		libs/policyengine-simulation-observability \
+		libs/policyengine-simulation-persistence \
 		libs/policyengine-stage12-persistence \
 		libs/policyengine-fastapi; do \
 		echo "Testing $$proj..."; \

changelog.d/897.added.md

@@ -0,0 +1,3 @@
+Added the `simulation` Postgres schema, in the database that holds the Public API's v2 tables, with its own Alembic migration chain at `migrations/` and the `policyengine-simulation-persistence` library. Each Simulation Entrypoint deploy now first runs the `migrate_simulation_database` job, which applies the chain's migrations on a deployment from `main` and only checks that the database is at the chain's head on a deployment from any other branch. The entrypoint gets `SIMULATION_DATABASE_URL` from Secret Manager, and `GET /ready` reports the schema's applied and expected revisions. No application data is stored in the schema yet.
+
+Every deploy now needs the `SIMULATION_MIGRATION_DATABASE_URL` environment secret, the `SIMULATION_DATABASE_URL_SECRET_NAME` environment variable and the Secret Manager secret it names, in both `beta` and `prod`; without them the deploy stops at the migration job.