From c249b7c07e7a380edc3350a58b142a4b54762309 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Tue, 8 Sep 2026 11:44:43 -0400 Subject: [PATCH 1/5] Exercise the migration runner and distinguish drift from operational failure Signed-off-by: Connor Tsui Signed-off-by: Will Manning --- bench/migrations/README.md | 19 +++ bench/scripts/migrate-schema.py | 9 +- bench/scripts/tests/test_migrate_schema.py | 138 +++++++++++++++++++++ bench/web/README.md | 2 +- bench/web/lib/test-harness.ts | 76 +++++------- 5 files changed, 190 insertions(+), 54 deletions(-) create mode 100644 bench/scripts/tests/test_migrate_schema.py diff --git a/bench/migrations/README.md b/bench/migrations/README.md index 3fa8adeb..c9c102ab 100644 --- a/bench/migrations/README.md +++ b/bench/migrations/README.md @@ -126,3 +126,22 @@ role (another `CREATE ROLE`, `ALTER DEFAULT PRIVILEGES`, DDL on master-owned tables, or other superuser-only DDL) should carry `-- migrate-schema: requires-superuser` on a comment line so the same preflight guards it. + +## Runner checks + +`status` returns 0 for a matching ledger, 1 for confirmed pending or orphaned filenames, and 2 +for connection, permission, filesystem, or usage failures. `apply` returns 0 on success and 2 +on failure. The schema workflow resolves Python dependencies before interpreting these codes, +so a dependency download failure cannot appear as an informational dry-run drift result. + +Run the CLI integration suite with Docker available: + +```bash +uv run --no-project --with 'psycopg[binary]>=3.2' python -m unittest discover -s scripts/tests -v +``` + +The suite applies the real migrations to disposable PostgreSQL 16, checks a second apply, +transaction rollback, ledger drift, role permissions, and a later migration as `migrator`. +A non-superuser CREATEROLE bootstrap role exercises the role-grant path. The local `rds_iam` +stand-in checks membership grants only. These tests do not verify RDS IAM authentication, +AWS trust, or RDS-specific administrative privileges. diff --git a/bench/scripts/migrate-schema.py b/bench/scripts/migrate-schema.py index 0cd57210..7310c36d 100644 --- a/bench/scripts/migrate-schema.py +++ b/bench/scripts/migrate-schema.py @@ -28,10 +28,10 @@ Exit codes ---------- -- `apply` : 0 on success (zero or more migrations applied). +- `apply` : 0 on success, 2 on operational or usage failure. - `status` : 0 when the on-disk migration set matches the ledger; 1 when there is drift (pending files OR applied-but-deleted - files). CI uses this for clean-tree gates. + files), 2 on operational or usage failure. CI must not treat 2 as drift. """ import argparse @@ -308,10 +308,7 @@ def main() -> int: print(f"{count} migration(s) applied", file=sys.stderr) return 0 return status(conn, args.migrations) - except FileNotFoundError as e: - # Translate the typed exception from `discover` into a clean CLI - # error; callers that import this module get the original exception - # via `discover` instead of an opaque SystemExit. + except (psycopg.Error, OSError, ValueError) as e: print(str(e), file=sys.stderr) return 2 diff --git a/bench/scripts/tests/test_migrate_schema.py b/bench/scripts/tests/test_migrate_schema.py new file mode 100644 index 00000000..71ca5b14 --- /dev/null +++ b/bench/scripts/tests/test_migrate_schema.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Exercise the real CLI against disposable PostgreSQL 16 (Docker required).""" + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import time +import unittest +import uuid + +import psycopg +from psycopg.conninfo import make_conninfo + +ROOT = Path(__file__).resolve().parents[2] +RUNNER = ROOT / "scripts/migrate-schema.py" + + +class MigrationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.container = subprocess.check_output( + ["docker", "run", "--rm", "-d", "-e", "POSTGRES_PASSWORD=test", + "-p", "127.0.0.1::5432", "postgres:16-alpine"], text=True + ).strip() + cls.addClassCleanup(subprocess.run, ["docker", "rm", "-f", cls.container], + check=True, stdout=subprocess.DEVNULL) + port = subprocess.check_output( + ["docker", "port", cls.container, "5432"], text=True + ).strip().rsplit(":", 1)[1] + cls.dsn = make_conninfo(host="127.0.0.1", port=port, user="postgres", + password="test", dbname="postgres", sslmode="disable", + connect_timeout=2) + for _ in range(60): + try: + with psycopg.connect(cls.dsn): + return + except psycopg.OperationalError: + time.sleep(0.5) + raise RuntimeError("Postgres did not start") + + def setUp(self): + database = "test_" + uuid.uuid4().hex + with psycopg.connect(self.dsn, autocommit=True) as conn: + conn.execute(f'CREATE DATABASE "{database}"') + self.target = make_conninfo(self.dsn, dbname=database) + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.migrations = Path(self.directory.name) + + def run_cli(self, command, expected, *, target=None, migrations=None): + result = subprocess.run( + [sys.executable, str(RUNNER), command, "--target", target or self.target, + "--migrations", str(migrations or self.migrations)], + capture_output=True, text=True, + ) + self.assertEqual(result.returncode, expected, result.stdout + result.stderr) + return result + + def test_status_distinguishes_drift_and_failure(self): + self.run_cli("status", 0) + (self.migrations / "001_first.sql").write_text("CREATE TABLE first (id int);") + self.run_cli("status", 1) + self.run_cli("apply", 0) + self.run_cli("status", 0) + (self.migrations / "001_first.sql").unlink() + self.run_cli("status", 1) + self.run_cli("status", 2, migrations=self.migrations / "missing") + self.run_cli("status", 2, target=make_conninfo(self.target, port=1)) + self.run_cli("invalid-command", 2) + with psycopg.connect(self.target, autocommit=True) as conn: + conn.execute("CREATE ROLE denied_status LOGIN PASSWORD 'test'") + self.run_cli("status", 2, target=make_conninfo(self.target, user="denied_status")) + + def test_failed_migration_rolls_back_only_its_ddl_and_ledger(self): + (self.migrations / "001_first.sql").write_text("CREATE TABLE first (id int);") + second = self.migrations / "002_second.sql" + second.write_text("CREATE TABLE second (id int); SELECT missing_column;") + self.run_cli("apply", 2) + with psycopg.connect(self.target) as conn: + self.assertEqual(conn.execute("SELECT filename FROM public._applied_migrations").fetchall(), + [("001_first.sql",)]) + self.assertIsNone(conn.execute("SELECT to_regclass('second')").fetchone()[0]) + self.assertIsNotNone(conn.execute("SELECT to_regclass('first')").fetchone()[0]) + second.write_text("CREATE TABLE second (id int);") + self.run_cli("apply", 0) + self.assertIn("0 migration(s) applied", self.run_cli("apply", 0).stderr) + self.run_cli("status", 0) + + def test_real_bootstrap_and_steady_state_permissions(self): + migrations = sorted((ROOT / "migrations").glob("*.sql")) + bootstrap_end = max(i for i, path in enumerate(migrations) + if "-- migrate-schema: requires-superuser" in path.read_text()) + for migration in migrations[:bootstrap_end + 1]: + shutil.copy(migration, self.migrations) + # CREATEROLE models the non-superuser bootstrap path. The local rds_iam + # role exercises guarded grants only, not RDS IAM authentication. + with psycopg.connect(self.target, autocommit=True) as conn: + conn.execute("CREATE ROLE rds_iam") + conn.execute("CREATE ROLE bootstrap LOGIN CREATEROLE PASSWORD 'test'") + conn.execute("GRANT rds_iam TO bootstrap WITH ADMIN TRUE") + conn.execute("GRANT ALL ON SCHEMA public TO bootstrap WITH GRANT OPTION") + bootstrap = make_conninfo(self.target, user="bootstrap") + self.run_cli("apply", 0, target=bootstrap) + self.run_cli("status", 0, target=bootstrap) + with psycopg.connect(self.target, autocommit=True) as conn: + for role in ("migrator", "bench_ingest", "bench_read"): + conn.execute(f"ALTER ROLE {role} PASSWORD 'test'") + migrator = make_conninfo(self.target, user="migrator") + for migration in migrations[bootstrap_end + 1:]: + shutil.copy(migration, self.migrations) + self.run_cli("apply", 0, target=migrator) + self.run_cli("status", 0, target=migrator) + self.assertIn("0 migration(s) applied", self.run_cli("apply", 0, target=migrator).stderr) + (self.migrations / "999_future.sql").write_text("CREATE TABLE future (id int);") + self.run_cli("apply", 0, target=migrator) + for role, allowed in (("bench_read", "SELECT"), ("bench_ingest", "SELECT,INSERT,UPDATE")): + with psycopg.connect(make_conninfo(self.target, user=role)) as conn: + for privilege in allowed.split(","): + self.assertTrue(conn.execute("SELECT has_table_privilege('future', %s)", (privilege,)).fetchone()[0]) + self.assertFalse(conn.execute("SELECT has_table_privilege('future', 'DELETE')").fetchone()[0]) + self.assertFalse(conn.execute("SELECT has_schema_privilege('public', 'CREATE')").fetchone()[0]) + self.assertFalse(conn.execute("SELECT has_table_privilege('_applied_migrations', 'SELECT')").fetchone()[0]) + if role == "bench_read": + self.assertFalse(conn.execute("SELECT has_table_privilege('commits', 'INSERT')").fetchone()[0]) + else: + conn.execute("INSERT INTO future VALUES (1)") + conn.execute("UPDATE future SET id = 2") + (self.migrations / "999_marked.sql").write_text( + "-- migrate-schema: requires-superuser\nCREATE TABLE forbidden (id int);" + ) + result = self.run_cli("apply", 2, target=migrator) + self.assertIn("master-capable", result.stderr) + with psycopg.connect(self.target) as conn: + self.assertIsNone(conn.execute("SELECT to_regclass('forbidden')").fetchone()[0]) diff --git a/bench/web/README.md b/bench/web/README.md index a893adc1..8a1078be 100644 --- a/bench/web/README.md +++ b/bench/web/README.md @@ -14,7 +14,7 @@ pnpm dev # needs BENCH_DB_* pointing at a database (see below) pnpm format:check # prettier pnpm lint # eslint pnpm build # next build; deliberately works WITHOUT a database -pnpm test # vitest; the Postgres integration suite needs a Docker daemon +pnpm test # vitest; the Postgres integration suite needs Docker and uv ``` `next build` never touches the database: every page and route is request-rendered diff --git a/bench/web/lib/test-harness.ts b/bench/web/lib/test-harness.ts index 39045df7..ad77e890 100644 --- a/bench/web/lib/test-harness.ts +++ b/bench/web/lib/test-harness.ts @@ -6,18 +6,15 @@ * is imported only by `*.test.ts` files; it never reaches a production bundle. * * It centralizes the pieces the suites previously copy-pasted (and that had - * already begun to drift in name): the Docker probe, the migrations DDL, the + * already begun to drift in name): the Docker probe, the migration runner, the * container-boot + `BENCH_DB_*` env wiring, and the canonical three-commit * chart fixture mirroring `server/tests/common/mod.rs`. */ -import { execSync } from 'node:child_process'; -import { readdirSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { execFileSync, execSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { PostgreSqlContainer, type StartedPostgreSqlContainer } from '@testcontainers/postgresql'; import type { Pool } from 'pg'; -import { getPool } from './db'; // Mirror the repo's Python `_docker_available()` precedent: the integration // tests need a Docker daemon, so they are skipped (not failed) when one is @@ -31,44 +28,8 @@ export function dockerAvailable(): boolean { } } -/** Absolute path of the repository's `migrations/` directory. */ -const MIGRATIONS_DIR = fileURLToPath(new URL('../../migrations', import.meta.url)); - -/** - * Every migration file in runner order (sorted filenames), so the suites - * exercise the same DDL sequence `scripts/migrate-schema.py apply` runs and a - * future schema migration is automatically covered by the web tests (the - * web-deploy workflow gates on `migrations/**` for exactly this reason). The - * full set is applicable here because migrations are substrate-portable by - * policy (002/004 guard their `rds_iam` grants behind existence checks) and - * the container connects as the superuser, which satisfies 004's - * requires-superuser marker. - */ -const MIGRATION_FILES: readonly string[] = readdirSync(MIGRATIONS_DIR, { withFileTypes: true }) - .filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith('.sql')) - .map((entry) => entry.name) - .sort(); - -/** - * The migration-ledger DDL, kept in lockstep with `scripts/migrate-schema.py` - * (`APPLIED_MIGRATIONS_DDL`): the runner creates the ledger BEFORE applying - * any file, and `migrations/003` grants on it, so applying the files without - * the ledger would fail. - */ -const LEDGER_DDL = `CREATE TABLE IF NOT EXISTS public._applied_migrations ( - filename TEXT PRIMARY KEY, - applied_at TIMESTAMPTZ NOT NULL DEFAULT now() -)`; - -/** - * Start a `postgres:16-alpine` testcontainer, point the connection lib at it - * via the `BENCH_DB_*` env vars (`BENCH_DB_PASSWORD` set means the IAM token - * path is bypassed), and apply the migration set unless `applySchema: false`. - * Each file runs as one statement batch; the runner's `-- migrate-schema:` - * directives are NOT interpreted here (none of the current migrations needs - * that: requires-superuser is satisfied by the container superuser, and no - * migration uses no-transaction). Callers own teardown: `await resetPool()` - * then `await container.stop()` in `afterAll`. +/** Start a disposable Postgres instance and apply the actual migration runner. + * Callers own teardown: resetPool(), then container.stop(). Requires uv on PATH. */ export async function startBenchContainer( options: { applySchema?: boolean } = {}, @@ -81,10 +42,31 @@ export async function startBenchContainer( process.env.BENCH_DB_PASSWORD = container.getPassword(); process.env.BENCH_DB_SSL = 'disable'; if (options.applySchema !== false) { - const pool = getPool(); - await pool.query(LEDGER_DDL); - for (const name of MIGRATION_FILES) { - await pool.query(readFileSync(join(MIGRATIONS_DIR, name), 'utf8')); + try { + execFileSync( + 'uv', + [ + 'run', + '--no-project', + fileURLToPath(new URL('../../scripts/migrate-schema.py', import.meta.url)), + 'apply', + ], + { + env: { + ...process.env, + PGHOST: container.getHost(), + PGPORT: String(container.getPort()), + PGDATABASE: container.getDatabase(), + PGUSER: container.getUsername(), + PGPASSWORD: container.getPassword(), + PGSSLMODE: 'disable', + }, + stdio: 'pipe', + }, + ); + } catch (error) { + await container.stop(); + throw error; } } return container; From 5840e68614d0217289f7920ff0b11da8c60ff9a5 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 28 Oct 2026 20:25:56 +0100 Subject: [PATCH 2/5] Port #37's workflow changes to the bench workflows #37 changed the standalone repository's schema-deploy.yml and web-ci.yml, which became bench-schema-deploy.yml and bench-ci.yml in this repository. - bench-schema-deploy prepares the migration interpreter in its own step, so a dependency failure can no longer pass as dry-run drift. A dry run now treats exit 1 as drift and any other non-zero exit as a failure. - bench-ci installs uv and runs the migration-runner tests. The web integration suite now also calls the runner through uv. - uv resolves the runner's Python dependencies when it runs, so both workflows set UV_EXCLUDE_NEWER to 14 days ago, matching the repository's cooldown. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Will Manning --- .github/workflows/bench-ci.yml | 16 +++++++++ .github/workflows/bench-schema-deploy.yml | 36 ++++++++++++++------- bench/docs/architecture/deploy-and-infra.md | 3 +- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/.github/workflows/bench-ci.yml b/.github/workflows/bench-ci.yml index a0fa9b2d..ce895fa9 100644 --- a/.github/workflows/bench-ci.yml +++ b/.github/workflows/bench-ci.yml @@ -95,5 +95,21 @@ jobs: - name: Require Docker for the integration suite run: docker info > /dev/null + # The migration-runner tests and the web integration suite both run + # bench/scripts/migrate-schema.py through uv. + - name: Install uv + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + enable-cache: false + + # uv resolves the runner's Python dependencies when it runs. Give them the + # repository's 14-day cooldown (see CLAUDE.md "Supply chain hardening"). + - name: Set the uv cooldown + run: echo "UV_EXCLUDE_NEWER=$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_ENV}" + + - name: Test migration runner + working-directory: bench + run: uv run --no-project --with 'psycopg[binary]>=3.2' python -m unittest discover -s scripts/tests -v + - name: Test run: pnpm test diff --git a/.github/workflows/bench-schema-deploy.yml b/.github/workflows/bench-schema-deploy.yml index 3a0665bf..e759addf 100644 --- a/.github/workflows/bench-schema-deploy.yml +++ b/.github/workflows/bench-schema-deploy.yml @@ -57,6 +57,18 @@ jobs: with: enable-cache: false + # Resolve dependencies before interpreting the runner's exit status. A uv + # download or environment failure must fail even during a dry run. The + # resolution gets the repository's 14-day cooldown, since this job holds + # the migrator role's credentials. + - name: Prepare migration interpreter + run: | + set -Eeuo pipefail + export UV_EXCLUDE_NEWER="$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ)" + uv venv "${RUNNER_TEMP}/migration-venv" + uv export --script scripts/migrate-schema.py --no-hashes > "${RUNNER_TEMP}/migration-requirements.txt" + uv pip install --python "${RUNNER_TEMP}/migration-venv/bin/python" -r "${RUNNER_TEMP}/migration-requirements.txt" + - name: Configure AWS credentials via OIDC uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: @@ -96,17 +108,17 @@ jobs: export PGSSLROOTCERT="${RUNNER_TEMP}/rds-global-bundle.pem" if [ "${DRY_RUN}" = "true" ]; then echo "dry_run: reporting status only, applying nothing" - # `status` exits 1 on drift (pending/orphaned migrations). For a dry_run that - # is the EXPECTED, informational result — not an infra error — so don't let it - # fail the step under `set -e`. (The `if` condition exempts the command from - # set -e.) The non-dry_run branch below intentionally lets a post-apply `status` - # non-zero fail the step: drift after an apply IS a real problem. - if uv run --no-project scripts/migrate-schema.py status; then - echo "dry_run: no drift — migration set matches the ledger." - else - echo "dry_run: drift detected (pending or orphaned migrations) — re-run without dry_run to apply." - fi + # `status` exits 1 on drift (pending or orphaned migrations), which a dry + # run reports without failing. Any other non-zero exit is an operational + # failure (connection, permission, filesystem) and fails the step. + status=0 + "${RUNNER_TEMP}/migration-venv/bin/python" scripts/migrate-schema.py status || status=$? + case "$status" in + 0) echo "dry_run: migration set matches the ledger." ;; + 1) echo "dry_run: pending or orphaned migrations detected." ;; + *) echo "::error::Migration status failed (exit $status)."; exit "$status" ;; + esac else - uv run --no-project scripts/migrate-schema.py apply - uv run --no-project scripts/migrate-schema.py status + "${RUNNER_TEMP}/migration-venv/bin/python" scripts/migrate-schema.py apply + "${RUNNER_TEMP}/migration-venv/bin/python" scripts/migrate-schema.py status fi diff --git a/bench/docs/architecture/deploy-and-infra.md b/bench/docs/architecture/deploy-and-infra.md index b849abf7..c8988aad 100644 --- a/bench/docs/architecture/deploy-and-infra.md +++ b/bench/docs/architecture/deploy-and-infra.md @@ -81,7 +81,8 @@ scripts/migrate-schema.py ──IAM token, verify-full TLS──▶ RDS as `mi role trusts only `main` of `vortex-data/vortex.dev`. Auto-applying on push to `main` is left as an operator's-call follow-up. - `dry_run: true` runs `status` only: it reports drift and exits 0 (drift is - informational in dry-run); a real apply still fails on post-apply drift. + informational in dry-run), but any other failure, such as a connection or + permission error, fails the run. A real apply still fails on post-apply drift. - Serialized concurrency (`cancel-in-progress: false`) prevents two operators racing an apply against the same database. - `migrate-schema.py` is a self-contained PEP 723 script run via `uv run From 80c2936f70006ab126f8e729c88ca953a6ee1a0f Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Tue, 8 Sep 2026 11:45:32 -0400 Subject: [PATCH 3/5] Gate web deployments on checks for the exact deployed commit Signed-off-by: Connor Tsui Signed-off-by: Will Manning --- bench/docs/architecture/deploy-and-infra.md | 35 ++++- bench/docs/runbooks/deploy-secrets-setup.md | 4 +- .../scripts/tests/verify-web-deploy.test.mjs | 67 ++++++++++ bench/scripts/verify-web-deploy.mjs | 121 ++++++++++++++++++ bench/web/README.md | 32 +++-- bench/web/next.config.js | 5 + 6 files changed, 251 insertions(+), 13 deletions(-) create mode 100644 bench/scripts/tests/verify-web-deploy.test.mjs create mode 100644 bench/scripts/verify-web-deploy.mjs diff --git a/bench/docs/architecture/deploy-and-infra.md b/bench/docs/architecture/deploy-and-infra.md index c8988aad..5213390b 100644 --- a/bench/docs/architecture/deploy-and-infra.md +++ b/bench/docs/architecture/deploy-and-infra.md @@ -93,13 +93,20 @@ scripts/migrate-schema.py ──IAM token, verify-full TLS──▶ RDS as `mi ``` push to main ──▶ production deploy -workflow_dispatch ──▶ preview or production (input) +workflow_dispatch ──▶ preview or production (production from main only) +same-repo PR ──▶ preview (event merge commit) │ ▼ -runner: pnpm install + `vercel build [--prod]` (builds web/ ON THE RUNNER) +shared checks on exact SHA: format + lint + build + test │ ▼ -`vercel deploy --prebuilt [--prod]` (uploads the prebuilt output) +runner: `vercel build [--prod]` on the same SHA + │ + ▼ +`vercel deploy --prebuilt [--prod]` + │ + ▼ +verify build SHA + read paths + CDN HIT (bounded retries) ``` - The Vercel project `benchmarks-website` is keyed by `VERCEL_ORG_ID` + @@ -113,9 +120,31 @@ runner: pnpm install + `vercel build [--prod]` (builds web/ ON THE RUNNER) - `main` **is** production — there is no staging gate. The per-PR CI (including a testcontainer migration test) is the gate; a merge to `main` ships straight to the v4 production domain. +- Push and manual production runs share one concurrency group without cancelling an active run. + This serializes promotion and verification across both triggers. Previews can cancel stale runs. +- Production verification uses `BENCH_SITE_BASE_URL`. A protected preview reports verification + as blocked. A production authentication error or failed probe fails the workflow. - A `bench-keep-warm` scheduled workflow pings the production deployment so the RDS connection pool and Data Cache stay warm between visits. +### Required merge check + +After the first standalone Web CI run completes, configure the branch protection rule or ruleset +for `develop` to require `format, lint, build, test`, with **GitHub Actions** as its source +(app ID `15368`). Enable **Require branches to be up to date before merging**. These are operator +settings. The workflows do not change repository protection. + +From a checkout of that tested commit, verify the context and integration before saving the rule: + +```bash +checked_sha="$(git rev-parse HEAD)" +gh api "repos/vortex-data/benchmarks-website/commits/${checked_sha}/check-runs" \ + --jq '.check_runs[] | select(.name == "format, lint, build, test") | {name, conclusion, app_id: .app.id}' +``` + +The completed check must report `success` and app ID `15368`. The shared composite preserves +this standalone check name when its internal steps change. + ## Legacy v3 host deploy (`ops/`) — decommissioned 2026-07-08 The v3 Rust server ran on an EC2 host under systemd, deployed by a **polling** diff --git a/bench/docs/runbooks/deploy-secrets-setup.md b/bench/docs/runbooks/deploy-secrets-setup.md index fd41abdd..cf57fa84 100644 --- a/bench/docs/runbooks/deploy-secrets-setup.md +++ b/bench/docs/runbooks/deploy-secrets-setup.md @@ -224,7 +224,7 @@ variable. | `VERCEL_TOKEN` (secret) | `web-deploy.yml` | | `VERCEL_ORG_ID` | `web-deploy.yml` | | `VERCEL_PROJECT_ID` | `web-deploy.yml` | -| `BENCH_SITE_BASE_URL` | `web-keep-warm.yml` | +| `BENCH_SITE_BASE_URL` | `web-keep-warm.yml`, production verification in `web-deploy.yml` | | `GH_BENCH_SCHEMA_ROLE_ARN` | `schema-deploy.yml` | | `RDS_BENCH_REGION` | `schema-deploy.yml` | | `RDS_BENCH_INSTANCE_ENDPOINT` | `schema-deploy.yml` | @@ -355,6 +355,8 @@ export VERCEL_ORG_ID="" # the value you set as t export VERCEL_PROJECT_ID="" read -rs VERCEL_TOKEN && export VERCEL_TOKEN # paste at the prompt; never echoed, never recorded in shell history # Run from the REPO ROOT (not web/) to match the workflow's project-resolution path: +BENCH_BUILD_SHA="$(git rev-parse HEAD)" +export BENCH_BUILD_SHA vercel pull --yes --environment=preview --token="$VERCEL_TOKEN" && vercel build --token="$VERCEL_TOKEN" unset VERCEL_TOKEN # drop the secret from the environment when done ``` diff --git a/bench/scripts/tests/verify-web-deploy.test.mjs b/bench/scripts/tests/verify-web-deploy.test.mjs new file mode 100644 index 00000000..d1f52e1b --- /dev/null +++ b/bench/scripts/tests/verify-web-deploy.test.mjs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { verifyDeployment } from '../verify-web-deploy.mjs'; + +const SHA = 'a'.repeat(40); +const chart = { commits: [{ sha: 'b'.repeat(40) }], series: { vortex: [1] } }; +function fixture(overrides = {}) { + const hits = new Map(); + return async (url) => { + const path = url.pathname; + hits.set(path, (hits.get(path) ?? 0) + 1); + if (overrides[path]) return overrides[path](); + if (path === '/api/health') return Response.json({ status: 'ok', build_sha: SHA, schema_version: 1 }); + if (path === '/api/groups') return Response.json({ groups: [{ slug: 'group', charts: [{ slug: 'chart' }] }] }); + if (path === '/api/group/group') return Response.json({ charts: [chart] }); + if (path === '/api/chart/chart') return Response.json(chart, { headers: { 'x-vercel-cache': hits.get(path) > 1 ? 'HIT' : 'MISS' } }); + return new Response('benchmarks', { + headers: { 'content-type': 'text/html', 'x-vercel-cache': hits.get(path) > 1 ? 'HIT' : 'MISS' }, + }); + }; +} +const noSleep = async () => {}; + +test('verifies a matching build and populated read paths after CDN warmup', async () => { + assert.equal(await verifyDeployment('https://bench.test', SHA, 'production', fixture(), noSleep), 'verified'); +}); +test('rejects an older healthy deployment', async () => { + await assert.rejects(verifyDeployment('https://bench.test', 'c'.repeat(40), 'production', fixture(), noSleep), /different build/); +}); +test('reports protected previews without claiming verification; production fails', async () => { + const fetcher = fixture({ '/api/health': () => new Response('', { status: 401 }) }); + assert.equal(await verifyDeployment('https://bench.test', SHA, 'preview', fetcher, noSleep), 'protected'); + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', fetcher, noSleep), /production route/); +}); +test('fails on empty group data and a CDN that never hits', async () => { + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', fixture({ + '/api/groups': () => Response.json({ groups: [] }), + }), noSleep), /populated groups/); + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', fixture({ + '/api/chart/chart': () => Response.json(chart, { headers: { 'x-vercel-cache': 'MISS' } }), + }), noSleep), /CDN did not report HIT/); +}); + +test('accepts a global commit window with no measurement for the selected chart', async () => { + assert.equal(await verifyDeployment('https://bench.test', SHA, 'production', fixture({ + '/api/group/group': () => Response.json({ charts: [{ commits: [{ sha: SHA }], series: { vortex: [null] } }] }), + }), noSleep), 'verified'); +}); + +test('rejects a successful HTTP response containing only an error shell', async () => { + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', fixture({ + '/': () => new Response('Error', { headers: { 'content-type': 'text/html' } }), + }), noSleep), /rendered page content is missing/); +}); + +test('recognizes Vercel SSO redirects only as protected previews', async () => { + const redirect = (location) => fixture({ + '/api/health': () => new Response(null, { status: 302, headers: { location } }), + }); + const sso = redirect('https://vercel.com/sso-api?url=https%3A%2F%2Fbench.test'); + assert.equal(await verifyDeployment('https://bench.test', SHA, 'preview', sso, noSleep), 'protected'); + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', sso, noSleep), /production route/); + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'preview', redirect('https://example.com/login'), noSleep), /HTTP 302/); +}); diff --git a/bench/scripts/verify-web-deploy.mjs b/bench/scripts/verify-web-deploy.mjs new file mode 100644 index 00000000..3c1d16cf --- /dev/null +++ b/bench/scripts/verify-web-deploy.mjs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import assert from 'node:assert/strict'; +import { appendFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; +import { setTimeout } from 'node:timers/promises'; + +/** Verify the public read paths within a three-minute deadline. */ +export async function verifyDeployment(base, sha, environment, fetchImpl = fetch, sleep = setTimeout) { + assert.match(sha, /^[0-9a-f]{40}$/); + assert.ok(['preview', 'production'].includes(environment)); + const origin = new URL(base); + assert.equal(origin.protocol, 'https:'); + assert.equal(origin.username + origin.password + origin.search + origin.hash, ''); + const deadline = Date.now() + 180_000; + + async function request(path, validate) { + let lastError; + for (let attempt = 0; attempt < 4; attempt++) { + const remaining = deadline - Date.now(); + assert.ok(remaining > 0, 'Deployment verification deadline exceeded'); + try { + const response = await fetchImpl(new URL(path, origin), { + redirect: 'manual', + signal: AbortSignal.timeout(Math.min(20_000, remaining)), + }); + const location = response.headers.get('location'); + const redirect = location && [301, 302, 303, 307, 308].includes(response.status) + ? new URL(location, origin) : null; + const vercelSso = redirect?.origin === 'https://vercel.com' && redirect.pathname === '/sso-api'; + if ([401, 403].includes(response.status) || vercelSso) { + if (environment === 'preview') return 'protected'; + throw new Error(`Public production route ${path} returned ${response.status}`); + } + assert.equal(response.status, 200, `${path}: HTTP ${response.status}`); + return await validate(response); + } catch (error) { + lastError = error; + if (attempt < 3) await sleep(Math.min(2_000, Math.max(0, deadline - Date.now()))); + } + } + throw lastError; + } + + async function json(path, validate) { + return request(path, async (response) => { + assert.match(response.headers.get('content-type') ?? '', /application\/json/); + const body = await response.json(); + validate(body); + return body; + }); + } + + function chart(body) { + assert.ok(Array.isArray(body.commits), 'Chart commits must be an array'); + assert.ok(body.series && typeof body.series === 'object' && !Array.isArray(body.series), + 'Chart series must be an object'); + for (const values of Object.values(body.series)) { + assert.ok(Array.isArray(values) && values.length === body.commits.length && + values.every((value) => value === null || (typeof value === 'number' && Number.isFinite(value))), + 'Chart series must align with its commit window'); + } + } + + const health = await json('/api/health', (body) => { + assert.equal(body.status, 'ok'); + assert.equal(body.build_sha, sha, 'Health reports a different build'); + assert.ok(Number.isInteger(body.schema_version)); + }); + if (health === 'protected') return 'protected'; + const groups = await json('/api/groups', (body) => { + assert.ok(Array.isArray(body.groups) && body.groups.some((group) => group.charts?.length), + 'No populated groups'); + }); + if (groups === 'protected') return 'protected'; + const group = groups.groups.find((item) => item.charts?.length); + const groupPath = `/api/group/${encodeURIComponent(group.slug)}?n=1`; + const groupBody = await json(groupPath, (body) => { + assert.ok(Array.isArray(body.charts) && body.charts.length > 0); + chart(body.charts[0]); + }); + if (groupBody === 'protected') return 'protected'; + const chartSlug = encodeURIComponent(group.charts[0].slug); + const chartBody = await json(`/api/chart/${chartSlug}`, chart); + if (chartBody === 'protected') return 'protected'; + for (const path of ['/', `/chart/${chartSlug}`]) { + const result = await request(path, async (response) => { + assert.match(response.headers.get('content-type') ?? '', /text\/html/); + const html = await response.text(); + assert.ok(html.includes(path === '/' + ? 'data-role="global-filter-bar"' + : `data-chart-slug="${chartSlug}"`), `${path}: rendered page content is missing`); + }); + if (result === 'protected') return 'protected'; + } + // Repeat the same URLs, without a cache-busting query, to verify CDN reuse. + for (const path of ['/', `/api/chart/${chartSlug}`]) { + const result = await request(path, async (response) => { + await response.arrayBuffer(); + assert.equal(response.headers.get('x-vercel-cache'), 'HIT', `${path}: CDN did not report HIT`); + }); + if (result === 'protected') return 'protected'; + } + return 'verified'; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const result = await verifyDeployment(...process.argv.slice(2)); + const message = result === 'protected' + ? 'Deployment created; preview protection blocked verification. Health, build identity, and CDN caching remain unverified.' + : 'Verified build SHA, health, representative group/chart JSON, HTML pages, and CDN HIT.'; + console.log(`${result === 'protected' ? '::notice::' : ''}${message}`); + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `result=${result}\n`); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${message}\n`); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/bench/web/README.md b/bench/web/README.md index 8a1078be..289a498e 100644 --- a/bench/web/README.md +++ b/bench/web/README.md @@ -54,10 +54,14 @@ warmth: on those routes: that header is consumed (and stripped) by Vercel's CDN alone at the highest precedence, so the CDN caches the rendered pages while browsers still revalidate every load. -Verify on a live deployment with `curl -sI | grep -i x-vercel-cache` (expect `MISS` then -`HIT` within the five-minute fresh window). No workflow runs this check, so run it by hand after a -deploy that changes caching. Deployment URLs sit behind Vercel Authentication, so check a preview -through `vercel curl` and production on the public domain. One deliberate +The deploy workflow verifies the expected build SHA through `/api/health`, a representative group +and chart JSON response, the landing and chart pages, and a CDN `HIT` on repeated landing/chart-API +requests. The probe has a three-minute deadline. Production uses the public `BENCH_SITE_BASE_URL` +and fails verification on authentication errors or redirects. A protected preview reports +verification as blocked, with no claim that its health, build identity, or caching passed. To check +a protected preview by hand, use `vercel curl`. + +One deliberate divergence from the API routes: the `vercel.json` header rules apply to every response status, so an unknown `/chart/:slug` 404 follows the HTML page rule: five minutes fresh and then eligible for Vercel's one-day stale-while-revalidate window. That is acceptable for opaque, never-linked @@ -68,10 +72,17 @@ were, the same HTML page rule would bound it). ## Deploys `.github/workflows/bench-ci.yml` runs the check suite on every PR touching `bench/`, including -`migrations/**` (the integration suite applies that DDL to its testcontainer). -`.github/workflows/bench-deploy.yml` deploys via the Vercel CLI (`vercel pull` / `vercel build` / -`vercel deploy --prebuilt`): a preview deployment per same-repo PR that touches `bench/`, and a -production deployment on each push to `main` that does. +`migrations/**` (the integration suite applies that DDL to its testcontainer). It is a required +check on `main`. `.github/workflows/bench-deploy.yml` deploys via the Vercel CLI (`vercel pull` / +`vercel build` / `vercel deploy --prebuilt`): a preview deployment per same-repo PR that touches +`bench/`, and a production deployment on each push to `main` that does. Fork PRs run CI but do +not receive deployment credentials. + +The `bench-production` environment limits manual production runs to `main`. Push and manual +production runs share one production queue and let an active deploy finish verification. New +preview runs can cancel older previews of the same event and ref. The Vercel CLI builds on the +runner and uploads the prebuilt output. `BENCH_BUILD_SHA` is embedded in that output so the +uncached health endpoint identifies the checked-out build. One-time operator setup: @@ -81,7 +92,10 @@ One-time operator setup: 2. Set the repo variables `VERCEL_ORG_ID` + `BENCH_VERCEL_PROJECT_ID` (from the Vercel project settings), and the secret `VERCEL_TOKEN` (a deploy token scoped to this Vercel project) in both the `bench-preview` and `bench-production` GitHub environments. Limit - `bench-production` to `main`. + `bench-production` to `main`. Set the repo variable `BENCH_SITE_BASE_URL` to the public + production URL as `https://`. The keep-warm workflow uses the same variable. + Production deployment requires it before uploading and verifies the public domain after + deployment. 3. Configure `BENCH_DB_*` on the Vercel project (Production and Preview environments). Two open wiring choices are deliberately left to this step, per environment: - **Endpoint**: the public RDS instance endpoint (the same endpoint CI writers use). There is diff --git a/bench/web/next.config.js b/bench/web/next.config.js index e40cc786..b08a72ba 100644 --- a/bench/web/next.config.js +++ b/bench/web/next.config.js @@ -7,6 +7,11 @@ const nextConfig = { // (workspace root, the v2 benchmarks-website project, and this app), so // Next.js cannot infer the correct root on its own. outputFileTracingRoot: __dirname, + // Embed the checked-out commit in prebuilt output, independent of runtime settings. + env: { + VERCEL_GIT_COMMIT_SHA: + process.env.BENCH_BUILD_SHA ?? process.env.VERCEL_GIT_COMMIT_SHA ?? 'unknown', + }, }; module.exports = nextConfig; From 450275c056aaf3917464e9a0a049700570342b56 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 28 Oct 2026 20:29:32 +0100 Subject: [PATCH 4/5] Port #38's deploy verification to the bench workflows #38 changed the standalone repository's web-deploy.yml and added a shared web-checks action. In this repository: - Push and manual production runs share one concurrency group and never cancel an active run. A newer preview can cancel an older one. - A production deploy requires BENCH_SITE_BASE_URL to be an HTTPS URL. - The build embeds BENCH_BUILD_SHA. After the deploy, verify-web-deploy.mjs checks the build commit, the read paths and CDN caching. - bench-ci runs the script's tests. The web-checks action and the pre-deploy check job are left out: bench CI is a required check on main, so merged code has already passed it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Will Manning --- .github/workflows/bench-ci.yml | 4 ++++ .github/workflows/bench-deploy.yml | 38 ++++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench-ci.yml b/.github/workflows/bench-ci.yml index ce895fa9..913b0ccf 100644 --- a/.github/workflows/bench-ci.yml +++ b/.github/workflows/bench-ci.yml @@ -111,5 +111,9 @@ jobs: working-directory: bench run: uv run --no-project --with 'psycopg[binary]>=3.2' python -m unittest discover -s scripts/tests -v + - name: Test deployment verification + working-directory: bench + run: node --test scripts/tests/verify-web-deploy.test.mjs + - name: Test run: pnpm test diff --git a/.github/workflows/bench-deploy.yml b/.github/workflows/bench-deploy.yml index 1bc5e308..7f2e9245 100644 --- a/.github/workflows/bench-deploy.yml +++ b/.github/workflows/bench-deploy.yml @@ -42,11 +42,11 @@ on: default: preview concurrency: - # Include the event name so a manual workflow_dispatch (e.g. a preview deploy) on - # main does NOT land in the same group as — and thus cancel — an in-flight - # push-triggered production deploy on the same ref. - group: bench-deploy-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: true + # Push and manual production runs share one queue and never cancel an active + # run, so each promotion finishes its verification before the next starts. A + # newer preview run can cancel an older preview of the same event and ref. + group: ${{ (github.event_name == 'push' || inputs.environment == 'production') && 'bench-deploy-production' || format('bench-deploy-preview-{0}-{1}', github.event_name, github.ref) }} + cancel-in-progress: ${{ github.event_name != 'push' && inputs.environment != 'production' }} permissions: contents: read @@ -116,6 +116,14 @@ jobs: --before="$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ)" \ "vercel@${VERCEL_CLI_VERSION}" + # Production verification probes the public domain, so it has to be set. + - name: Require a public production verification URL + if: env.TARGET == 'production' + env: + PROD_URL: ${{ vars.BENCH_SITE_BASE_URL }} + run: | + [[ "${PROD_URL}" == https://* ]] || { echo "::error::Set BENCH_SITE_BASE_URL to the public HTTPS production URL."; exit 1; } + - name: Pull Vercel project settings env: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} @@ -124,6 +132,8 @@ jobs: - name: Build env: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + # next.config.js embeds this, so /api/health reports the deployed commit. + BENCH_BUILD_SHA: ${{ github.sha }} run: | set -Eeuo pipefail if [ "${TARGET}" = "production" ]; then @@ -155,3 +165,21 @@ jobs: with: header: bench-preview message: "Bench preview for `${{ github.event.pull_request.head.sha }}`: ${{ steps.deploy.outputs.url }}" + + # Check the build commit, the read paths and CDN caching. Production is + # probed on the public domain, and any failure fails the run. Previews sit + # behind Vercel Authentication, so their verification reports as blocked. + - name: Verify deployed build and read paths + id: verify + timeout-minutes: 4 + env: + DEPLOY_URL: ${{ steps.deploy.outputs.url }} + PROD_URL: ${{ vars.BENCH_SITE_BASE_URL }} + EXPECTED_SHA: ${{ github.sha }} + run: | + set -Eeuo pipefail + if [ "${TARGET}" = "production" ]; then + node bench/scripts/verify-web-deploy.mjs "${PROD_URL}" "${EXPECTED_SHA}" production + else + node bench/scripts/verify-web-deploy.mjs "${DEPLOY_URL}" "${EXPECTED_SHA}" preview + fi From afa06b97068196b58fccb99264fe0b56138e2739 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 28 Oct 2026 20:37:33 +0100 Subject: [PATCH 5/5] Check the exact deployed commit before a bench deploy Codex pointed out that the port dropped #38's pre-deploy checks. Required PR checks do not cover the deployed commit: main takes squash merges without requiring an up-to-date branch, and Bench CI on main runs beside bench-deploy, not before it. So a merged commit could ship unchecked. Restore Connor's design. Bench CI's steps move into the bench-checks composite action. Bench CI runs it, and bench-deploy runs it in a checks job on the commit it deploys, with the deploy job needing that job. The required merge check docs pointed at develop, the old check name and the old repository. They now describe vortex.dev's main ruleset and the deploy-time check. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Will Manning --- .github/actions/bench-checks/action.yml | 95 +++++++++++++++++++++ .github/workflows/bench-ci.yml | 67 +-------------- .github/workflows/bench-deploy.yml | 27 ++++++ bench/docs/architecture/deploy-and-infra.md | 21 ++--- bench/web/README.md | 5 +- 5 files changed, 140 insertions(+), 75 deletions(-) create mode 100644 .github/actions/bench-checks/action.yml diff --git a/.github/actions/bench-checks/action.yml b/.github/actions/bench-checks/action.yml new file mode 100644 index 00000000..875ce67e --- /dev/null +++ b/.github/actions/bench-checks/action.yml @@ -0,0 +1,95 @@ +name: Bench checks +description: >- + bench's full check suite: infrastructure trust policies, format, lint, build, + the migration-runner and deployment-verification tests, and the web tests with + their Postgres integration suite. Bench CI runs it on every change to bench, + and bench-deploy runs it on the exact commit it deploys. Run it after a + checkout. + +inputs: + pnpm-version: + description: Keep in lockstep with `packageManager` in bench/web/package.json. + required: true + +runs: + using: composite + steps: + - name: Check infrastructure trust policies offline + shell: bash + working-directory: bench + run: | + bash -n infra/provision.sh + python3 -m unittest discover -s infra/tests -v + + # pnpm must exist before setup-node so its `cache: pnpm` can resolve the store. + - name: Install pnpm + shell: bash + env: + PNPM_VERSION: ${{ inputs.pnpm-version }} + run: npm install -g "pnpm@${PNPM_VERSION}" + + - name: Setup Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + # The repository-wide Node pin (see CLAUDE.md "Node version"). Keep + # @types/node in bench/web/package.json on the same major. + node-version-file: .nvmrc + cache: pnpm + cache-dependency-path: bench/web/pnpm-lock.yaml + + - name: Install dependencies + shell: bash + working-directory: bench/web + run: pnpm install --frozen-lockfile + + - name: Format check + shell: bash + working-directory: bench/web + run: pnpm format:check + + - name: Lint + shell: bash + working-directory: bench/web + run: pnpm lint + + # `next build` must succeed without a reachable database (every page and route + # is request-rendered), so no BENCH_DB_* env vars are provided here on purpose. + - name: Build + shell: bash + working-directory: bench/web + run: pnpm build + + # testcontainers self-skips the Postgres integration suite when no Docker daemon + # is available (a local-dev convenience); in CI that would silently drop the only + # Postgres integration coverage, so fail loudly if the daemon is missing. + - name: Require Docker for the integration suite + shell: bash + run: docker info > /dev/null + + # The migration-runner tests and the web integration suite both run + # bench/scripts/migrate-schema.py through uv. + - name: Install uv + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + enable-cache: false + + # uv resolves the runner's Python dependencies when it runs. Give them the + # repository's 14-day cooldown (see CLAUDE.md "Supply chain hardening"). + - name: Set the uv cooldown + shell: bash + run: echo "UV_EXCLUDE_NEWER=$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_ENV}" + + - name: Test migration runner + shell: bash + working-directory: bench + run: uv run --no-project --with 'psycopg[binary]>=3.2' python -m unittest discover -s scripts/tests -v + + - name: Test deployment verification + shell: bash + working-directory: bench + run: node --test scripts/tests/verify-web-deploy.test.mjs + + - name: Test + shell: bash + working-directory: bench/web + run: pnpm test diff --git a/.github/workflows/bench-ci.yml b/.github/workflows/bench-ci.yml index 913b0ccf..57b71ac4 100644 --- a/.github/workflows/bench-ci.yml +++ b/.github/workflows/bench-ci.yml @@ -47,73 +47,14 @@ jobs: if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.bench == 'true') }} runs-on: ubuntu-24.04 timeout-minutes: 30 - defaults: - run: - working-directory: bench/web steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Check infrastructure trust policies offline - working-directory: bench - run: | - bash -n infra/provision.sh - python3 -m unittest discover -s infra/tests -v - - # pnpm must exist before setup-node so its `cache: pnpm` can resolve the store. - - name: Install pnpm - run: npm install -g "pnpm@${PNPM_VERSION}" - - - name: Setup Node - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + # The same checks bench-deploy runs on the commit it deploys. + - name: Bench checks + uses: ./.github/actions/bench-checks with: - # The repository-wide Node pin (see CLAUDE.md "Node version"). Keep - # @types/node in bench/web/package.json on the same major. - node-version-file: .nvmrc - cache: pnpm - cache-dependency-path: bench/web/pnpm-lock.yaml - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Format check - run: pnpm format:check - - - name: Lint - run: pnpm lint - - # `next build` must succeed without a reachable database (every page and route - # is request-rendered), so no BENCH_DB_* env vars are provided here on purpose. - - name: Build - run: pnpm build - - # testcontainers self-skips the Postgres integration suite when no Docker daemon - # is available (a local-dev convenience); in CI that would silently drop the only - # Postgres integration coverage, so fail loudly if the daemon is missing. - - name: Require Docker for the integration suite - run: docker info > /dev/null - - # The migration-runner tests and the web integration suite both run - # bench/scripts/migrate-schema.py through uv. - - name: Install uv - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - with: - enable-cache: false - - # uv resolves the runner's Python dependencies when it runs. Give them the - # repository's 14-day cooldown (see CLAUDE.md "Supply chain hardening"). - - name: Set the uv cooldown - run: echo "UV_EXCLUDE_NEWER=$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_ENV}" - - - name: Test migration runner - working-directory: bench - run: uv run --no-project --with 'psycopg[binary]>=3.2' python -m unittest discover -s scripts/tests -v - - - name: Test deployment verification - working-directory: bench - run: node --test scripts/tests/verify-web-deploy.test.mjs - - - name: Test - run: pnpm test + pnpm-version: ${{ env.PNPM_VERSION }} diff --git a/.github/workflows/bench-deploy.yml b/.github/workflows/bench-deploy.yml index 7f2e9245..9d07b76f 100644 --- a/.github/workflows/bench-deploy.yml +++ b/.github/workflows/bench-deploy.yml @@ -64,8 +64,35 @@ env: VERCEL_CLI_VERSION: "64.0.2" jobs: + # Run bench's checks on the exact commit this run deploys. main takes squash + # merges and does not require a branch to be up to date before it merges, so + # the pushed commit itself is never checked before it lands, and Bench CI on + # main runs beside this workflow, not before it. + checks: + name: Check the deployed commit + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + if: >- + vars.BENCH_VERCEL_PROJECT_ID != '' && ( + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + ) + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Bench checks + uses: ./.github/actions/bench-checks + with: + pnpm-version: ${{ env.PNPM_VERSION }} + deploy: name: Build & deploy to Vercel + needs: checks runs-on: ubuntu-24.04 timeout-minutes: 20 # Fork PRs have no access to VERCEL_TOKEN, and we do not auto-deploy untrusted diff --git a/bench/docs/architecture/deploy-and-infra.md b/bench/docs/architecture/deploy-and-infra.md index 5213390b..094b28ff 100644 --- a/bench/docs/architecture/deploy-and-infra.md +++ b/bench/docs/architecture/deploy-and-infra.md @@ -119,7 +119,7 @@ verify build SHA + read paths + CDN HIT (bounded retries) prebuilt output is then uploaded, eliminating a build race. - `main` **is** production — there is no staging gate. The per-PR CI (including a testcontainer migration test) is the gate; a merge to `main` - ships straight to the v4 production domain. + ships to the v4 production domain after the deploy workflow checks the same commit. - Push and manual production runs share one concurrency group without cancelling an active run. This serializes promotion and verification across both triggers. Previews can cancel stale runs. - Production verification uses `BENCH_SITE_BASE_URL`. A protected preview reports verification @@ -129,21 +129,22 @@ verify build SHA + read paths + CDN HIT (bounded retries) ### Required merge check -After the first standalone Web CI run completes, configure the branch protection rule or ruleset -for `develop` to require `format, lint, build, test`, with **GitHub Actions** as its source -(app ID `15368`). Enable **Require branches to be up to date before merging**. These are operator -settings. The workflows do not change repository protection. +The `main` ruleset of `vortex-data/vortex.dev` requires `Bench: format, lint, build, test` from +**GitHub Actions** (app ID `15368`), beside the marketing site's checks. It does not require +branches to be up to date before merging, so a merged commit is never checked before it lands. +That is why `bench-deploy.yml` runs the same checks (the `bench-checks` composite action) on the +exact commit it deploys, and deploys only if they pass. -From a checkout of that tested commit, verify the context and integration before saving the rule: +To see the check for a commit: ```bash checked_sha="$(git rev-parse HEAD)" -gh api "repos/vortex-data/benchmarks-website/commits/${checked_sha}/check-runs" \ - --jq '.check_runs[] | select(.name == "format, lint, build, test") | {name, conclusion, app_id: .app.id}' +gh api "repos/vortex-data/vortex.dev/commits/${checked_sha}/check-runs" \ + --jq '.check_runs[] | select(.name == "Bench: format, lint, build, test") | {name, conclusion, app_id: .app.id}' ``` -The completed check must report `success` and app ID `15368`. The shared composite preserves -this standalone check name when its internal steps change. +A commit that changes bench must report `success` and app ID `15368`. The composite action keeps +the check name the same when its steps change. ## Legacy v3 host deploy (`ops/`) — decommissioned 2026-07-08 diff --git a/bench/web/README.md b/bench/web/README.md index 289a498e..75baf507 100644 --- a/bench/web/README.md +++ b/bench/web/README.md @@ -75,8 +75,9 @@ were, the same HTML page rule would bound it). `migrations/**` (the integration suite applies that DDL to its testcontainer). It is a required check on `main`. `.github/workflows/bench-deploy.yml` deploys via the Vercel CLI (`vercel pull` / `vercel build` / `vercel deploy --prebuilt`): a preview deployment per same-repo PR that touches -`bench/`, and a production deployment on each push to `main` that does. Fork PRs run CI but do -not receive deployment credentials. +`bench/`, and a production deployment on each push to `main` that does. Before it builds, the +deploy workflow runs the same checks (the `bench-checks` composite action) on the exact commit it +deploys. Fork PRs run CI but do not receive deployment credentials. The `bench-production` environment limits manual production runs to `main`. Push and manual production runs share one production queue and let an active deploy finish verification. New