diff --git a/.github/actions/bench-checks/action.yml b/.github/actions/bench-checks/action.yml new file mode 100644 index 00000000..875ce67e --- /dev/null +++ b/.github/actions/bench-checks/action.yml @@ -0,0 +1,95 @@ +name: Bench checks +description: >- + bench's full check suite: infrastructure trust policies, format, lint, build, + the migration-runner and deployment-verification tests, and the web tests with + their Postgres integration suite. Bench CI runs it on every change to bench, + and bench-deploy runs it on the exact commit it deploys. Run it after a + checkout. + +inputs: + pnpm-version: + description: Keep in lockstep with `packageManager` in bench/web/package.json. + required: true + +runs: + using: composite + steps: + - name: Check infrastructure trust policies offline + shell: bash + working-directory: bench + run: | + bash -n infra/provision.sh + python3 -m unittest discover -s infra/tests -v + + # pnpm must exist before setup-node so its `cache: pnpm` can resolve the store. + - name: Install pnpm + shell: bash + env: + PNPM_VERSION: ${{ inputs.pnpm-version }} + run: npm install -g "pnpm@${PNPM_VERSION}" + + - name: Setup Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + # The repository-wide Node pin (see CLAUDE.md "Node version"). Keep + # @types/node in bench/web/package.json on the same major. + node-version-file: .nvmrc + cache: pnpm + cache-dependency-path: bench/web/pnpm-lock.yaml + + - name: Install dependencies + shell: bash + working-directory: bench/web + run: pnpm install --frozen-lockfile + + - name: Format check + shell: bash + working-directory: bench/web + run: pnpm format:check + + - name: Lint + shell: bash + working-directory: bench/web + run: pnpm lint + + # `next build` must succeed without a reachable database (every page and route + # is request-rendered), so no BENCH_DB_* env vars are provided here on purpose. + - name: Build + shell: bash + working-directory: bench/web + run: pnpm build + + # testcontainers self-skips the Postgres integration suite when no Docker daemon + # is available (a local-dev convenience); in CI that would silently drop the only + # Postgres integration coverage, so fail loudly if the daemon is missing. + - name: Require Docker for the integration suite + shell: bash + run: docker info > /dev/null + + # The migration-runner tests and the web integration suite both run + # bench/scripts/migrate-schema.py through uv. + - name: Install uv + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + enable-cache: false + + # uv resolves the runner's Python dependencies when it runs. Give them the + # repository's 14-day cooldown (see CLAUDE.md "Supply chain hardening"). + - name: Set the uv cooldown + shell: bash + run: echo "UV_EXCLUDE_NEWER=$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_ENV}" + + - name: Test migration runner + shell: bash + working-directory: bench + run: uv run --no-project --with 'psycopg[binary]>=3.2' python -m unittest discover -s scripts/tests -v + + - name: Test deployment verification + shell: bash + working-directory: bench + run: node --test scripts/tests/verify-web-deploy.test.mjs + + - name: Test + shell: bash + working-directory: bench/web + run: pnpm test diff --git a/.github/workflows/bench-ci.yml b/.github/workflows/bench-ci.yml index ce895fa9..57b71ac4 100644 --- a/.github/workflows/bench-ci.yml +++ b/.github/workflows/bench-ci.yml @@ -47,69 +47,14 @@ jobs: if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.bench == 'true') }} runs-on: ubuntu-24.04 timeout-minutes: 30 - defaults: - run: - working-directory: bench/web steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Check infrastructure trust policies offline - working-directory: bench - run: | - bash -n infra/provision.sh - python3 -m unittest discover -s infra/tests -v - - # pnpm must exist before setup-node so its `cache: pnpm` can resolve the store. - - name: Install pnpm - run: npm install -g "pnpm@${PNPM_VERSION}" - - - name: Setup Node - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - # The repository-wide Node pin (see CLAUDE.md "Node version"). Keep - # @types/node in bench/web/package.json on the same major. - node-version-file: .nvmrc - cache: pnpm - cache-dependency-path: bench/web/pnpm-lock.yaml - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Format check - run: pnpm format:check - - - name: Lint - run: pnpm lint - - # `next build` must succeed without a reachable database (every page and route - # is request-rendered), so no BENCH_DB_* env vars are provided here on purpose. - - name: Build - run: pnpm build - - # testcontainers self-skips the Postgres integration suite when no Docker daemon - # is available (a local-dev convenience); in CI that would silently drop the only - # Postgres integration coverage, so fail loudly if the daemon is missing. - - name: Require Docker for the integration suite - run: docker info > /dev/null - - # The migration-runner tests and the web integration suite both run - # bench/scripts/migrate-schema.py through uv. - - name: Install uv - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + # The same checks bench-deploy runs on the commit it deploys. + - name: Bench checks + uses: ./.github/actions/bench-checks with: - enable-cache: false - - # uv resolves the runner's Python dependencies when it runs. Give them the - # repository's 14-day cooldown (see CLAUDE.md "Supply chain hardening"). - - name: Set the uv cooldown - run: echo "UV_EXCLUDE_NEWER=$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_ENV}" - - - name: Test migration runner - working-directory: bench - run: uv run --no-project --with 'psycopg[binary]>=3.2' python -m unittest discover -s scripts/tests -v - - - name: Test - run: pnpm test + pnpm-version: ${{ env.PNPM_VERSION }} diff --git a/.github/workflows/bench-deploy.yml b/.github/workflows/bench-deploy.yml index 1bc5e308..9d07b76f 100644 --- a/.github/workflows/bench-deploy.yml +++ b/.github/workflows/bench-deploy.yml @@ -42,11 +42,11 @@ on: default: preview concurrency: - # Include the event name so a manual workflow_dispatch (e.g. a preview deploy) on - # main does NOT land in the same group as — and thus cancel — an in-flight - # push-triggered production deploy on the same ref. - group: bench-deploy-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: true + # Push and manual production runs share one queue and never cancel an active + # run, so each promotion finishes its verification before the next starts. A + # newer preview run can cancel an older preview of the same event and ref. + group: ${{ (github.event_name == 'push' || inputs.environment == 'production') && 'bench-deploy-production' || format('bench-deploy-preview-{0}-{1}', github.event_name, github.ref) }} + cancel-in-progress: ${{ github.event_name != 'push' && inputs.environment != 'production' }} permissions: contents: read @@ -64,8 +64,35 @@ env: VERCEL_CLI_VERSION: "64.0.2" jobs: + # Run bench's checks on the exact commit this run deploys. main takes squash + # merges and does not require a branch to be up to date before it merges, so + # the pushed commit itself is never checked before it lands, and Bench CI on + # main runs beside this workflow, not before it. + checks: + name: Check the deployed commit + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + if: >- + vars.BENCH_VERCEL_PROJECT_ID != '' && ( + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + ) + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Bench checks + uses: ./.github/actions/bench-checks + with: + pnpm-version: ${{ env.PNPM_VERSION }} + deploy: name: Build & deploy to Vercel + needs: checks runs-on: ubuntu-24.04 timeout-minutes: 20 # Fork PRs have no access to VERCEL_TOKEN, and we do not auto-deploy untrusted @@ -116,6 +143,14 @@ jobs: --before="$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ)" \ "vercel@${VERCEL_CLI_VERSION}" + # Production verification probes the public domain, so it has to be set. + - name: Require a public production verification URL + if: env.TARGET == 'production' + env: + PROD_URL: ${{ vars.BENCH_SITE_BASE_URL }} + run: | + [[ "${PROD_URL}" == https://* ]] || { echo "::error::Set BENCH_SITE_BASE_URL to the public HTTPS production URL."; exit 1; } + - name: Pull Vercel project settings env: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} @@ -124,6 +159,8 @@ jobs: - name: Build env: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + # next.config.js embeds this, so /api/health reports the deployed commit. + BENCH_BUILD_SHA: ${{ github.sha }} run: | set -Eeuo pipefail if [ "${TARGET}" = "production" ]; then @@ -155,3 +192,21 @@ jobs: with: header: bench-preview message: "Bench preview for `${{ github.event.pull_request.head.sha }}`: ${{ steps.deploy.outputs.url }}" + + # Check the build commit, the read paths and CDN caching. Production is + # probed on the public domain, and any failure fails the run. Previews sit + # behind Vercel Authentication, so their verification reports as blocked. + - name: Verify deployed build and read paths + id: verify + timeout-minutes: 4 + env: + DEPLOY_URL: ${{ steps.deploy.outputs.url }} + PROD_URL: ${{ vars.BENCH_SITE_BASE_URL }} + EXPECTED_SHA: ${{ github.sha }} + run: | + set -Eeuo pipefail + if [ "${TARGET}" = "production" ]; then + node bench/scripts/verify-web-deploy.mjs "${PROD_URL}" "${EXPECTED_SHA}" production + else + node bench/scripts/verify-web-deploy.mjs "${DEPLOY_URL}" "${EXPECTED_SHA}" preview + fi diff --git a/bench/docs/architecture/deploy-and-infra.md b/bench/docs/architecture/deploy-and-infra.md index c8988aad..094b28ff 100644 --- a/bench/docs/architecture/deploy-and-infra.md +++ b/bench/docs/architecture/deploy-and-infra.md @@ -93,13 +93,20 @@ scripts/migrate-schema.py ──IAM token, verify-full TLS──▶ RDS as `mi ``` push to main ──▶ production deploy -workflow_dispatch ──▶ preview or production (input) +workflow_dispatch ──▶ preview or production (production from main only) +same-repo PR ──▶ preview (event merge commit) │ ▼ -runner: pnpm install + `vercel build [--prod]` (builds web/ ON THE RUNNER) +shared checks on exact SHA: format + lint + build + test │ ▼ -`vercel deploy --prebuilt [--prod]` (uploads the prebuilt output) +runner: `vercel build [--prod]` on the same SHA + │ + ▼ +`vercel deploy --prebuilt [--prod]` + │ + ▼ +verify build SHA + read paths + CDN HIT (bounded retries) ``` - The Vercel project `benchmarks-website` is keyed by `VERCEL_ORG_ID` + @@ -112,10 +119,33 @@ runner: pnpm install + `vercel build [--prod]` (builds web/ ON THE RUNNER) prebuilt output is then uploaded, eliminating a build race. - `main` **is** production — there is no staging gate. The per-PR CI (including a testcontainer migration test) is the gate; a merge to `main` - ships straight to the v4 production domain. + ships to the v4 production domain after the deploy workflow checks the same commit. +- Push and manual production runs share one concurrency group without cancelling an active run. + This serializes promotion and verification across both triggers. Previews can cancel stale runs. +- Production verification uses `BENCH_SITE_BASE_URL`. A protected preview reports verification + as blocked. A production authentication error or failed probe fails the workflow. - A `bench-keep-warm` scheduled workflow pings the production deployment so the RDS connection pool and Data Cache stay warm between visits. +### Required merge check + +The `main` ruleset of `vortex-data/vortex.dev` requires `Bench: format, lint, build, test` from +**GitHub Actions** (app ID `15368`), beside the marketing site's checks. It does not require +branches to be up to date before merging, so a merged commit is never checked before it lands. +That is why `bench-deploy.yml` runs the same checks (the `bench-checks` composite action) on the +exact commit it deploys, and deploys only if they pass. + +To see the check for a commit: + +```bash +checked_sha="$(git rev-parse HEAD)" +gh api "repos/vortex-data/vortex.dev/commits/${checked_sha}/check-runs" \ + --jq '.check_runs[] | select(.name == "Bench: format, lint, build, test") | {name, conclusion, app_id: .app.id}' +``` + +A commit that changes bench must report `success` and app ID `15368`. The composite action keeps +the check name the same when its steps change. + ## Legacy v3 host deploy (`ops/`) — decommissioned 2026-07-08 The v3 Rust server ran on an EC2 host under systemd, deployed by a **polling** diff --git a/bench/docs/runbooks/deploy-secrets-setup.md b/bench/docs/runbooks/deploy-secrets-setup.md index fd41abdd..cf57fa84 100644 --- a/bench/docs/runbooks/deploy-secrets-setup.md +++ b/bench/docs/runbooks/deploy-secrets-setup.md @@ -224,7 +224,7 @@ variable. | `VERCEL_TOKEN` (secret) | `web-deploy.yml` | | `VERCEL_ORG_ID` | `web-deploy.yml` | | `VERCEL_PROJECT_ID` | `web-deploy.yml` | -| `BENCH_SITE_BASE_URL` | `web-keep-warm.yml` | +| `BENCH_SITE_BASE_URL` | `web-keep-warm.yml`, production verification in `web-deploy.yml` | | `GH_BENCH_SCHEMA_ROLE_ARN` | `schema-deploy.yml` | | `RDS_BENCH_REGION` | `schema-deploy.yml` | | `RDS_BENCH_INSTANCE_ENDPOINT` | `schema-deploy.yml` | @@ -355,6 +355,8 @@ export VERCEL_ORG_ID="" # the value you set as t export VERCEL_PROJECT_ID="" read -rs VERCEL_TOKEN && export VERCEL_TOKEN # paste at the prompt; never echoed, never recorded in shell history # Run from the REPO ROOT (not web/) to match the workflow's project-resolution path: +BENCH_BUILD_SHA="$(git rev-parse HEAD)" +export BENCH_BUILD_SHA vercel pull --yes --environment=preview --token="$VERCEL_TOKEN" && vercel build --token="$VERCEL_TOKEN" unset VERCEL_TOKEN # drop the secret from the environment when done ``` diff --git a/bench/scripts/tests/verify-web-deploy.test.mjs b/bench/scripts/tests/verify-web-deploy.test.mjs new file mode 100644 index 00000000..d1f52e1b --- /dev/null +++ b/bench/scripts/tests/verify-web-deploy.test.mjs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { verifyDeployment } from '../verify-web-deploy.mjs'; + +const SHA = 'a'.repeat(40); +const chart = { commits: [{ sha: 'b'.repeat(40) }], series: { vortex: [1] } }; +function fixture(overrides = {}) { + const hits = new Map(); + return async (url) => { + const path = url.pathname; + hits.set(path, (hits.get(path) ?? 0) + 1); + if (overrides[path]) return overrides[path](); + if (path === '/api/health') return Response.json({ status: 'ok', build_sha: SHA, schema_version: 1 }); + if (path === '/api/groups') return Response.json({ groups: [{ slug: 'group', charts: [{ slug: 'chart' }] }] }); + if (path === '/api/group/group') return Response.json({ charts: [chart] }); + if (path === '/api/chart/chart') return Response.json(chart, { headers: { 'x-vercel-cache': hits.get(path) > 1 ? 'HIT' : 'MISS' } }); + return new Response('benchmarks', { + headers: { 'content-type': 'text/html', 'x-vercel-cache': hits.get(path) > 1 ? 'HIT' : 'MISS' }, + }); + }; +} +const noSleep = async () => {}; + +test('verifies a matching build and populated read paths after CDN warmup', async () => { + assert.equal(await verifyDeployment('https://bench.test', SHA, 'production', fixture(), noSleep), 'verified'); +}); +test('rejects an older healthy deployment', async () => { + await assert.rejects(verifyDeployment('https://bench.test', 'c'.repeat(40), 'production', fixture(), noSleep), /different build/); +}); +test('reports protected previews without claiming verification; production fails', async () => { + const fetcher = fixture({ '/api/health': () => new Response('', { status: 401 }) }); + assert.equal(await verifyDeployment('https://bench.test', SHA, 'preview', fetcher, noSleep), 'protected'); + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', fetcher, noSleep), /production route/); +}); +test('fails on empty group data and a CDN that never hits', async () => { + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', fixture({ + '/api/groups': () => Response.json({ groups: [] }), + }), noSleep), /populated groups/); + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', fixture({ + '/api/chart/chart': () => Response.json(chart, { headers: { 'x-vercel-cache': 'MISS' } }), + }), noSleep), /CDN did not report HIT/); +}); + +test('accepts a global commit window with no measurement for the selected chart', async () => { + assert.equal(await verifyDeployment('https://bench.test', SHA, 'production', fixture({ + '/api/group/group': () => Response.json({ charts: [{ commits: [{ sha: SHA }], series: { vortex: [null] } }] }), + }), noSleep), 'verified'); +}); + +test('rejects a successful HTTP response containing only an error shell', async () => { + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', fixture({ + '/': () => new Response('Error', { headers: { 'content-type': 'text/html' } }), + }), noSleep), /rendered page content is missing/); +}); + +test('recognizes Vercel SSO redirects only as protected previews', async () => { + const redirect = (location) => fixture({ + '/api/health': () => new Response(null, { status: 302, headers: { location } }), + }); + const sso = redirect('https://vercel.com/sso-api?url=https%3A%2F%2Fbench.test'); + assert.equal(await verifyDeployment('https://bench.test', SHA, 'preview', sso, noSleep), 'protected'); + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'production', sso, noSleep), /production route/); + await assert.rejects(verifyDeployment('https://bench.test', SHA, 'preview', redirect('https://example.com/login'), noSleep), /HTTP 302/); +}); diff --git a/bench/scripts/verify-web-deploy.mjs b/bench/scripts/verify-web-deploy.mjs new file mode 100644 index 00000000..3c1d16cf --- /dev/null +++ b/bench/scripts/verify-web-deploy.mjs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import assert from 'node:assert/strict'; +import { appendFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; +import { setTimeout } from 'node:timers/promises'; + +/** Verify the public read paths within a three-minute deadline. */ +export async function verifyDeployment(base, sha, environment, fetchImpl = fetch, sleep = setTimeout) { + assert.match(sha, /^[0-9a-f]{40}$/); + assert.ok(['preview', 'production'].includes(environment)); + const origin = new URL(base); + assert.equal(origin.protocol, 'https:'); + assert.equal(origin.username + origin.password + origin.search + origin.hash, ''); + const deadline = Date.now() + 180_000; + + async function request(path, validate) { + let lastError; + for (let attempt = 0; attempt < 4; attempt++) { + const remaining = deadline - Date.now(); + assert.ok(remaining > 0, 'Deployment verification deadline exceeded'); + try { + const response = await fetchImpl(new URL(path, origin), { + redirect: 'manual', + signal: AbortSignal.timeout(Math.min(20_000, remaining)), + }); + const location = response.headers.get('location'); + const redirect = location && [301, 302, 303, 307, 308].includes(response.status) + ? new URL(location, origin) : null; + const vercelSso = redirect?.origin === 'https://vercel.com' && redirect.pathname === '/sso-api'; + if ([401, 403].includes(response.status) || vercelSso) { + if (environment === 'preview') return 'protected'; + throw new Error(`Public production route ${path} returned ${response.status}`); + } + assert.equal(response.status, 200, `${path}: HTTP ${response.status}`); + return await validate(response); + } catch (error) { + lastError = error; + if (attempt < 3) await sleep(Math.min(2_000, Math.max(0, deadline - Date.now()))); + } + } + throw lastError; + } + + async function json(path, validate) { + return request(path, async (response) => { + assert.match(response.headers.get('content-type') ?? '', /application\/json/); + const body = await response.json(); + validate(body); + return body; + }); + } + + function chart(body) { + assert.ok(Array.isArray(body.commits), 'Chart commits must be an array'); + assert.ok(body.series && typeof body.series === 'object' && !Array.isArray(body.series), + 'Chart series must be an object'); + for (const values of Object.values(body.series)) { + assert.ok(Array.isArray(values) && values.length === body.commits.length && + values.every((value) => value === null || (typeof value === 'number' && Number.isFinite(value))), + 'Chart series must align with its commit window'); + } + } + + const health = await json('/api/health', (body) => { + assert.equal(body.status, 'ok'); + assert.equal(body.build_sha, sha, 'Health reports a different build'); + assert.ok(Number.isInteger(body.schema_version)); + }); + if (health === 'protected') return 'protected'; + const groups = await json('/api/groups', (body) => { + assert.ok(Array.isArray(body.groups) && body.groups.some((group) => group.charts?.length), + 'No populated groups'); + }); + if (groups === 'protected') return 'protected'; + const group = groups.groups.find((item) => item.charts?.length); + const groupPath = `/api/group/${encodeURIComponent(group.slug)}?n=1`; + const groupBody = await json(groupPath, (body) => { + assert.ok(Array.isArray(body.charts) && body.charts.length > 0); + chart(body.charts[0]); + }); + if (groupBody === 'protected') return 'protected'; + const chartSlug = encodeURIComponent(group.charts[0].slug); + const chartBody = await json(`/api/chart/${chartSlug}`, chart); + if (chartBody === 'protected') return 'protected'; + for (const path of ['/', `/chart/${chartSlug}`]) { + const result = await request(path, async (response) => { + assert.match(response.headers.get('content-type') ?? '', /text\/html/); + const html = await response.text(); + assert.ok(html.includes(path === '/' + ? 'data-role="global-filter-bar"' + : `data-chart-slug="${chartSlug}"`), `${path}: rendered page content is missing`); + }); + if (result === 'protected') return 'protected'; + } + // Repeat the same URLs, without a cache-busting query, to verify CDN reuse. + for (const path of ['/', `/api/chart/${chartSlug}`]) { + const result = await request(path, async (response) => { + await response.arrayBuffer(); + assert.equal(response.headers.get('x-vercel-cache'), 'HIT', `${path}: CDN did not report HIT`); + }); + if (result === 'protected') return 'protected'; + } + return 'verified'; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const result = await verifyDeployment(...process.argv.slice(2)); + const message = result === 'protected' + ? 'Deployment created; preview protection blocked verification. Health, build identity, and CDN caching remain unverified.' + : 'Verified build SHA, health, representative group/chart JSON, HTML pages, and CDN HIT.'; + console.log(`${result === 'protected' ? '::notice::' : ''}${message}`); + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `result=${result}\n`); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${message}\n`); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/bench/web/README.md b/bench/web/README.md index 8a1078be..75baf507 100644 --- a/bench/web/README.md +++ b/bench/web/README.md @@ -54,10 +54,14 @@ warmth: on those routes: that header is consumed (and stripped) by Vercel's CDN alone at the highest precedence, so the CDN caches the rendered pages while browsers still revalidate every load. -Verify on a live deployment with `curl -sI | grep -i x-vercel-cache` (expect `MISS` then -`HIT` within the five-minute fresh window). No workflow runs this check, so run it by hand after a -deploy that changes caching. Deployment URLs sit behind Vercel Authentication, so check a preview -through `vercel curl` and production on the public domain. One deliberate +The deploy workflow verifies the expected build SHA through `/api/health`, a representative group +and chart JSON response, the landing and chart pages, and a CDN `HIT` on repeated landing/chart-API +requests. The probe has a three-minute deadline. Production uses the public `BENCH_SITE_BASE_URL` +and fails verification on authentication errors or redirects. A protected preview reports +verification as blocked, with no claim that its health, build identity, or caching passed. To check +a protected preview by hand, use `vercel curl`. + +One deliberate divergence from the API routes: the `vercel.json` header rules apply to every response status, so an unknown `/chart/:slug` 404 follows the HTML page rule: five minutes fresh and then eligible for Vercel's one-day stale-while-revalidate window. That is acceptable for opaque, never-linked @@ -68,10 +72,18 @@ were, the same HTML page rule would bound it). ## Deploys `.github/workflows/bench-ci.yml` runs the check suite on every PR touching `bench/`, including -`migrations/**` (the integration suite applies that DDL to its testcontainer). -`.github/workflows/bench-deploy.yml` deploys via the Vercel CLI (`vercel pull` / `vercel build` / -`vercel deploy --prebuilt`): a preview deployment per same-repo PR that touches `bench/`, and a -production deployment on each push to `main` that does. +`migrations/**` (the integration suite applies that DDL to its testcontainer). It is a required +check on `main`. `.github/workflows/bench-deploy.yml` deploys via the Vercel CLI (`vercel pull` / +`vercel build` / `vercel deploy --prebuilt`): a preview deployment per same-repo PR that touches +`bench/`, and a production deployment on each push to `main` that does. Before it builds, the +deploy workflow runs the same checks (the `bench-checks` composite action) on the exact commit it +deploys. Fork PRs run CI but do not receive deployment credentials. + +The `bench-production` environment limits manual production runs to `main`. Push and manual +production runs share one production queue and let an active deploy finish verification. New +preview runs can cancel older previews of the same event and ref. The Vercel CLI builds on the +runner and uploads the prebuilt output. `BENCH_BUILD_SHA` is embedded in that output so the +uncached health endpoint identifies the checked-out build. One-time operator setup: @@ -81,7 +93,10 @@ One-time operator setup: 2. Set the repo variables `VERCEL_ORG_ID` + `BENCH_VERCEL_PROJECT_ID` (from the Vercel project settings), and the secret `VERCEL_TOKEN` (a deploy token scoped to this Vercel project) in both the `bench-preview` and `bench-production` GitHub environments. Limit - `bench-production` to `main`. + `bench-production` to `main`. Set the repo variable `BENCH_SITE_BASE_URL` to the public + production URL as `https://`. The keep-warm workflow uses the same variable. + Production deployment requires it before uploading and verifies the public domain after + deployment. 3. Configure `BENCH_DB_*` on the Vercel project (Production and Preview environments). Two open wiring choices are deliberately left to this step, per environment: - **Endpoint**: the public RDS instance endpoint (the same endpoint CI writers use). There is diff --git a/bench/web/next.config.js b/bench/web/next.config.js index e40cc786..b08a72ba 100644 --- a/bench/web/next.config.js +++ b/bench/web/next.config.js @@ -7,6 +7,11 @@ const nextConfig = { // (workspace root, the v2 benchmarks-website project, and this app), so // Next.js cannot infer the correct root on its own. outputFileTracingRoot: __dirname, + // Embed the checked-out commit in prebuilt output, independent of runtime settings. + env: { + VERCEL_GIT_COMMIT_SHA: + process.env.BENCH_BUILD_SHA ?? process.env.VERCEL_GIT_COMMIT_SHA ?? 'unknown', + }, }; module.exports = nextConfig;