diff --git a/.github/workflows/observability-preview-sweep.yml b/.github/workflows/observability-preview-sweep.yml index 2aac837b6a3..1297866c2bc 100644 --- a/.github/workflows/observability-preview-sweep.yml +++ b/.github/workflows/observability-preview-sweep.yml @@ -1,10 +1,11 @@ name: Observability — preview sweep # Daily sweep that removes stale preview folders + dashboards from the -# staging Grafana. Safety net for observability-preview.yml's cleanup-on- -# close job: a missed `closed` webhook, a job that errored mid-delete, or a -# preview pushed against a PR that was later deleted from GitHub all leave -# orphan `pr-*` resources behind. This workflow notices and cleans up. +# staging and production Grafanas. Safety net for +# observability-preview.yml's cleanup-on-close job: a missed `closed` +# webhook, a job that errored mid-delete, or a preview pushed against a PR +# that was later deleted from GitHub all leave orphan `pr-*` resources +# behind. This workflow notices and cleans up. # # Decision rule per PR number (derived from the `pr-` UID prefix): # - PR is open → keep @@ -41,14 +42,50 @@ concurrency: env: AWS_REGION: us-east-1 - AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-apply jobs: sweep: - name: Sweep stale previews from staging + name: Sweep stale previews from ${{ matrix.env }} runs-on: ubuntu-latest + strategy: + # One environment's Grafana being unreachable shouldn't stop the other + # from being swept — the two legs share nothing but this workflow file. + fail-fast: false + matrix: + include: + # The dedicated preview role — it grants the grafanactl token the + # sweep needs to list and delete `pr-` resources, and nothing + # else. See observability-preview.yml for why the apply role is + # the wrong one to reach for here. + - env: staging + role_arn: arn:aws:iam::680542703984:role/boxel-observability-preview + - env: production + role_arn: arn:aws:iam::120317779495:role/boxel-observability-preview + steps: + - name: Refuse to sweep production from a non-main ref + # `schedule` always runs the default branch; `workflow_dispatch` lets + # an operator choose any ref. This guard catches dispatching a stale + # but honest branch, which is a real mistake — nothing more. + # + # It cannot stop a ref whose copy of this workflow was rewritten, + # because workflow_dispatch runs the chosen ref's copy of the guard + # too. What limits the damage there is the credential, not this step: + # the sweep assumes boxel-observability-preview, whose Grafana account + # is Viewer plus Admin on the `Previews` folder, so a rewritten copy + # still cannot touch a canonical dashboard. + if: matrix.env == 'production' + env: + REF: ${{ github.ref }} + run: | + set -eo pipefail + if [[ "$REF" != "refs/heads/main" ]]; then + echo "::error::The production sweep may only run from refs/heads/main. Got: $REF" >&2 + exit 1 + fi + echo "Confirmed run from main." + - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -58,13 +95,15 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 with: - role-to-assume: ${{ env.AWS_ROLE_ARN }} + role-to-assume: ${{ matrix.role_arn }} aws-region: ${{ env.AWS_REGION }} - name: Source GRAFANA_TOKEN + env: + ENV_NAME: ${{ matrix.env }} run: | GRAFANA_TOKEN="$(aws ssm get-parameter \ - --name /staging/grafana/grafanactl_token \ + --name "/${ENV_NAME}/grafana/preview_grafanactl_token" \ --with-decryption \ --query 'Parameter.Value' \ --output text)" @@ -77,19 +116,20 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} + ENV_NAME: ${{ matrix.env }} run: | set -eo pipefail remote="$(mktemp -d)" - cfg="$(./scripts/render-config.sh staging)" + cfg="$(./scripts/render-config.sh "$ENV_NAME")" trap 'rm -rf "$remote"; rm -f "$cfg"' EXIT - grafanactl --config "$cfg" --context staging \ + grafanactl --config "$cfg" --context "$ENV_NAME" \ resources pull dashboards folders --path "$remote" >/dev/null # Extract the set of unique PR numbers present in preview UIDs. # `grep ... || true` because `set -eo pipefail` would otherwise # abort the step when grep finds no matches — that's the legitimate - # "no previews live in staging" path, not an error. + # "no previews live in this environment" path, not an error. pr_numbers="$(find "$remote" -type f -name '*.json' \ -exec jq -r '.metadata.name // ""' {} \; \ | { grep -E '^pr[0-9]+-' || true; } \ @@ -161,11 +201,12 @@ jobs: working-directory: packages/observability env: STALE_PRS: ${{ steps.identify.outputs.stale_prs }} + ENV_NAME: ${{ matrix.env }} run: | set -eo pipefail for pr in $STALE_PRS; do echo "::group::Cleanup PR #${pr}" - ./scripts/cleanup-preview.sh --pr "$pr" --env staging + ./scripts/cleanup-preview.sh --pr "$pr" --env "$ENV_NAME" echo "::endgroup::" done @@ -173,9 +214,10 @@ jobs: if: always() env: STALE_PRS: ${{ steps.identify.outputs.stale_prs }} + ENV_NAME: ${{ matrix.env }} run: | { - echo "## Preview sweep" + echo "## Preview sweep (${ENV_NAME})" echo "" if [[ -z "${STALE_PRS:-}" ]]; then echo "No stale previews." diff --git a/.github/workflows/observability-preview.yml b/.github/workflows/observability-preview.yml index e9457b6252d..f2dfd7790f4 100644 --- a/.github/workflows/observability-preview.yml +++ b/.github/workflows/observability-preview.yml @@ -1,11 +1,25 @@ name: Observability — PR preview -# Spins up a per-PR preview of the dashboards a PR changes inside the staging -# Grafana, under per-PR-prefixed UIDs (`pr-*`) in a per-PR folder so the -# preview coexists with the canonical staging copies without overwriting -# them. On PR close the preview is torn down. A separate sweep workflow -# (observability-preview-sweep.yml) is the safety net for closes the -# preview-cleanup job missed. +# Spins up a per-PR preview of the dashboards a PR changes inside BOTH the +# staging and the production Grafana, under per-PR-prefixed UIDs (`pr-*`) +# in a per-PR folder so the preview coexists with the canonical copies +# without overwriting them. On PR close both previews are torn down. A +# separate sweep workflow (observability-preview-sweep.yml) is the safety net +# for closes the preview-cleanup job missed. +# +# Both environments are previewed from one job so the PR gets a single +# sticky comment linking each changed dashboard in both Grafanas, rather +# than one comment per environment. Staging is applied first: a production +# Grafana problem then costs the run its red status but not the staging +# preview or the comment, which is posted from whatever succeeded. +# +# The two environments render the same dashboard set under the same preview +# UIDs — only the apply-time substitutions differ — so the comment lists +# each dashboard once with a link per environment. +# +# CAUTION: a preview carries the real operator secret, so the action buttons +# on a preview dashboard act on the real environment. A button clicked on a +# production preview performs the production mutation. # # Scope: dashboards + folders only. Data sources, alert rules, and the # home-dashboard preference are file-provisioned at Grafana startup and @@ -37,19 +51,37 @@ concurrency: env: AWS_REGION: us-east-1 - # Reuses the apply role from cardstack/infra#665 (same role - # observability-apply-staging.yml uses). Permissions cover every SSM path - # below — see that workflow's header for the up-to-date list. - AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-apply + # The dedicated preview role, NOT boxel-observability-apply. This workflow + # runs on `pull_request`, so GitHub loads it — and every script it calls — + # from the pull request itself; whatever the assumed role can read, a + # contributor can read. boxel-observability-preview grants three + # parameters per environment and nothing else: + # + # //grafana/preview_grafanactl_token scoped Grafana account + # //boxel-grafana/realm_server_url fill a template variable + # //boxel/GRAFANA_SECRET make operator buttons work + # + # The Grafana token belongs to `observability-preview`: Viewer, plus Admin + # on the `Previews` folder. So a pull request cannot overwrite or delete a + # canonical dashboard. It can still read them, and therefore read + # GRAFANA_SECRET out of the six that carry it — that half closes when + # operator calls move to a Grafana datasource proxy. + # + # The apply role additionally grants the Grafana database credentials and + # the Loki / Prometheus / DB-host values. Never point this workflow at it. + STAGING_AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-preview + PRODUCTION_AWS_ROLE_ARN: arn:aws:iam::120317779495:role/boxel-observability-preview PREVIEW_MARKER: "" STAGING_GRAFANA_BASE_URL: "https://dashboard-staging.stack.cards" + PRODUCTION_GRAFANA_BASE_URL: "https://dashboard.boxel.ai" jobs: preview-apply: - name: Apply preview to staging - # Fork-PR guard: this job assumes AWS and pushes to staging Grafana, so - # it must not run on PRs whose code we don't trust. Same pattern as - # observability-diff.yml and observability-apply-staging.yml. + name: Apply preview + # Fork-PR guard: this job assumes AWS and pushes to staging and + # production Grafana, so it must not run on PRs whose code we don't + # trust. Same pattern as observability-diff.yml and + # observability-apply-staging.yml. if: >- github.event.action != 'closed' && github.event.pull_request.head.repo.full_name == github.repository @@ -77,94 +109,97 @@ jobs: - name: Install grafanactl uses: ./.github/actions/install-grafanactl - - name: Configure AWS credentials + - name: Configure AWS credentials (staging) uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 with: - role-to-assume: ${{ env.AWS_ROLE_ARN }} + role-to-assume: ${{ env.STAGING_AWS_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - - name: Source preview-time secrets from SSM - # Subset of what observability-apply-staging.yml fetches. The preview - # only touches dashboards + folders, so we don't need the LOKI_URL / - # BOXEL_DB_* / WORKER_LOG_* set — those are read by apply-datasources - # / apply-alerting, which apply-preview.sh never invokes. - run: | - GRAFANA_TOKEN="$(aws ssm get-parameter \ - --name /staging/grafana/grafanactl_token \ - --with-decryption \ - --query 'Parameter.Value' \ - --output text)" - echo "::add-mask::$GRAFANA_TOKEN" - echo "GRAFANA_TOKEN=$GRAFANA_TOKEN" >> "$GITHUB_ENV" - - REALM_SERVER_URL="$(aws ssm get-parameter \ - --name /staging/boxel-grafana/realm_server_url \ - --query 'Parameter.Value' \ - --output text)" - echo "REALM_SERVER_URL=$REALM_SERVER_URL" >> "$GITHUB_ENV" - - GRAFANA_SECRET="$(aws ssm get-parameter \ - --name /staging/boxel/GRAFANA_SECRET \ - --with-decryption \ - --query 'Parameter.Value' \ - --output text)" - echo "::add-mask::$GRAFANA_SECRET" - echo "GRAFANA_SECRET=$GRAFANA_SECRET" >> "$GITHUB_ENV" - - - name: Apply preview - id: apply + - name: Apply preview to staging + id: apply-staging working-directory: packages/observability env: + ENV_NAME: staging PR_NUMBER: ${{ github.event.pull_request.number }} PR_BASE_REF: ${{ github.event.pull_request.base.ref }} - run: | - set -eo pipefail - OUT_FILE="$(mktemp)" - ./scripts/apply-preview.sh \ - --pr "$PR_NUMBER" \ - --base-ref "origin/${PR_BASE_REF}" \ - --env staging \ - > "$OUT_FILE" - echo "out_file=$OUT_FILE" >> "$GITHUB_OUTPUT" - if [[ -s "$OUT_FILE" ]]; then - echo "empty=false" >> "$GITHUB_OUTPUT" - else - echo "empty=true" >> "$GITHUB_OUTPUT" - fi - echo "::group::Preview summary" - cat "$OUT_FILE" - echo "::endgroup::" - - - name: Cleanup stale preview (no changes left) - # When the current head of the PR has no dashboard changes vs base, - # apply-preview emits empty output and pushes nothing. But a prior - # push to the same PR may have already applied a preview that's - # still live in staging — revert/rebase scenarios. Run cleanup so - # the PR's staging footprint matches its current head; otherwise - # the daily sweep wouldn't catch this until PR close. - if: steps.apply.outputs.empty == 'true' + run: ./scripts/ci-apply-preview.sh + + # Production is applied after staging, and each environment's + # credentials live only in the step that uses them — the AWS creds in + # the runner environment and the Grafana token inside the step's own + # shell. Nothing is written to $GITHUB_ENV, so there's no window in + # which one environment's token is live while the other's push runs. + - name: Configure AWS credentials (production) + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 + with: + role-to-assume: ${{ env.PRODUCTION_AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Apply preview to production + id: apply-production working-directory: packages/observability env: + ENV_NAME: production PR_NUMBER: ${{ github.event.pull_request.number }} - run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env staging + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + run: ./scripts/ci-apply-preview.sh - name: Post sticky PR comment + # Runs even when an apply step failed, so a production-side outage + # still leaves reviewers the staging links (and a note saying which + # environment is missing). The job is still red — the failed step + # decides that, not this one. + # # Pinned to SHA (matches observability-diff.yml's reasoning): # pull-requests: write + AWS role = non-trivial supply-chain exposure. + if: ${{ !cancelled() }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - OUT_FILE: ${{ steps.apply.outputs.out_file }} + STAGING_OUT_FILE: ${{ steps.apply-staging.outputs.out_file }} + STAGING_OUTCOME: ${{ steps.apply-staging.outcome }} + PRODUCTION_OUT_FILE: ${{ steps.apply-production.outputs.out_file }} + PRODUCTION_OUTCOME: ${{ steps.apply-production.outcome }} with: script: | const fs = require('fs'); const marker = process.env.PREVIEW_MARKER; - const grafanaBase = process.env.STAGING_GRAFANA_BASE_URL; const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const raw = fs.readFileSync(process.env.OUT_FILE, 'utf8').trim(); + // Parse one environment's `::` summary (the + // structured stdout of apply-preview.sh). A missing file means + // the apply step never got far enough to write one. + const readSummary = (outFile) => { + if (!outFile || !fs.existsSync(outFile)) return null; + const folders = []; + const dashboards = []; + for (const line of fs.readFileSync(outFile, 'utf8').trim().split('\n')) { + // Title may contain `:`, so only the first two fields are + // delimited. + const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/); + if (!m) continue; + const [, kind, uid, title] = m; + (kind === 'FOLDER' ? folders : dashboards).push({ uid, title }); + } + return { folders, dashboards }; + }; + + const environments = [ + { + name: 'staging', + baseUrl: process.env.STAGING_GRAFANA_BASE_URL, + outcome: process.env.STAGING_OUTCOME, + summary: readSummary(process.env.STAGING_OUT_FILE), + }, + { + name: 'production', + baseUrl: process.env.PRODUCTION_GRAFANA_BASE_URL, + outcome: process.env.PRODUCTION_OUTCOME, + summary: readSummary(process.env.PRODUCTION_OUT_FILE), + }, + ]; - // Find any prior preview sticky on this PR; we either update or - // delete it based on whether there's anything to preview now. + // Find any prior preview sticky on this PR; we either update, + // delete, or deliberately leave it alone below. const comments = await github.paginate( github.rest.issues.listComments, { @@ -178,12 +213,28 @@ jobs: c => c.body && c.body.startsWith(marker), ); - if (!raw) { - // No dashboards changed at this HEAD. Don't post a "nothing - // to see" comment — if a prior push left a sticky behind, - // delete it so the PR's comment stream matches its current - // state. (Stale preview *resources* from a prior push are - // handled by the "Cleanup stale preview" step above.) + const applied = environments.filter( + e => e.outcome === 'success' && e.summary && e.summary.dashboards.length, + ); + const missing = environments.filter(e => e.outcome !== 'success'); + + if (!applied.length) { + if (missing.length) { + // At least one apply failed and nothing is left to link. + // Leave any prior sticky in place: preview resources from + // an earlier push may well still be live, and the failed + // step already makes the run red. + core.info( + `No preview to link and ${missing.map(e => e.name).join(', ')} did not apply — leaving any existing comment alone.`, + ); + return; + } + // Every environment applied cleanly and none had dashboards: + // this HEAD changes no dashboards. Don't post a "nothing to + // see" comment — if a prior push left a sticky behind, delete + // it so the PR's comment stream matches its current state. + // (Stale preview *resources* from a prior push are deleted by + // ci-apply-preview.sh.) if (existing) { await github.rest.issues.deleteComment({ owner: context.repo.owner, @@ -197,36 +248,61 @@ jobs: return; } - const folders = []; - const dashboards = []; - for (const line of raw.split('\n')) { - // Format: `<KIND>:<uid>:<title>` (title may contain `:`). - const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/); - if (!m) continue; - const [, kind, uid, title] = m; - (kind === 'FOLDER' ? folders : dashboards).push({ uid, title }); - } + // Both environments render the same preview UIDs, so collapse + // them into one row per resource carrying a link per + // environment. Keyed by UID and built in the order the first + // applied environment reported, so a partial apply still lists + // everything that made it somewhere. + const collapse = (key) => { + const byUid = new Map(); + for (const env of applied) { + for (const { uid, title } of env.summary[key]) { + if (!byUid.has(uid)) byUid.set(uid, { title, envs: [] }); + byUid.get(uid).envs.push(env); + } + } + return [...byUid.entries()].map(([uid, v]) => ({ uid, ...v })); + }; - const folderLines = folders.map(f => - `- **Folder:** [${f.title}](${grafanaBase}/dashboards/f/${f.uid}/)`, + const links = (envs, path) => + envs + .map(e => `[${e.name}](${e.baseUrl}${path})`) + .join(' · '); + + const folders = collapse('folders'); + const dashboards = collapse('dashboards'); + + const folderLines = folders.map( + f => `- **Folder:** ${f.title} — ${links(f.envs, `/dashboards/f/${f.uid}/`)}`, ); - const dashboardLines = dashboards.map(d => - `- [${d.title}](${grafanaBase}/d/${d.uid}/)`, + const dashboardLines = dashboards.map( + d => `- ${d.title} — ${links(d.envs, `/d/${d.uid}/`)}`, ); + const appliedNames = applied.map(e => e.name).join(' and '); + + const warning = missing.length + ? [ + '', + '> [!WARNING]', + `> The **${missing.map(e => e.name).join('** and **')}** preview did not apply on this run — see the run log below.`, + ] + : []; + const body = [ marker, '## Grafana preview', '', - `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in the staging Grafana.`, - 'Cross-dashboard drill-throughs still point at the canonical staging dashboards.', + `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in ${appliedNames}.`, + "Cross-dashboard drill-throughs still point at each environment's canonical dashboards.", + ...warning, '', ...folderLines, '', '**Dashboards:**', ...dashboardLines, '', - 'Preview is torn down automatically when this PR is closed or merged.', + 'Previews are torn down automatically when this PR is closed or merged.', '', `_(Run: ${runUrl})_`, ].join('\n'); @@ -250,9 +326,10 @@ jobs: } preview-cleanup: - name: Cleanup preview from staging + name: Cleanup preview # Cleanup runs on the `closed` action (covers both merge and close- - # without-merge). Same fork-PR guard — cleanup hits staging. + # without-merge). Same fork-PR guard — cleanup hits staging and + # production. if: >- github.event.action == 'closed' && github.event.pull_request.head.repo.full_name == github.repository @@ -264,36 +341,40 @@ jobs: - name: Install grafanactl uses: ./.github/actions/install-grafanactl - - name: Configure AWS credentials + - name: Configure AWS credentials (staging) uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 with: - role-to-assume: ${{ env.AWS_ROLE_ARN }} + role-to-assume: ${{ env.STAGING_AWS_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - - name: Source GRAFANA_TOKEN - # Cleanup doesn't need any of the URL/secret substitution values — - # it lists live state and deletes by selector. Only the grafanactl - # token matters. - run: | - GRAFANA_TOKEN="$(aws ssm get-parameter \ - --name /staging/grafana/grafanactl_token \ - --with-decryption \ - --query 'Parameter.Value' \ - --output text)" - echo "::add-mask::$GRAFANA_TOKEN" - echo "GRAFANA_TOKEN=$GRAFANA_TOKEN" >> "$GITHUB_ENV" - - - name: Cleanup preview + - name: Cleanup preview from staging working-directory: packages/observability env: PR_NUMBER: ${{ github.event.pull_request.number }} run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env staging + - name: Configure AWS credentials (production) + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 + with: + role-to-assume: ${{ env.PRODUCTION_AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Cleanup preview from production + working-directory: packages/observability + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env production + - name: Delete sticky PR comment # After cleanup, there's nothing left to point at — delete the # sticky entirely rather than leaving a "preview cleaned up" # gravestone. Matches the apply-time policy of not posting a # comment when there's nothing to show. + # + # Runs even if a cleanup step failed: the daily sweep + # (observability-preview-sweep.yml) will catch whatever survived, + # and a comment pointing at deleted dashboards is worse than none. + if: ${{ !cancelled() }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | diff --git a/packages/observability/README.md b/packages/observability/README.md index 06439873440..d6682814a3b 100644 --- a/packages/observability/README.md +++ b/packages/observability/README.md @@ -468,6 +468,68 @@ CI runs `apply.sh --env staging` on merge to main (workflow: `apply.sh --env production` on the production workflow (CS-10936). +A PR that changes anything under `grafanactl/resources/dashboards/` also gets +a live preview of just the dashboards it touches, pushed to **both** the +staging and the production Grafana under `pr<n>-`-prefixed UIDs in a per-PR +folder, so it coexists with the canonical copies rather than overwriting them. +One sticky PR comment links each changed dashboard in both environments, and +both previews are deleted when the PR closes. +`.github/workflows/observability-preview-sweep.yml` sweeps both environments +daily for any the close hook missed. + +**The preview runs pull-request-controlled code, so it holds a deliberately +weak credential.** It runs on `pull_request`, and GitHub loads that workflow — +and every script it calls — from the PR itself. Whatever the assumed role can +read, a contributor can read. So the preview uses its own IAM role, +`boxel-observability-preview` (cardstack/infra, +`configs/boxel-observability-preview/`), granting three parameters per +environment: + +``` +/<env>/grafana/preview_grafanactl_token scoped Grafana service account +/<env>/boxel-grafana/realm_server_url fill the realm_server variable +/<env>/boxel/GRAFANA_SECRET make the operator buttons work +``` + +Not `boxel-observability-apply`. That role grants the Grafana database +credentials and, worse, the **Admin** `grafanactl_token`. Nothing in Grafana +scopes that token to `pr<n>-` UIDs, so a PR holding it could overwrite or +delete any canonical dashboard — and could read `GRAFANA_SECRET` without being +granted it, since `apply.sh` substitutes the live secret into six canonical +dashboards and `cleanup-preview.sh` pulls every dashboard on each PR close. + +The preview token therefore belongs to a separate Grafana service account, +`observability-preview` (cardstack/infra, `configs/grafana/base/main.tf`): +**Viewer, plus Admin on the `Previews` folder and nothing else.** Every per-PR +folder is nested inside `Previews`, which is what makes that single grant +sufficient — a root-level preview folder is refused. + +What a PR can still do, stated plainly: + +- write and delete dashboards and folders **inside `Previews`** +- read every dashboard, and so read `GRAFANA_SECRET` out of the six carrying it +- use `GRAFANA_SECRET` to call the realm-server operator endpoints + +It **cannot** overwrite or delete a canonical dashboard. That was the +irreversible risk and it is closed. The read half stays open until operator +calls move to a Grafana datasource proxy and the secret leaves dashboard +bodies; the preview grant should be dropped when that lands. + +`role = "None"` would close the read half too, and Grafana enforces the folder +scope correctly under it — but grafanactl drives the App Platform APIs, which +403 for an account with no basic role however its folder permissions are set. +Push, pull and delete all fail, so the preview tooling cannot run under it. +Only the legacy REST API honours folder scoping. + +Two mechanisms keep the preview inside its lane, so keep them true: + +- **Previews cannot address a canonical dashboard.** `render-preview.sh` + rewrites every UID it emits, then fails closed unless each dashboard sits in + this PR's own folder and each preview folder's only parent is `Previews`. +- **Operator buttons act on the real environment.** A preview carries the real + `GRAFANA_SECRET`, so a button clicked on a production preview performs the + production mutation. Prefer the staging preview for exercising one. + ## Operator actions Some dashboards carry buttons that POST to a realm-server operator endpoint — diff --git a/packages/observability/scripts/apply-preview.sh b/packages/observability/scripts/apply-preview.sh index 86fb4c1eb66..5d5fdf772ff 100755 --- a/packages/observability/scripts/apply-preview.sh +++ b/packages/observability/scripts/apply-preview.sh @@ -3,7 +3,8 @@ # # Wraps render-preview.sh (changed-dashboard filtering + UID rewriting) and # `grafanactl resources push`. Used by the observability-preview.yml workflow -# on PR open/synchronize/reopen. +# on PR open/synchronize/reopen, once per environment (see +# ci-apply-preview.sh). # # Usage: # ./scripts/apply-preview.sh --pr <n> --base-ref <ref> [--env staging|local|production] @@ -59,6 +60,11 @@ done cd "$(dirname "$0")/.." +# Previews authenticate as the scoped `observability-preview` account, never +# the Admin one the canonical apply uses. See cardstack/infra, +# configs/grafana/base/main.tf. +export GRAFANA_TOKEN_SSM_PATH="${GRAFANA_TOKEN_SSM_PATH:-/${env_name}/grafana/preview_grafanactl_token}" + # shellcheck source=./grafanactl-env.sh source ./scripts/grafanactl-env.sh "$env_name" diff --git a/packages/observability/scripts/ci-apply-preview.sh b/packages/observability/scripts/ci-apply-preview.sh new file mode 100755 index 00000000000..f1404f15ec0 --- /dev/null +++ b/packages/observability/scripts/ci-apply-preview.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# ci-apply-preview.sh — one environment's half of the PR preview. +# +# Called once per environment by observability-preview.yml. +# +# Fetches the apply-time secrets for $ENV_NAME from SSM, pushes the per-PR +# preview with apply-preview.sh, and reports the result back to the workflow +# on $GITHUB_OUTPUT: +# +# out_file=<path> structured `<KIND>:<uid>:<title>` summary (see +# apply-preview.sh); empty file when nothing was pushed +# empty=true|false whether this PR's HEAD changes any dashboards +# +# Lives here rather than inline in either workflow so staging and production +# run byte-identical logic — the environments differ only in $ENV_NAME and +# in which AWS credentials the caller configured before invoking this. +# +# Environment (all set by the workflow step): +# ENV_NAME staging | production +# PR_NUMBER the pull request number +# PR_BASE_REF the PR's base branch name (e.g. `main`), without `origin/` +# +# Prereqs: AWS credentials for $ENV_NAME already configured, grafanactl on +# PATH, CWD anywhere inside the repo. +set -eo pipefail + +require_env() { + [[ -n "${!1:-}" ]] || { echo "error: $1 is required" >&2; exit 2; } +} + +require_env ENV_NAME +require_env PR_NUMBER +require_env PR_BASE_REF +require_env GITHUB_OUTPUT + +case "$ENV_NAME" in + staging | production) ;; + *) echo "error: ENV_NAME must be staging or production (got: $ENV_NAME)" >&2; exit 2 ;; +esac + +cd "$(dirname "$0")/.." + +# These three parameters are the whole grant of the +# boxel-observability-preview role, and the list stays short on purpose: +# this script runs from the pull request's own checkout, so whatever it can +# read, a pull request can read. Adding an SSM read here means widening that +# role and handing the new value to unreviewed code. The Grafana database +# credentials are excluded for exactly that reason. +# +# GRAFANA_SECRET is included because the operator buttons on a preview +# dashboard are part of what a reviewer needs to see working. It is a weaker +# secret than it looks: apply.sh substitutes it into six canonical dashboards +# as a `constant` template variable, so every Grafana viewer can already read +# it out of the dashboard JSON. Withholding it here would not keep it from +# anyone who can reach the preview. +# +# The Grafana token is the `observability-preview` service account, not the +# Admin `grafanactl_token` the canonical apply uses. It is Viewer plus Admin +# on the `Previews` folder, so a pull request holding it cannot overwrite or +# delete a canonical dashboard. It can still read them, which is why +# GRAFANA_SECRET below is not meaningfully protected by withholding it. +# +# GRAFANA_TOKEN is re-fetched by grafanactl-env.sh inside apply-preview.sh; +# fetching it here too is what registers the `::add-mask::`, so the value is +# redacted if grafanactl ever echoes it on a failure path. +ssm() { + aws ssm get-parameter --name "$1" "${@:2}" \ + --query 'Parameter.Value' --output text +} + +GRAFANA_TOKEN="$(ssm "/${ENV_NAME}/grafana/preview_grafanactl_token" --with-decryption)" +echo "::add-mask::$GRAFANA_TOKEN" + +GRAFANA_SECRET="$(ssm "/${ENV_NAME}/boxel/GRAFANA_SECRET" --with-decryption)" +echo "::add-mask::$GRAFANA_SECRET" + +REALM_SERVER_URL="$(ssm "/${ENV_NAME}/boxel-grafana/realm_server_url")" + +export GRAFANA_TOKEN GRAFANA_SECRET REALM_SERVER_URL + +out_file="$(mktemp)" +echo "out_file=$out_file" >> "$GITHUB_OUTPUT" + +./scripts/apply-preview.sh \ + --pr "$PR_NUMBER" \ + --base-ref "origin/${PR_BASE_REF}" \ + --env "$ENV_NAME" \ + > "$out_file" + +if [[ -s "$out_file" ]]; then + echo "empty=false" >> "$GITHUB_OUTPUT" + echo "::group::Preview summary (${ENV_NAME})" + cat "$out_file" + echo "::endgroup::" + exit 0 +fi + +echo "empty=true" >> "$GITHUB_OUTPUT" + +# When the current head of the PR has no dashboard changes vs base, +# apply-preview emits empty output and pushes nothing. But a prior push to +# the same PR may have already applied a preview that's still live — revert / +# rebase scenarios. Run cleanup so the PR's footprint in this environment +# matches its current head; otherwise the daily sweep wouldn't catch this +# until PR close. +echo "no dashboard changes at this HEAD — clearing any preview left in ${ENV_NAME}" +./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env "$ENV_NAME" diff --git a/packages/observability/scripts/cleanup-preview.sh b/packages/observability/scripts/cleanup-preview.sh index 529bcfba85f..9a728fa4f54 100755 --- a/packages/observability/scripts/cleanup-preview.sh +++ b/packages/observability/scripts/cleanup-preview.sh @@ -21,7 +21,10 @@ # - apply-preview.sh runs this before every push, so a PR that shrinks # its changed-dashboard set (e.g. reverts one of two prior edits) # gets the old preview resources removed before the new tree lands. -# - observability-preview.yml runs this on pull_request `closed`. +# - ci-apply-preview.sh runs this when a PR stops changing dashboards +# altogether, to clear a preview an earlier push left behind. +# - observability-preview.yml runs this on pull_request `closed`, once +# per environment. # - observability-preview-sweep.yml runs this for PRs the daily sweep # has decided are stale. set -eo pipefail @@ -53,6 +56,10 @@ done cd "$(dirname "$0")/.." +# Same scoped account as apply-preview.sh — teardown needs no more reach +# than the push did. +export GRAFANA_TOKEN_SSM_PATH="${GRAFANA_TOKEN_SSM_PATH:-/${env_name}/grafana/preview_grafanactl_token}" + # shellcheck source=./grafanactl-env.sh source ./scripts/grafanactl-env.sh "$env_name" diff --git a/packages/observability/scripts/grafanactl-env.sh b/packages/observability/scripts/grafanactl-env.sh index 3b8a7a9df67..a8396afc32d 100755 --- a/packages/observability/scripts/grafanactl-env.sh +++ b/packages/observability/scripts/grafanactl-env.sh @@ -7,8 +7,12 @@ # source ./scripts/grafanactl-env.sh production # # Local is a no-op (admin/admin in-config is sufficient). -# Staging and production fetch the token from SSM at /<env>/grafana/grafanactl_token, -# which is populated by the cardstack/infra Terraform. +# Staging and production fetch the token from SSM. The default parameter is +# /<env>/grafana/grafanactl_token — the Admin service account that applies +# the canonical dashboards. Set GRAFANA_TOKEN_SSM_PATH to override it: the +# preview tooling points at /<env>/grafana/preview_grafanactl_token, a +# Viewer account holding Admin on the `Previews` folder only. Both are +# populated by the cardstack/infra Terraform. # # Requires AWS credentials with ssm:GetParameter on /<env>/grafana/* — the same # creds devs already use for `/terraform-plan workspace=grafana/<env>`. @@ -36,12 +40,13 @@ _grafanactl_env_main() { staging | production) # Capture stderr so we can show it on failure. if ! token_val="$(aws ssm get-parameter \ - --name "/${env_name}/grafana/grafanactl_token" \ + --name "${GRAFANA_TOKEN_SSM_PATH:-/${env_name}/grafana/grafanactl_token}" \ --with-decryption \ --query 'Parameter.Value' \ --output text 2>&1)"; then fetch_err="$token_val" - printf 'error: failed to fetch /%s/grafana/grafanactl_token from SSM:\n' "$env_name" >&2 + printf 'error: failed to fetch %s from SSM:\n' \ + "${GRAFANA_TOKEN_SSM_PATH:-/${env_name}/grafana/grafanactl_token}" >&2 printf '%s\n' "$fetch_err" >&2 printf 'hint: confirm AWS creds active for the right account and ssm:GetParameter is granted.\n' >&2 return 1 diff --git a/packages/observability/scripts/render-preview.sh b/packages/observability/scripts/render-preview.sh index aaaf68d30b0..56e7fa0135c 100755 --- a/packages/observability/scripts/render-preview.sh +++ b/packages/observability/scripts/render-preview.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash -# render-preview.sh — Render a per-PR preview tree for staging Grafana. +# render-preview.sh — Render a per-PR preview tree for a Grafana environment. # # Output: prints a tempdir path on stdout. The tree contains only the # dashboards the PR changed (vs --base-ref) plus the folder(s) they live in, # with `metadata.name` (UID), `spec.title`, and each dashboard's # `metadata.annotations["grafana.app/folder"]` rewritten so the resources -# can coexist with the canonical staging copies. Push or delete the result -# with `grafanactl resources push|delete --path <out>`. +# can coexist with the canonical copies in whichever environment they are +# pushed to. Push or delete the result with +# `grafanactl resources push|delete --path <out>`. # # When the PR doesn't change any dashboards, exits 0 with no stdout. # @@ -24,9 +25,9 @@ # # Cross-dashboard drill-through links inside the dashboard JSON (`/d/<uid>/` # strings) are intentionally NOT rewritten — they stay pointing at the -# canonical staging dashboards (CS-11106 design call). UID references inside -# `datasource.uid` fields are data-source UIDs, not dashboard UIDs, and are -# also left alone. +# canonical dashboards of whichever environment the preview lands in +# (CS-11106 design call). UID references inside `datasource.uid` fields are +# data-source UIDs, not dashboard UIDs, and are also left alone. set -eo pipefail usage_error() { echo "error: $1" >&2; exit 2; } @@ -94,10 +95,26 @@ if [[ -z "$changed_paths" ]]; then fi # Per-env substitution defaults mirror apply.sh exactly so the rendered -# preview matches what `apply.sh --env <env>` would push. +# preview matches what `apply.sh --env <env>` would push. The real +# GRAFANA_SECRET goes in, so the operator buttons on a preview dashboard +# (Reindex, Add Credit, Grant Permission, Delete Job) behave as they will +# once the change merges — a preview whose buttons 401 cannot show whether +# a button change works. +# +# CAUTION: those buttons act on the real environment. A button clicked on a +# production preview performs the production mutation; it is not a +# rehearsal. Prefer the staging preview for exercising one. +# UID of the shared parent folder that every per-PR preview folder nests +# under. Fixed, because cardstack/infra creates it with this UID and grants +# the preview service account Admin on it by name. +preview_parent_folder="previews" + case "$env_name" in local) realm_server_url="${REALM_SERVER_URL:-http://localhost:4201/}" + # The docker-compose Grafana has no `Previews` folder, and pushing a + # child of a missing parent fails. + preview_parent_folder="" if [[ -n "${GRAFANA_SECRET:-}" ]]; then grafana_secret="$GRAFANA_SECRET" else @@ -220,8 +237,14 @@ done < <(find "$rendered/dashboards" -type f -name '*.json' -print0) find "$rendered/dashboards" -type d -empty -delete 2>/dev/null || true # Step 3 — folders: keep only those referenced by a kept dashboard, rewrite -# UID + title. Drop the rest (so the push doesn't create unused per-PR -# folders for unchanged areas). +# UID + title, and nest them under the shared `Previews` parent. Drop the +# rest (so the push doesn't create unused per-PR folders for unchanged +# areas). +# +# The nesting is what makes the scoped credential sufficient. The +# `observability-preview` service account is Viewer plus Admin on `Previews` +# only, so it can create a folder inside that parent and nowhere else; a +# root-level preview folder is refused. while IFS= read -r -d '' f; do orig_uid="$(jq -r '.metadata.name // ""' "$f")" if [[ -z "$orig_uid" ]] || [[ "$referenced_folders" != *"${orig_uid}"$'\n'* ]]; then @@ -229,9 +252,15 @@ while IFS= read -r -d '' f; do continue fi new_uid="$(prefix_uid "$pr_number" "$orig_uid")" - jq --arg uid "$new_uid" --arg prefix "Preview – PR #${pr_number}: " ' + jq --arg uid "$new_uid" \ + --arg prefix "Preview – PR #${pr_number}: " \ + --arg parent "$preview_parent_folder" ' .metadata.name = $uid | (if .spec.title then .spec.title = ($prefix + .spec.title) else . end) + | (if $parent != "" then + .metadata.annotations = + ((.metadata.annotations // {}) + {"grafana.app/folder": $parent}) + else . end) ' "$f" > "$f.tmp" mv "$f.tmp" "$f" done < <(find "$rendered/folders" -type f -name '*.json' -print0) @@ -245,4 +274,38 @@ if [[ -z "$(find "$rendered/dashboards" -type f -name '*.json' 2>/dev/null)" ]]; exit 0 fi +# Fail closed if any emitted manifest kept a canonical UID. The rewrites +# above already guarantee the `pr<n>-` prefix on every dashboard, every +# folder, and every dashboard's folder pointer; this check makes that +# guarantee explicit so a future change to them cannot silently produce a +# tree that overwrites a real dashboard. The tree is pushed to production as +# well as staging, so losing the prefix would cost a live production +# dashboard. +while IFS= read -r -d '' f; do + uid="$(jq -r '.metadata.name // ""' "$f")" + if [[ "$uid" != "pr${pr_number}-"* ]]; then + echo "error: $f carries UID '$uid', which lacks the pr${pr_number}- prefix" >&2 + rm -rf "$rendered" + exit 1 + fi + folder="$(jq -r '(.metadata.annotations // {})["grafana.app/folder"] // ""' "$f")" + kind="$(jq -r '.kind // ""' "$f")" + # A preview FOLDER is allowed exactly one parent: the shared `Previews` + # folder, which has no `pr<n>-` prefix because it is not per-PR. Everything + # else — every dashboard — must sit in this PR's own folder, so that a + # manifest cannot drop a dashboard into a canonical folder. + allowed_parent="" + if [[ "$kind" == "Folder" ]]; then + allowed_parent="$preview_parent_folder" + fi + if [[ -n "$folder" \ + && "$folder" != "pr${pr_number}-"* \ + && ( -z "$allowed_parent" || "$folder" != "$allowed_parent" ) ]]; then + echo "error: $f points at folder '$folder', which is neither this PR's" \ + "folder nor the shared preview parent" >&2 + rm -rf "$rendered" + exit 1 + fi +done < <(find "$rendered" -type f -name '*.json' -print0) + echo "$rendered"