From a57d7b16497ab4dd23048306ea12439ea46e8dc9 Mon Sep 17 00:00:00 2001
From: Buck Doyle <buck.doyle@cardstack.com>
Date: Thu, 17 Sep 2026 11:18:43 +0200
Subject: [PATCH 1/6] Add production previews to the observability PR preview
 workflow

The workflow pushed the changed dashboards to the staging Grafana only. It
now pushes them to the production Grafana also.

- The workflow applies the preview to both environments in one job. The job
  posts one sticky comment. The comment shows each changed dashboard one
  time, with one link for each environment.
- The workflow applies staging first. The comment step runs also when an
  apply step fails. If production fails, the comment keeps the staging links
  and shows a warning.
- The new script scripts/ci-apply-preview.sh holds the steps for one
  environment. Staging and production run the same code.
- Each Grafana token stays in its own step. The workflow writes no token to
  $GITHUB_ENV.
- The cleanup job deletes the preview from both environments when the PR
  closes.
- The daily sweep workflow sweeps both environments.

The production IAM role trusts repo:cardstack/boxel:* with no ref condition.
The role grants the three SSM parameters that the preview reads. This change
needs no infrastructure work.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../workflows/observability-preview-sweep.yml |  42 ++-
 .github/workflows/observability-preview.yml   | 302 +++++++++++-------
 packages/observability/README.md              |   9 +
 .../observability/scripts/apply-preview.sh    |   3 +-
 .../observability/scripts/ci-apply-preview.sh |  91 ++++++
 .../observability/scripts/cleanup-preview.sh  |   5 +-
 .../observability/scripts/render-preview.sh   |  13 +-
 7 files changed, 326 insertions(+), 139 deletions(-)
 create mode 100755 packages/observability/scripts/ci-apply-preview.sh

diff --git a/.github/workflows/observability-preview-sweep.yml b/.github/workflows/observability-preview-sweep.yml
index 2aac837b6a3..98fcc457c20 100644
--- a/.github/workflows/observability-preview-sweep.yml
+++ b/.github/workflows/observability-preview-sweep.yml
@@ -1,10 +1,11 @@
 name: Observability — preview sweep
 
 # Daily sweep that removes stale preview folders + dashboards from the
-# staging Grafana. Safety net for observability-preview.yml's cleanup-on-
-# close job: a missed `closed` webhook, a job that errored mid-delete, or a
-# preview pushed against a PR that was later deleted from GitHub all leave
-# orphan `pr<n>-*` resources behind. This workflow notices and cleans up.
+# staging and production Grafanas. Safety net for
+# observability-preview.yml's cleanup-on-close job: a missed `closed`
+# webhook, a job that errored mid-delete, or a preview pushed against a PR
+# that was later deleted from GitHub all leave orphan `pr<n>-*` resources
+# behind. This workflow notices and cleans up.
 #
 # Decision rule per PR number (derived from the `pr<n>-` UID prefix):
 #   - PR is open                → keep
@@ -41,13 +42,23 @@ concurrency:
 
 env:
   AWS_REGION: us-east-1
-  AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-apply
 
 jobs:
   sweep:
-    name: Sweep stale previews from staging
+    name: Sweep stale previews from ${{ matrix.env }}
     runs-on: ubuntu-latest
 
+    strategy:
+      # One environment's Grafana being unreachable shouldn't stop the other
+      # from being swept — the two legs share nothing but this workflow file.
+      fail-fast: false
+      matrix:
+        include:
+          - env: staging
+            role_arn: arn:aws:iam::680542703984:role/boxel-observability-apply
+          - env: production
+            role_arn: arn:aws:iam::120317779495:role/boxel-observability-apply
+
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -58,13 +69,15 @@ jobs:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
         with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          role-to-assume: ${{ matrix.role_arn }}
           aws-region: ${{ env.AWS_REGION }}
 
       - name: Source GRAFANA_TOKEN
+        env:
+          ENV_NAME: ${{ matrix.env }}
         run: |
           GRAFANA_TOKEN="$(aws ssm get-parameter \
-            --name /staging/grafana/grafanactl_token \
+            --name "/${ENV_NAME}/grafana/grafanactl_token" \
             --with-decryption \
             --query 'Parameter.Value' \
             --output text)"
@@ -77,19 +90,20 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO: ${{ github.repository }}
+          ENV_NAME: ${{ matrix.env }}
         run: |
           set -eo pipefail
           remote="$(mktemp -d)"
-          cfg="$(./scripts/render-config.sh staging)"
+          cfg="$(./scripts/render-config.sh "$ENV_NAME")"
           trap 'rm -rf "$remote"; rm -f "$cfg"' EXIT
 
-          grafanactl --config "$cfg" --context staging \
+          grafanactl --config "$cfg" --context "$ENV_NAME" \
             resources pull dashboards folders --path "$remote" >/dev/null
 
           # Extract the set of unique PR numbers present in preview UIDs.
           # `grep ... || true` because `set -eo pipefail` would otherwise
           # abort the step when grep finds no matches — that's the legitimate
-          # "no previews live in staging" path, not an error.
+          # "no previews live in this environment" path, not an error.
           pr_numbers="$(find "$remote" -type f -name '*.json' \
             -exec jq -r '.metadata.name // ""' {} \; \
             | { grep -E '^pr[0-9]+-' || true; } \
@@ -161,11 +175,12 @@ jobs:
         working-directory: packages/observability
         env:
           STALE_PRS: ${{ steps.identify.outputs.stale_prs }}
+          ENV_NAME: ${{ matrix.env }}
         run: |
           set -eo pipefail
           for pr in $STALE_PRS; do
             echo "::group::Cleanup PR #${pr}"
-            ./scripts/cleanup-preview.sh --pr "$pr" --env staging
+            ./scripts/cleanup-preview.sh --pr "$pr" --env "$ENV_NAME"
             echo "::endgroup::"
           done
 
@@ -173,9 +188,10 @@ jobs:
         if: always()
         env:
           STALE_PRS: ${{ steps.identify.outputs.stale_prs }}
+          ENV_NAME: ${{ matrix.env }}
         run: |
           {
-            echo "## Preview sweep"
+            echo "## Preview sweep (${ENV_NAME})"
             echo ""
             if [[ -z "${STALE_PRS:-}" ]]; then
               echo "No stale previews."
diff --git a/.github/workflows/observability-preview.yml b/.github/workflows/observability-preview.yml
index e9457b6252d..13797a7bc8f 100644
--- a/.github/workflows/observability-preview.yml
+++ b/.github/workflows/observability-preview.yml
@@ -1,11 +1,22 @@
 name: Observability — PR preview
 
-# Spins up a per-PR preview of the dashboards a PR changes inside the staging
-# Grafana, under per-PR-prefixed UIDs (`pr<n>-*`) in a per-PR folder so the
-# preview coexists with the canonical staging copies without overwriting
-# them. On PR close the preview is torn down. A separate sweep workflow
-# (observability-preview-sweep.yml) is the safety net for closes the
-# preview-cleanup job missed.
+# Spins up a per-PR preview of the dashboards a PR changes inside BOTH the
+# staging and the production Grafana, under per-PR-prefixed UIDs (`pr<n>-*`)
+# in a per-PR folder so the preview coexists with the canonical copies
+# without overwriting them. On PR close both previews are torn down. A
+# separate sweep workflow (observability-preview-sweep.yml) is the safety net
+# for closes the preview-cleanup job missed.
+#
+# Both environments are previewed from one job so the PR gets a single
+# sticky comment linking each changed dashboard in both Grafanas, rather
+# than one comment per environment. Staging is applied first: a production
+# Grafana problem then costs the run its red status but not the staging
+# preview or the comment, which is posted from whatever succeeded.
+#
+# The two environments render the same dashboard set under the same preview
+# UIDs — only the apply-time substitutions (realm-server URL, Grafana
+# operator secret) differ, so the comment lists each dashboard once with a
+# link per environment.
 #
 # Scope: dashboards + folders only. Data sources, alert rules, and the
 # home-dashboard preference are file-provisioned at Grafana startup and
@@ -37,19 +48,25 @@ concurrency:
 
 env:
   AWS_REGION: us-east-1
-  # Reuses the apply role from cardstack/infra#665 (same role
-  # observability-apply-staging.yml uses). Permissions cover every SSM path
-  # below — see that workflow's header for the up-to-date list.
-  AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-apply
+  # Reuses the apply roles from cardstack/infra#665 (staging) and #666
+  # (production) — the same roles observability-apply-staging.yml and
+  # observability-apply-production.yml use. Both trust
+  # `repo:cardstack/boxel:*` with no ref condition, so a pull_request-
+  # triggered run can assume them. Permissions cover every SSM path read
+  # below — see those workflows' headers for the up-to-date list.
+  STAGING_AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-apply
+  PRODUCTION_AWS_ROLE_ARN: arn:aws:iam::120317779495:role/boxel-observability-apply
   PREVIEW_MARKER: "<!-- observability-preview -->"
   STAGING_GRAFANA_BASE_URL: "https://dashboard-staging.stack.cards"
+  PRODUCTION_GRAFANA_BASE_URL: "https://dashboard.boxel.ai"
 
 jobs:
   preview-apply:
-    name: Apply preview to staging
-    # Fork-PR guard: this job assumes AWS and pushes to staging Grafana, so
-    # it must not run on PRs whose code we don't trust. Same pattern as
-    # observability-diff.yml and observability-apply-staging.yml.
+    name: Apply preview
+    # Fork-PR guard: this job assumes AWS and pushes to staging and
+    # production Grafana, so it must not run on PRs whose code we don't
+    # trust. Same pattern as observability-diff.yml and
+    # observability-apply-staging.yml.
     if: >-
       github.event.action != 'closed'
       && github.event.pull_request.head.repo.full_name == github.repository
@@ -77,94 +94,97 @@ jobs:
       - name: Install grafanactl
         uses: ./.github/actions/install-grafanactl
 
-      - name: Configure AWS credentials
+      - name: Configure AWS credentials (staging)
         uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
         with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          role-to-assume: ${{ env.STAGING_AWS_ROLE_ARN }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Source preview-time secrets from SSM
-        # Subset of what observability-apply-staging.yml fetches. The preview
-        # only touches dashboards + folders, so we don't need the LOKI_URL /
-        # BOXEL_DB_* / WORKER_LOG_* set — those are read by apply-datasources
-        # / apply-alerting, which apply-preview.sh never invokes.
-        run: |
-          GRAFANA_TOKEN="$(aws ssm get-parameter \
-            --name /staging/grafana/grafanactl_token \
-            --with-decryption \
-            --query 'Parameter.Value' \
-            --output text)"
-          echo "::add-mask::$GRAFANA_TOKEN"
-          echo "GRAFANA_TOKEN=$GRAFANA_TOKEN" >> "$GITHUB_ENV"
-
-          REALM_SERVER_URL="$(aws ssm get-parameter \
-            --name /staging/boxel-grafana/realm_server_url \
-            --query 'Parameter.Value' \
-            --output text)"
-          echo "REALM_SERVER_URL=$REALM_SERVER_URL" >> "$GITHUB_ENV"
-
-          GRAFANA_SECRET="$(aws ssm get-parameter \
-            --name /staging/boxel/GRAFANA_SECRET \
-            --with-decryption \
-            --query 'Parameter.Value' \
-            --output text)"
-          echo "::add-mask::$GRAFANA_SECRET"
-          echo "GRAFANA_SECRET=$GRAFANA_SECRET" >> "$GITHUB_ENV"
-
-      - name: Apply preview
-        id: apply
+      - name: Apply preview to staging
+        id: apply-staging
         working-directory: packages/observability
         env:
+          ENV_NAME: staging
           PR_NUMBER: ${{ github.event.pull_request.number }}
           PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
-        run: |
-          set -eo pipefail
-          OUT_FILE="$(mktemp)"
-          ./scripts/apply-preview.sh \
-            --pr "$PR_NUMBER" \
-            --base-ref "origin/${PR_BASE_REF}" \
-            --env staging \
-            > "$OUT_FILE"
-          echo "out_file=$OUT_FILE" >> "$GITHUB_OUTPUT"
-          if [[ -s "$OUT_FILE" ]]; then
-            echo "empty=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "empty=true" >> "$GITHUB_OUTPUT"
-          fi
-          echo "::group::Preview summary"
-          cat "$OUT_FILE"
-          echo "::endgroup::"
-
-      - name: Cleanup stale preview (no changes left)
-        # When the current head of the PR has no dashboard changes vs base,
-        # apply-preview emits empty output and pushes nothing. But a prior
-        # push to the same PR may have already applied a preview that's
-        # still live in staging — revert/rebase scenarios. Run cleanup so
-        # the PR's staging footprint matches its current head; otherwise
-        # the daily sweep wouldn't catch this until PR close.
-        if: steps.apply.outputs.empty == 'true'
+        run: ./scripts/ci-apply-preview.sh
+
+      # Production is applied after staging, and each environment's
+      # credentials live only in the step that uses them — the AWS creds in
+      # the runner environment and the Grafana token inside the step's own
+      # shell. Nothing is written to $GITHUB_ENV, so there's no window in
+      # which one environment's token is live while the other's push runs.
+      - name: Configure AWS credentials (production)
+        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
+        with:
+          role-to-assume: ${{ env.PRODUCTION_AWS_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Apply preview to production
+        id: apply-production
         working-directory: packages/observability
         env:
+          ENV_NAME: production
           PR_NUMBER: ${{ github.event.pull_request.number }}
-        run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env staging
+          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
+        run: ./scripts/ci-apply-preview.sh
 
       - name: Post sticky PR comment
+        # Runs even when an apply step failed, so a production-side outage
+        # still leaves reviewers the staging links (and a note saying which
+        # environment is missing). The job is still red — the failed step
+        # decides that, not this one.
+        #
         # Pinned to SHA (matches observability-diff.yml's reasoning):
         # pull-requests: write + AWS role = non-trivial supply-chain exposure.
+        if: ${{ !cancelled() }}
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
-          OUT_FILE: ${{ steps.apply.outputs.out_file }}
+          STAGING_OUT_FILE: ${{ steps.apply-staging.outputs.out_file }}
+          STAGING_OUTCOME: ${{ steps.apply-staging.outcome }}
+          PRODUCTION_OUT_FILE: ${{ steps.apply-production.outputs.out_file }}
+          PRODUCTION_OUTCOME: ${{ steps.apply-production.outcome }}
         with:
           script: |
             const fs = require('fs');
             const marker = process.env.PREVIEW_MARKER;
-            const grafanaBase = process.env.STAGING_GRAFANA_BASE_URL;
             const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
 
-            const raw = fs.readFileSync(process.env.OUT_FILE, 'utf8').trim();
+            // Parse one environment's `<KIND>:<uid>:<title>` summary (the
+            // structured stdout of apply-preview.sh). A missing file means
+            // the apply step never got far enough to write one.
+            const readSummary = (outFile) => {
+              if (!outFile || !fs.existsSync(outFile)) return null;
+              const folders = [];
+              const dashboards = [];
+              for (const line of fs.readFileSync(outFile, 'utf8').trim().split('\n')) {
+                // Title may contain `:`, so only the first two fields are
+                // delimited.
+                const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/);
+                if (!m) continue;
+                const [, kind, uid, title] = m;
+                (kind === 'FOLDER' ? folders : dashboards).push({ uid, title });
+              }
+              return { folders, dashboards };
+            };
+
+            const environments = [
+              {
+                name: 'staging',
+                baseUrl: process.env.STAGING_GRAFANA_BASE_URL,
+                outcome: process.env.STAGING_OUTCOME,
+                summary: readSummary(process.env.STAGING_OUT_FILE),
+              },
+              {
+                name: 'production',
+                baseUrl: process.env.PRODUCTION_GRAFANA_BASE_URL,
+                outcome: process.env.PRODUCTION_OUTCOME,
+                summary: readSummary(process.env.PRODUCTION_OUT_FILE),
+              },
+            ];
 
-            // Find any prior preview sticky on this PR; we either update or
-            // delete it based on whether there's anything to preview now.
+            // Find any prior preview sticky on this PR; we either update,
+            // delete, or deliberately leave it alone below.
             const comments = await github.paginate(
               github.rest.issues.listComments,
               {
@@ -178,12 +198,28 @@ jobs:
               c => c.body && c.body.startsWith(marker),
             );
 
-            if (!raw) {
-              // No dashboards changed at this HEAD. Don't post a "nothing
-              // to see" comment — if a prior push left a sticky behind,
-              // delete it so the PR's comment stream matches its current
-              // state. (Stale preview *resources* from a prior push are
-              // handled by the "Cleanup stale preview" step above.)
+            const applied = environments.filter(
+              e => e.outcome === 'success' && e.summary && e.summary.dashboards.length,
+            );
+            const missing = environments.filter(e => e.outcome !== 'success');
+
+            if (!applied.length) {
+              if (missing.length) {
+                // At least one apply failed and nothing is left to link.
+                // Leave any prior sticky in place: preview resources from
+                // an earlier push may well still be live, and the failed
+                // step already makes the run red.
+                core.info(
+                  `No preview to link and ${missing.map(e => e.name).join(', ')} did not apply — leaving any existing comment alone.`,
+                );
+                return;
+              }
+              // Every environment applied cleanly and none had dashboards:
+              // this HEAD changes no dashboards. Don't post a "nothing to
+              // see" comment — if a prior push left a sticky behind, delete
+              // it so the PR's comment stream matches its current state.
+              // (Stale preview *resources* from a prior push are deleted by
+              // ci-apply-preview.sh.)
               if (existing) {
                 await github.rest.issues.deleteComment({
                   owner: context.repo.owner,
@@ -197,36 +233,61 @@ jobs:
               return;
             }
 
-            const folders = [];
-            const dashboards = [];
-            for (const line of raw.split('\n')) {
-              // Format: `<KIND>:<uid>:<title>` (title may contain `:`).
-              const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/);
-              if (!m) continue;
-              const [, kind, uid, title] = m;
-              (kind === 'FOLDER' ? folders : dashboards).push({ uid, title });
-            }
+            // Both environments render the same preview UIDs, so collapse
+            // them into one row per resource carrying a link per
+            // environment. Keyed by UID and built in the order the first
+            // applied environment reported, so a partial apply still lists
+            // everything that made it somewhere.
+            const collapse = (key) => {
+              const byUid = new Map();
+              for (const env of applied) {
+                for (const { uid, title } of env.summary[key]) {
+                  if (!byUid.has(uid)) byUid.set(uid, { title, envs: [] });
+                  byUid.get(uid).envs.push(env);
+                }
+              }
+              return [...byUid.entries()].map(([uid, v]) => ({ uid, ...v }));
+            };
 
-            const folderLines = folders.map(f =>
-              `- **Folder:** [${f.title}](${grafanaBase}/dashboards/f/${f.uid}/)`,
+            const links = (envs, path) =>
+              envs
+                .map(e => `[${e.name}](${e.baseUrl}${path})`)
+                .join(' · ');
+
+            const folders = collapse('folders');
+            const dashboards = collapse('dashboards');
+
+            const folderLines = folders.map(
+              f => `- **Folder:** ${f.title} — ${links(f.envs, `/dashboards/f/${f.uid}/`)}`,
             );
-            const dashboardLines = dashboards.map(d =>
-              `- [${d.title}](${grafanaBase}/d/${d.uid}/)`,
+            const dashboardLines = dashboards.map(
+              d => `- ${d.title} — ${links(d.envs, `/d/${d.uid}/`)}`,
             );
 
+            const appliedNames = applied.map(e => e.name).join(' and ');
+
+            const warning = missing.length
+              ? [
+                  '',
+                  '> [!WARNING]',
+                  `> The **${missing.map(e => e.name).join('** and **')}** preview did not apply on this run — see the run log below.`,
+                ]
+              : [];
+
             const body = [
               marker,
               '## Grafana preview',
               '',
-              `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in the staging Grafana.`,
-              'Cross-dashboard drill-throughs still point at the canonical staging dashboards.',
+              `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in ${appliedNames}.`,
+              "Cross-dashboard drill-throughs still point at each environment's canonical dashboards.",
+              ...warning,
               '',
               ...folderLines,
               '',
               '**Dashboards:**',
               ...dashboardLines,
               '',
-              'Preview is torn down automatically when this PR is closed or merged.',
+              'Previews are torn down automatically when this PR is closed or merged.',
               '',
               `_(Run: ${runUrl})_`,
             ].join('\n');
@@ -250,9 +311,10 @@ jobs:
             }
 
   preview-cleanup:
-    name: Cleanup preview from staging
+    name: Cleanup preview
     # Cleanup runs on the `closed` action (covers both merge and close-
-    # without-merge). Same fork-PR guard — cleanup hits staging.
+    # without-merge). Same fork-PR guard — cleanup hits staging and
+    # production.
     if: >-
       github.event.action == 'closed'
       && github.event.pull_request.head.repo.full_name == github.repository
@@ -264,36 +326,40 @@ jobs:
       - name: Install grafanactl
         uses: ./.github/actions/install-grafanactl
 
-      - name: Configure AWS credentials
+      - name: Configure AWS credentials (staging)
         uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
         with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          role-to-assume: ${{ env.STAGING_AWS_ROLE_ARN }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Source GRAFANA_TOKEN
-        # Cleanup doesn't need any of the URL/secret substitution values —
-        # it lists live state and deletes by selector. Only the grafanactl
-        # token matters.
-        run: |
-          GRAFANA_TOKEN="$(aws ssm get-parameter \
-            --name /staging/grafana/grafanactl_token \
-            --with-decryption \
-            --query 'Parameter.Value' \
-            --output text)"
-          echo "::add-mask::$GRAFANA_TOKEN"
-          echo "GRAFANA_TOKEN=$GRAFANA_TOKEN" >> "$GITHUB_ENV"
-
-      - name: Cleanup preview
+      - name: Cleanup preview from staging
         working-directory: packages/observability
         env:
           PR_NUMBER: ${{ github.event.pull_request.number }}
         run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env staging
 
+      - name: Configure AWS credentials (production)
+        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
+        with:
+          role-to-assume: ${{ env.PRODUCTION_AWS_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Cleanup preview from production
+        working-directory: packages/observability
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env production
+
       - name: Delete sticky PR comment
         # After cleanup, there's nothing left to point at — delete the
         # sticky entirely rather than leaving a "preview cleaned up"
         # gravestone. Matches the apply-time policy of not posting a
         # comment when there's nothing to show.
+        #
+        # Runs even if a cleanup step failed: the daily sweep
+        # (observability-preview-sweep.yml) will catch whatever survived,
+        # and a comment pointing at deleted dashboards is worse than none.
+        if: ${{ !cancelled() }}
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
diff --git a/packages/observability/README.md b/packages/observability/README.md
index fffc7d01aa0..7916c1c1412 100644
--- a/packages/observability/README.md
+++ b/packages/observability/README.md
@@ -452,6 +452,15 @@ CI runs `apply.sh --env staging` on merge to main (workflow:
 `apply.sh --env production` on the production workflow
 (CS-10936).
 
+A PR that changes anything under `grafanactl/resources/dashboards/` also gets
+a live preview of just the dashboards it touches, pushed to **both** the
+staging and the production Grafana under `pr<n>-`-prefixed UIDs in a per-PR
+folder, so it coexists with the canonical copies rather than overwriting them
+(`.github/workflows/observability-preview.yml`). One sticky PR comment links
+each changed dashboard in both environments. Previews are deleted when the PR
+closes; `.github/workflows/observability-preview-sweep.yml` sweeps both
+environments daily for any the close hook missed.
+
 ## Operator actions
 
 Some dashboards carry buttons that POST to a realm-server operator endpoint —
diff --git a/packages/observability/scripts/apply-preview.sh b/packages/observability/scripts/apply-preview.sh
index 86fb4c1eb66..69d533d7a00 100755
--- a/packages/observability/scripts/apply-preview.sh
+++ b/packages/observability/scripts/apply-preview.sh
@@ -3,7 +3,8 @@
 #
 # Wraps render-preview.sh (changed-dashboard filtering + UID rewriting) and
 # `grafanactl resources push`. Used by the observability-preview.yml workflow
-# on PR open/synchronize/reopen.
+# on PR open/synchronize/reopen, once per environment (see
+# ci-apply-preview.sh).
 #
 # Usage:
 #   ./scripts/apply-preview.sh --pr <n> --base-ref <ref> [--env staging|local|production]
diff --git a/packages/observability/scripts/ci-apply-preview.sh b/packages/observability/scripts/ci-apply-preview.sh
new file mode 100755
index 00000000000..4a58d6ed755
--- /dev/null
+++ b/packages/observability/scripts/ci-apply-preview.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# ci-apply-preview.sh — one environment's half of observability-preview.yml's
+# apply job.
+#
+# Fetches the apply-time secrets for $ENV_NAME from SSM, pushes the per-PR
+# preview with apply-preview.sh, and reports the result back to the workflow
+# on $GITHUB_OUTPUT:
+#
+#   out_file=<path>   structured `<KIND>:<uid>:<title>` summary (see
+#                     apply-preview.sh); empty file when nothing was pushed
+#   empty=true|false  whether this PR's HEAD changes any dashboards
+#
+# Lives here rather than inline in the workflow so staging and production
+# run byte-identical logic — the environments differ only in $ENV_NAME and
+# in which AWS credentials the workflow configured before calling this.
+#
+# Environment (all set by the workflow step):
+#   ENV_NAME      staging | production
+#   PR_NUMBER     the pull request number
+#   PR_BASE_REF   the PR's base branch name (e.g. `main`), without `origin/`
+#
+# Prereqs: AWS credentials for $ENV_NAME already configured, grafanactl on
+# PATH, CWD anywhere inside the repo.
+set -eo pipefail
+
+require_env() {
+  [[ -n "${!1:-}" ]] || { echo "error: $1 is required" >&2; exit 2; }
+}
+
+require_env ENV_NAME
+require_env PR_NUMBER
+require_env PR_BASE_REF
+require_env GITHUB_OUTPUT
+
+case "$ENV_NAME" in
+  staging | production) ;;
+  *) echo "error: ENV_NAME must be staging or production (got: $ENV_NAME)" >&2; exit 2 ;;
+esac
+
+cd "$(dirname "$0")/.."
+
+# Subset of what observability-apply-<env>.yml fetches. The preview only
+# touches dashboards + folders, so we don't need the LOKI_URL / BOXEL_DB_* /
+# WORKER_LOG_* set — those are read by apply-datasources / apply-alerting,
+# which apply-preview.sh never invokes.
+#
+# GRAFANA_TOKEN is re-fetched by grafanactl-env.sh inside apply-preview.sh;
+# fetching it here too is what registers the `::add-mask::`, so the value is
+# redacted if grafanactl ever echoes it on a failure path.
+ssm() {
+  aws ssm get-parameter --name "$1" "${@:2}" \
+    --query 'Parameter.Value' --output text
+}
+
+GRAFANA_TOKEN="$(ssm "/${ENV_NAME}/grafana/grafanactl_token" --with-decryption)"
+echo "::add-mask::$GRAFANA_TOKEN"
+
+GRAFANA_SECRET="$(ssm "/${ENV_NAME}/boxel/GRAFANA_SECRET" --with-decryption)"
+echo "::add-mask::$GRAFANA_SECRET"
+
+REALM_SERVER_URL="$(ssm "/${ENV_NAME}/boxel-grafana/realm_server_url")"
+
+export GRAFANA_TOKEN GRAFANA_SECRET REALM_SERVER_URL
+
+out_file="$(mktemp)"
+echo "out_file=$out_file" >> "$GITHUB_OUTPUT"
+
+./scripts/apply-preview.sh \
+  --pr "$PR_NUMBER" \
+  --base-ref "origin/${PR_BASE_REF}" \
+  --env "$ENV_NAME" \
+  > "$out_file"
+
+if [[ -s "$out_file" ]]; then
+  echo "empty=false" >> "$GITHUB_OUTPUT"
+  echo "::group::Preview summary (${ENV_NAME})"
+  cat "$out_file"
+  echo "::endgroup::"
+  exit 0
+fi
+
+echo "empty=true" >> "$GITHUB_OUTPUT"
+
+# When the current head of the PR has no dashboard changes vs base,
+# apply-preview emits empty output and pushes nothing. But a prior push to
+# the same PR may have already applied a preview that's still live — revert /
+# rebase scenarios. Run cleanup so the PR's footprint in this environment
+# matches its current head; otherwise the daily sweep wouldn't catch this
+# until PR close.
+echo "no dashboard changes at this HEAD — clearing any preview left in ${ENV_NAME}"
+./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env "$ENV_NAME"
diff --git a/packages/observability/scripts/cleanup-preview.sh b/packages/observability/scripts/cleanup-preview.sh
index 529bcfba85f..297a46f69dd 100755
--- a/packages/observability/scripts/cleanup-preview.sh
+++ b/packages/observability/scripts/cleanup-preview.sh
@@ -21,7 +21,10 @@
 #   - apply-preview.sh runs this before every push, so a PR that shrinks
 #     its changed-dashboard set (e.g. reverts one of two prior edits)
 #     gets the old preview resources removed before the new tree lands.
-#   - observability-preview.yml runs this on pull_request `closed`.
+#   - ci-apply-preview.sh runs this when a PR stops changing dashboards
+#     altogether, to clear a preview an earlier push left behind.
+#   - observability-preview.yml runs this on pull_request `closed`, once
+#     per environment.
 #   - observability-preview-sweep.yml runs this for PRs the daily sweep
 #     has decided are stale.
 set -eo pipefail
diff --git a/packages/observability/scripts/render-preview.sh b/packages/observability/scripts/render-preview.sh
index aaaf68d30b0..d4c7b975312 100755
--- a/packages/observability/scripts/render-preview.sh
+++ b/packages/observability/scripts/render-preview.sh
@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
-# render-preview.sh — Render a per-PR preview tree for staging Grafana.
+# render-preview.sh — Render a per-PR preview tree for a Grafana environment.
 #
 # Output: prints a tempdir path on stdout. The tree contains only the
 # dashboards the PR changed (vs --base-ref) plus the folder(s) they live in,
 # with `metadata.name` (UID), `spec.title`, and each dashboard's
 # `metadata.annotations["grafana.app/folder"]` rewritten so the resources
-# can coexist with the canonical staging copies. Push or delete the result
-# with `grafanactl resources push|delete --path <out>`.
+# can coexist with the canonical copies in whichever environment they are
+# pushed to. Push or delete the result with
+# `grafanactl resources push|delete --path <out>`.
 #
 # When the PR doesn't change any dashboards, exits 0 with no stdout.
 #
@@ -24,9 +25,9 @@
 #
 # Cross-dashboard drill-through links inside the dashboard JSON (`/d/<uid>/`
 # strings) are intentionally NOT rewritten — they stay pointing at the
-# canonical staging dashboards (CS-11106 design call). UID references inside
-# `datasource.uid` fields are data-source UIDs, not dashboard UIDs, and are
-# also left alone.
+# canonical dashboards of whichever environment the preview lands in
+# (CS-11106 design call). UID references inside `datasource.uid` fields are
+# data-source UIDs, not dashboard UIDs, and are also left alone.
 set -eo pipefail
 
 usage_error() { echo "error: $1" >&2; exit 2; }

From 299d13791c84a09fd0fa2d75aab06d557040f275 Mon Sep 17 00:00:00 2001
From: Buck Doyle <buck.doyle@cardstack.com>
Date: Thu, 17 Sep 2026 11:47:42 +0200
Subject: [PATCH 2/6] Move the production preview into a workflow_run workflow

A code review found that the previous design gave production credentials to
pull-request-controlled code. GitHub loads a `pull_request` workflow, and
every script that workflow calls, from the pull request itself. A contributor
could therefore change those files and read the temporary production AWS
credentials, the decrypted production grafanactl token, and GRAFANA_SECRET.

GitHub always loads a `workflow_run` workflow from the default branch. The
production preview moves there:

- observability-preview.yml keeps staging credentials only.
- observability-preview-production.yml applies the production preview, and
  removes it when the pull request closes. It restores every script from the
  default branch, then overlays only
  packages/observability/grafanactl/resources from the pull request. Those
  manifests are data, and no step runs them.
- render-preview.sh fails now if a manifest that it emits lacks the `pr<n>-`
  prefix. A crafted manifest cannot address a canonical dashboard.
- The production workflow asserts the production server URL before it reads
  a credential.
- The production sweep refuses a ref other than main.

The staging workflow posts a comment for staging. The production workflow
rewrites that comment with one link for each environment.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../observability-preview-production.yml      | 328 ++++++++++++++++++
 .../workflows/observability-preview-sweep.yml |  17 +
 .github/workflows/observability-preview.yml   | 265 +++++---------
 packages/observability/README.md              |  22 +-
 .../observability/scripts/ci-apply-preview.sh |  12 +-
 .../observability/scripts/cleanup-preview.sh  |   5 +-
 .../observability/scripts/render-preview.sh   |  22 ++
 7 files changed, 477 insertions(+), 194 deletions(-)
 create mode 100644 .github/workflows/observability-preview-production.yml

diff --git a/.github/workflows/observability-preview-production.yml b/.github/workflows/observability-preview-production.yml
new file mode 100644
index 00000000000..48d78345c9b
--- /dev/null
+++ b/.github/workflows/observability-preview-production.yml
@@ -0,0 +1,328 @@
+name: Observability — PR preview (production)
+
+# Applies the per-PR dashboard preview to the PRODUCTION Grafana, and tears
+# it down again when the PR closes.
+#
+# WHY THIS IS A SEPARATE WORKFLOW
+#
+# observability-preview.yml runs on `pull_request`. GitHub loads that file,
+# and every script it calls, from the pull request's own merge ref — a
+# contributor controls both. Giving it the production role would let any
+# same-repository PR run arbitrary code with production credentials live and
+# exfiltrate the temporary AWS credentials, the decrypted production
+# grafanactl token, and GRAFANA_SECRET. It would also defeat the main-only
+# guard in observability-apply-production.yml.
+#
+# GitHub always loads a `workflow_run` workflow from the default branch, no
+# matter which branch the triggering run used. This file is therefore out of
+# a pull request's reach, and it is the only place production credentials are
+# safe to use for a preview.
+#
+# THE TRUST BOUNDARY
+#
+# The one thing this job takes from the pull request is the contents of
+# packages/observability/grafanactl/resources — dashboard and folder
+# manifests. Those are data, not code: nothing here executes them, and
+# render-preview.sh (restored from the default branch, like every other
+# script) rewrites every UID it emits to the `pr<n>-` form, so a crafted
+# manifest cannot address a canonical production dashboard. The "Overlay"
+# step below is what enforces that split; read it before changing it.
+
+on:
+  workflow_run:
+    workflows: ["Observability — PR preview"]
+    types: [completed]
+
+permissions:
+  contents: read
+  id-token: write
+  pull-requests: write
+
+concurrency:
+  # Serialize per branch: a force-push while a previous production apply is
+  # mid-flight must not race it. Never cancel in-flight — that could leave
+  # the production Grafana half-pushed.
+  group: observability-preview-production-${{ github.event.workflow_run.head_branch }}
+  cancel-in-progress: false
+
+env:
+  AWS_REGION: us-east-1
+  # The same narrowly-scoped role observability-apply-production.yml uses
+  # (cardstack/infra#666). Permissions are ssm:GetParameter on the three
+  # paths ci-apply-preview.sh reads for production.
+  AWS_ROLE_ARN: arn:aws:iam::120317779495:role/boxel-observability-apply
+  EXPECTED_GRAFANA_HOSTNAME: dashboard.boxel.ai
+  PREVIEW_MARKER: "<!-- observability-preview -->"
+  STAGING_GRAFANA_BASE_URL: "https://dashboard-staging.stack.cards"
+  PRODUCTION_GRAFANA_BASE_URL: "https://dashboard.boxel.ai"
+
+jobs:
+  preview-production:
+    name: Apply preview to production
+    # Fork guard. `workflow_run` fires even when the triggering job itself
+    # was skipped, so this has to be re-checked here rather than inherited.
+    if: >-
+      github.event.workflow_run.head_repository.full_name == github.repository
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the default branch
+        # Trusted code. Everything this job executes comes from here; the
+        # pull request's manifests are overlaid further down, after
+        # grafanactl is already installed.
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          fetch-depth: 0
+
+      - name: Install grafanactl
+        # Runs while the workspace still holds only default-branch code, so
+        # this `uses: ./` cannot resolve to a pull-request-authored action.
+        uses: ./.github/actions/install-grafanactl
+
+      - name: Resolve the pull request
+        id: pr
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          set -eo pipefail
+          # Ask GitHub which PR this run belongs to rather than trusting
+          # anything the triggering run produced. `workflow_run` does carry a
+          # `pull_requests` array, but it is empty often enough that it can't
+          # be relied on.
+          pr_json="$(gh api "repos/${REPO}/commits/${HEAD_SHA}/pulls" \
+            --jq '[.[] | select(.head.sha == env.HEAD_SHA)][0] // empty')"
+
+          if [[ -z "$pr_json" ]]; then
+            echo "No pull request found for ${HEAD_SHA} — nothing to do."
+            echo "found=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          number="$(jq -r '.number' <<< "$pr_json")"
+          base_ref="$(jq -r '.base.ref' <<< "$pr_json")"
+          state="$(jq -r '.state' <<< "$pr_json")"
+          head_repo="$(jq -r '.head.repo.full_name // ""' <<< "$pr_json")"
+
+          # Second fork guard, on the PR record itself.
+          if [[ "$head_repo" != "$REPO" ]]; then
+            echo "PR #${number} comes from ${head_repo} — refusing to touch production."
+            echo "found=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          echo "PR #${number}, base ${base_ref}, state ${state}"
+          {
+            echo "found=true"
+            echo "number=${number}"
+            echo "base_ref=${base_ref}"
+            echo "state=${state}"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Overlay the pull request's dashboard manifests
+        # THE TRUST BOUNDARY. After this step the working tree holds
+        # default-branch code everywhere except
+        # packages/observability/grafanactl/resources, which holds the pull
+        # request's manifests. HEAD is the pull request's commit so that
+        # render-preview.sh's `git diff <base>...HEAD` still sees the right
+        # changed set.
+        #
+        # Skipped for a closed PR: teardown reads live Grafana state and
+        # needs no manifests at all.
+        if: steps.pr.outputs.found == 'true' && steps.pr.outputs.state == 'open'
+        env:
+          PR_NUMBER: ${{ steps.pr.outputs.number }}
+          PR_BASE_REF: ${{ steps.pr.outputs.base_ref }}
+        run: |
+          set -eo pipefail
+          main_sha="$(git rev-parse HEAD)"
+
+          git fetch --no-tags origin "pull/${PR_NUMBER}/head"
+          git checkout --quiet --detach FETCH_HEAD
+
+          # Restore every path this job executes or reads for configuration.
+          # packages/observability covers the scripts and grafanactl/config.yaml
+          # (which names the production server URL); .github covers any action
+          # a later step might resolve through `uses: ./`.
+          git checkout --quiet "$main_sha" -- .github packages/observability
+
+          # ...then take back ONLY the manifests. This is the sole pull-request
+          # input to the production push.
+          git checkout --quiet FETCH_HEAD -- packages/observability/grafanactl/resources
+
+          git fetch --no-tags origin "$PR_BASE_REF"
+
+          echo "Overlaid manifests from PR #${PR_NUMBER} onto ${main_sha}."
+
+      - name: Pre-flight — assert config.yaml's production server URL
+        # Runs before any credential is fetched. config.yaml comes from the
+        # default branch (restored above), so this catches a config-as-code
+        # mistake rather than a hostile edit — but it is the check that makes
+        # "the token can only go to the real production Grafana" verifiable.
+        #
+        # The awk uses an in-block flag rather than awk's range syntax
+        # (`/start/,/end/`), because `/^  production:$/,/^  [a-z]/` would
+        # match only the start line — the end pattern `^  [a-z]` matches the
+        # start line itself, terminating the range immediately.
+        if: steps.pr.outputs.found == 'true'
+        working-directory: packages/observability
+        run: |
+          set -eo pipefail
+          ACTUAL="$(awk '
+            /^  production:$/      { in_prod=1; next }
+            in_prod && /^  [a-z]/  { exit }
+            in_prod && /^      server:/ { print $2; exit }
+          ' grafanactl/config.yaml)"
+          echo "config.yaml production server: $ACTUAL"
+          echo "expected:                       https://${EXPECTED_GRAFANA_HOSTNAME}"
+          if [[ "$ACTUAL" != "https://${EXPECTED_GRAFANA_HOSTNAME}" ]]; then
+            echo "::error::config.yaml's production context server URL doesn't match expected hostname." >&2
+            echo "If this is an intentional rename, update EXPECTED_GRAFANA_HOSTNAME in this workflow to match." >&2
+            exit 1
+          fi
+
+      - name: Configure AWS credentials
+        if: steps.pr.outputs.found == 'true'
+        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
+        with:
+          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Apply preview
+        id: apply
+        if: steps.pr.outputs.found == 'true' && steps.pr.outputs.state == 'open'
+        working-directory: packages/observability
+        env:
+          ENV_NAME: production
+          PR_NUMBER: ${{ steps.pr.outputs.number }}
+          PR_BASE_REF: ${{ steps.pr.outputs.base_ref }}
+        run: ./scripts/ci-apply-preview.sh
+
+      - name: Cleanup preview
+        if: steps.pr.outputs.found == 'true' && steps.pr.outputs.state == 'closed'
+        working-directory: packages/observability
+        env:
+          PR_NUMBER: ${{ steps.pr.outputs.number }}
+        run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env production
+
+      - name: Rewrite the sticky PR comment
+        # observability-preview.yml already posted a staging-only sticky.
+        # Replace it with one covering both environments. The preview UIDs and
+        # titles are environment-independent — render-preview.sh varies only
+        # the apply-time substitutions — so this job's own summary describes
+        # the staging resources just as well as the production ones.
+        if: steps.pr.outputs.found == 'true' && steps.pr.outputs.state == 'open'
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          OUT_FILE: ${{ steps.apply.outputs.out_file }}
+          PR_NUMBER: ${{ steps.pr.outputs.number }}
+          STAGING_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
+          STAGING_RUN_URL: ${{ github.event.workflow_run.html_url }}
+        with:
+          script: |
+            const fs = require('fs');
+            const marker = process.env.PREVIEW_MARKER;
+            const prNumber = Number(process.env.PR_NUMBER);
+            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+
+            const outFile = process.env.OUT_FILE;
+            const raw = outFile && fs.existsSync(outFile)
+              ? fs.readFileSync(outFile, 'utf8').trim()
+              : '';
+
+            if (!raw) {
+              // This HEAD changes no dashboards, so there is nothing to link
+              // and the staging job already removed any stale sticky.
+              core.info('No dashboard changes — leaving the comment alone.');
+              return;
+            }
+
+            const folders = [];
+            const dashboards = [];
+            for (const line of raw.split('\n')) {
+              // Format: `<KIND>:<uid>:<title>` (title may contain `:`).
+              const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/);
+              if (!m) continue;
+              const [, kind, uid, title] = m;
+              (kind === 'FOLDER' ? folders : dashboards).push({ uid, title });
+            }
+
+            const stagingOk = process.env.STAGING_CONCLUSION === 'success';
+            const environments = [
+              ...(stagingOk
+                ? [{ name: 'staging', baseUrl: process.env.STAGING_GRAFANA_BASE_URL }]
+                : []),
+              { name: 'production', baseUrl: process.env.PRODUCTION_GRAFANA_BASE_URL },
+            ];
+
+            const links = (path) =>
+              environments
+                .map(e => `[${e.name}](${e.baseUrl}${path})`)
+                .join(' · ');
+
+            const folderLines = folders.map(
+              f => `- **Folder:** ${f.title} — ${links(`/dashboards/f/${f.uid}/`)}`,
+            );
+            const dashboardLines = dashboards.map(
+              d => `- ${d.title} — ${links(`/d/${d.uid}/`)}`,
+            );
+
+            const warning = stagingOk
+              ? []
+              : [
+                  '',
+                  '> [!WARNING]',
+                  `> The **staging** preview did not apply — see [its run](${process.env.STAGING_RUN_URL}). Only the production links are listed.`,
+                ];
+
+            const envNames = environments.map(e => e.name).join(' and ');
+
+            const body = [
+              marker,
+              '## Grafana preview',
+              '',
+              `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in ${envNames}.`,
+              "Cross-dashboard drill-throughs still point at each environment's canonical dashboards.",
+              ...warning,
+              '',
+              ...folderLines,
+              '',
+              '**Dashboards:**',
+              ...dashboardLines,
+              '',
+              'Previews are torn down automatically when this PR is closed or merged.',
+              '',
+              `_(Run: ${runUrl})_`,
+            ].join('\n');
+
+            const comments = await github.paginate(
+              github.rest.issues.listComments,
+              {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                per_page: 100,
+              },
+            );
+            const existing = comments.find(
+              c => c.body && c.body.startsWith(marker),
+            );
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+              core.info(`Updated sticky comment ${existing.id}`);
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body,
+              });
+              core.info('Created sticky comment');
+            }
diff --git a/.github/workflows/observability-preview-sweep.yml b/.github/workflows/observability-preview-sweep.yml
index 98fcc457c20..119e7d59021 100644
--- a/.github/workflows/observability-preview-sweep.yml
+++ b/.github/workflows/observability-preview-sweep.yml
@@ -60,6 +60,23 @@ jobs:
             role_arn: arn:aws:iam::120317779495:role/boxel-observability-apply
 
     steps:
+      - name: Refuse to sweep production from a non-main ref
+        # `schedule` always runs the default branch, but
+        # `workflow_dispatch` lets an operator choose any ref — including
+        # one whose copy of this workflow was rewritten to do something
+        # else with the production role. Mirrors the guard in
+        # observability-apply-production.yml.
+        if: matrix.env == 'production'
+        env:
+          REF: ${{ github.ref }}
+        run: |
+          set -eo pipefail
+          if [[ "$REF" != "refs/heads/main" ]]; then
+            echo "::error::The production sweep may only run from refs/heads/main. Got: $REF" >&2
+            exit 1
+          fi
+          echo "Confirmed run from main."
+
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
diff --git a/.github/workflows/observability-preview.yml b/.github/workflows/observability-preview.yml
index 13797a7bc8f..58703443e78 100644
--- a/.github/workflows/observability-preview.yml
+++ b/.github/workflows/observability-preview.yml
@@ -1,22 +1,23 @@
 name: Observability — PR preview
 
-# Spins up a per-PR preview of the dashboards a PR changes inside BOTH the
-# staging and the production Grafana, under per-PR-prefixed UIDs (`pr<n>-*`)
-# in a per-PR folder so the preview coexists with the canonical copies
-# without overwriting them. On PR close both previews are torn down. A
-# separate sweep workflow (observability-preview-sweep.yml) is the safety net
-# for closes the preview-cleanup job missed.
+# Spins up a per-PR preview of the dashboards a PR changes inside the staging
+# Grafana, under per-PR-prefixed UIDs (`pr<n>-*`) in a per-PR folder so the
+# preview coexists with the canonical staging copies without overwriting
+# them. On PR close the preview is torn down. A separate sweep workflow
+# (observability-preview-sweep.yml) is the safety net for closes the
+# preview-cleanup job missed.
 #
-# Both environments are previewed from one job so the PR gets a single
-# sticky comment linking each changed dashboard in both Grafanas, rather
-# than one comment per environment. Staging is applied first: a production
-# Grafana problem then costs the run its red status but not the staging
-# preview or the comment, which is posted from whatever succeeded.
+# This workflow runs on `pull_request`, so GitHub loads it — and every script
+# it calls — from the pull request's own merge ref, where a contributor can
+# change both. It therefore handles STAGING CREDENTIALS ONLY. The production
+# preview lives in observability-preview-production.yml, which GitHub starts
+# with `workflow_run` and always loads from the default branch, out of a pull
+# request's reach. Never add a production role, a production SSM read, or a
+# production Grafana push to this file.
 #
-# The two environments render the same dashboard set under the same preview
-# UIDs — only the apply-time substitutions (realm-server URL, Grafana
-# operator secret) differ, so the comment lists each dashboard once with a
-# link per environment.
+# That production workflow rewrites the sticky comment below into one that
+# covers both environments. While it is still running, or when it fails, the
+# staging-only comment stands on its own.
 #
 # Scope: dashboards + folders only. Data sources, alert rules, and the
 # home-dashboard preference are file-provisioned at Grafana startup and
@@ -48,25 +49,20 @@ concurrency:
 
 env:
   AWS_REGION: us-east-1
-  # Reuses the apply roles from cardstack/infra#665 (staging) and #666
-  # (production) — the same roles observability-apply-staging.yml and
-  # observability-apply-production.yml use. Both trust
-  # `repo:cardstack/boxel:*` with no ref condition, so a pull_request-
-  # triggered run can assume them. Permissions cover every SSM path read
-  # below — see those workflows' headers for the up-to-date list.
-  STAGING_AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-apply
-  PRODUCTION_AWS_ROLE_ARN: arn:aws:iam::120317779495:role/boxel-observability-apply
+  # Reuses the apply role from cardstack/infra#665 (same role
+  # observability-apply-staging.yml uses). Permissions cover every SSM path
+  # ci-apply-preview.sh reads — see that workflow's header for the up-to-date
+  # list.
+  AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-apply
   PREVIEW_MARKER: "<!-- observability-preview -->"
   STAGING_GRAFANA_BASE_URL: "https://dashboard-staging.stack.cards"
-  PRODUCTION_GRAFANA_BASE_URL: "https://dashboard.boxel.ai"
 
 jobs:
   preview-apply:
-    name: Apply preview
-    # Fork-PR guard: this job assumes AWS and pushes to staging and
-    # production Grafana, so it must not run on PRs whose code we don't
-    # trust. Same pattern as observability-diff.yml and
-    # observability-apply-staging.yml.
+    name: Apply preview to staging
+    # Fork-PR guard: this job assumes AWS and pushes to staging Grafana, so
+    # it must not run on PRs whose code we don't trust. Same pattern as
+    # observability-diff.yml and observability-apply-staging.yml.
     if: >-
       github.event.action != 'closed'
       && github.event.pull_request.head.repo.full_name == github.repository
@@ -94,14 +90,14 @@ jobs:
       - name: Install grafanactl
         uses: ./.github/actions/install-grafanactl
 
-      - name: Configure AWS credentials (staging)
+      - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
         with:
-          role-to-assume: ${{ env.STAGING_AWS_ROLE_ARN }}
+          role-to-assume: ${{ env.AWS_ROLE_ARN }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Apply preview to staging
-        id: apply-staging
+      - name: Apply preview
+        id: apply
         working-directory: packages/observability
         env:
           ENV_NAME: staging
@@ -109,82 +105,26 @@ jobs:
           PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
         run: ./scripts/ci-apply-preview.sh
 
-      # Production is applied after staging, and each environment's
-      # credentials live only in the step that uses them — the AWS creds in
-      # the runner environment and the Grafana token inside the step's own
-      # shell. Nothing is written to $GITHUB_ENV, so there's no window in
-      # which one environment's token is live while the other's push runs.
-      - name: Configure AWS credentials (production)
-        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
-        with:
-          role-to-assume: ${{ env.PRODUCTION_AWS_ROLE_ARN }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Apply preview to production
-        id: apply-production
-        working-directory: packages/observability
-        env:
-          ENV_NAME: production
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
-        run: ./scripts/ci-apply-preview.sh
-
       - name: Post sticky PR comment
-        # Runs even when an apply step failed, so a production-side outage
-        # still leaves reviewers the staging links (and a note saying which
-        # environment is missing). The job is still red — the failed step
-        # decides that, not this one.
-        #
         # Pinned to SHA (matches observability-diff.yml's reasoning):
         # pull-requests: write + AWS role = non-trivial supply-chain exposure.
-        if: ${{ !cancelled() }}
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
-          STAGING_OUT_FILE: ${{ steps.apply-staging.outputs.out_file }}
-          STAGING_OUTCOME: ${{ steps.apply-staging.outcome }}
-          PRODUCTION_OUT_FILE: ${{ steps.apply-production.outputs.out_file }}
-          PRODUCTION_OUTCOME: ${{ steps.apply-production.outcome }}
+          OUT_FILE: ${{ steps.apply.outputs.out_file }}
         with:
           script: |
             const fs = require('fs');
             const marker = process.env.PREVIEW_MARKER;
+            const grafanaBase = process.env.STAGING_GRAFANA_BASE_URL;
             const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
 
-            // Parse one environment's `<KIND>:<uid>:<title>` summary (the
-            // structured stdout of apply-preview.sh). A missing file means
-            // the apply step never got far enough to write one.
-            const readSummary = (outFile) => {
-              if (!outFile || !fs.existsSync(outFile)) return null;
-              const folders = [];
-              const dashboards = [];
-              for (const line of fs.readFileSync(outFile, 'utf8').trim().split('\n')) {
-                // Title may contain `:`, so only the first two fields are
-                // delimited.
-                const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/);
-                if (!m) continue;
-                const [, kind, uid, title] = m;
-                (kind === 'FOLDER' ? folders : dashboards).push({ uid, title });
-              }
-              return { folders, dashboards };
-            };
-
-            const environments = [
-              {
-                name: 'staging',
-                baseUrl: process.env.STAGING_GRAFANA_BASE_URL,
-                outcome: process.env.STAGING_OUTCOME,
-                summary: readSummary(process.env.STAGING_OUT_FILE),
-              },
-              {
-                name: 'production',
-                baseUrl: process.env.PRODUCTION_GRAFANA_BASE_URL,
-                outcome: process.env.PRODUCTION_OUTCOME,
-                summary: readSummary(process.env.PRODUCTION_OUT_FILE),
-              },
-            ];
+            const outFile = process.env.OUT_FILE;
+            const raw = outFile && fs.existsSync(outFile)
+              ? fs.readFileSync(outFile, 'utf8').trim()
+              : '';
 
-            // Find any prior preview sticky on this PR; we either update,
-            // delete, or deliberately leave it alone below.
+            // Find any prior preview sticky on this PR; we either update or
+            // delete it based on whether there's anything to preview now.
             const comments = await github.paginate(
               github.rest.issues.listComments,
               {
@@ -198,28 +138,12 @@ jobs:
               c => c.body && c.body.startsWith(marker),
             );
 
-            const applied = environments.filter(
-              e => e.outcome === 'success' && e.summary && e.summary.dashboards.length,
-            );
-            const missing = environments.filter(e => e.outcome !== 'success');
-
-            if (!applied.length) {
-              if (missing.length) {
-                // At least one apply failed and nothing is left to link.
-                // Leave any prior sticky in place: preview resources from
-                // an earlier push may well still be live, and the failed
-                // step already makes the run red.
-                core.info(
-                  `No preview to link and ${missing.map(e => e.name).join(', ')} did not apply — leaving any existing comment alone.`,
-                );
-                return;
-              }
-              // Every environment applied cleanly and none had dashboards:
-              // this HEAD changes no dashboards. Don't post a "nothing to
-              // see" comment — if a prior push left a sticky behind, delete
-              // it so the PR's comment stream matches its current state.
-              // (Stale preview *resources* from a prior push are deleted by
-              // ci-apply-preview.sh.)
+            if (!raw) {
+              // No dashboards changed at this HEAD. Don't post a "nothing
+              // to see" comment — if a prior push left a sticky behind,
+              // delete it so the PR's comment stream matches its current
+              // state. (Stale preview *resources* from a prior push are
+              // handled by ci-apply-preview.sh.)
               if (existing) {
                 await github.rest.issues.deleteComment({
                   owner: context.repo.owner,
@@ -233,61 +157,38 @@ jobs:
               return;
             }
 
-            // Both environments render the same preview UIDs, so collapse
-            // them into one row per resource carrying a link per
-            // environment. Keyed by UID and built in the order the first
-            // applied environment reported, so a partial apply still lists
-            // everything that made it somewhere.
-            const collapse = (key) => {
-              const byUid = new Map();
-              for (const env of applied) {
-                for (const { uid, title } of env.summary[key]) {
-                  if (!byUid.has(uid)) byUid.set(uid, { title, envs: [] });
-                  byUid.get(uid).envs.push(env);
-                }
-              }
-              return [...byUid.entries()].map(([uid, v]) => ({ uid, ...v }));
-            };
-
-            const links = (envs, path) =>
-              envs
-                .map(e => `[${e.name}](${e.baseUrl}${path})`)
-                .join(' · ');
-
-            const folders = collapse('folders');
-            const dashboards = collapse('dashboards');
+            const folders = [];
+            const dashboards = [];
+            for (const line of raw.split('\n')) {
+              // Format: `<KIND>:<uid>:<title>` (title may contain `:`).
+              const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/);
+              if (!m) continue;
+              const [, kind, uid, title] = m;
+              (kind === 'FOLDER' ? folders : dashboards).push({ uid, title });
+            }
 
-            const folderLines = folders.map(
-              f => `- **Folder:** ${f.title} — ${links(f.envs, `/dashboards/f/${f.uid}/`)}`,
+            const folderLines = folders.map(f =>
+              `- **Folder:** [${f.title}](${grafanaBase}/dashboards/f/${f.uid}/)`,
             );
-            const dashboardLines = dashboards.map(
-              d => `- ${d.title} — ${links(d.envs, `/d/${d.uid}/`)}`,
+            const dashboardLines = dashboards.map(d =>
+              `- [${d.title}](${grafanaBase}/d/${d.uid}/)`,
             );
 
-            const appliedNames = applied.map(e => e.name).join(' and ');
-
-            const warning = missing.length
-              ? [
-                  '',
-                  '> [!WARNING]',
-                  `> The **${missing.map(e => e.name).join('** and **')}** preview did not apply on this run — see the run log below.`,
-                ]
-              : [];
-
             const body = [
               marker,
               '## Grafana preview',
               '',
-              `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in ${appliedNames}.`,
-              "Cross-dashboard drill-throughs still point at each environment's canonical dashboards.",
-              ...warning,
+              `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in the staging Grafana.`,
+              'Cross-dashboard drill-throughs still point at the canonical staging dashboards.',
               '',
               ...folderLines,
               '',
               '**Dashboards:**',
               ...dashboardLines,
               '',
-              'Previews are torn down automatically when this PR is closed or merged.',
+              'The production preview runs next and rewrites this comment with a link for each environment.',
+              '',
+              'Preview is torn down automatically when this PR is closed or merged.',
               '',
               `_(Run: ${runUrl})_`,
             ].join('\n');
@@ -311,10 +212,12 @@ jobs:
             }
 
   preview-cleanup:
-    name: Cleanup preview
+    name: Cleanup preview from staging
     # Cleanup runs on the `closed` action (covers both merge and close-
-    # without-merge). Same fork-PR guard — cleanup hits staging and
-    # production.
+    # without-merge). Same fork-PR guard — cleanup hits staging.
+    #
+    # The production preview is torn down by
+    # observability-preview-production.yml, which this run triggers.
     if: >-
       github.event.action == 'closed'
       && github.event.pull_request.head.repo.full_name == github.repository
@@ -326,40 +229,36 @@ jobs:
       - name: Install grafanactl
         uses: ./.github/actions/install-grafanactl
 
-      - name: Configure AWS credentials (staging)
+      - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
         with:
-          role-to-assume: ${{ env.STAGING_AWS_ROLE_ARN }}
+          role-to-assume: ${{ env.AWS_ROLE_ARN }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Cleanup preview from staging
+      - name: Source GRAFANA_TOKEN
+        # Cleanup doesn't need any of the URL/secret substitution values —
+        # it lists live state and deletes by selector. Only the grafanactl
+        # token matters.
+        run: |
+          GRAFANA_TOKEN="$(aws ssm get-parameter \
+            --name /staging/grafana/grafanactl_token \
+            --with-decryption \
+            --query 'Parameter.Value' \
+            --output text)"
+          echo "::add-mask::$GRAFANA_TOKEN"
+          echo "GRAFANA_TOKEN=$GRAFANA_TOKEN" >> "$GITHUB_ENV"
+
+      - name: Cleanup preview
         working-directory: packages/observability
         env:
           PR_NUMBER: ${{ github.event.pull_request.number }}
         run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env staging
 
-      - name: Configure AWS credentials (production)
-        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
-        with:
-          role-to-assume: ${{ env.PRODUCTION_AWS_ROLE_ARN }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Cleanup preview from production
-        working-directory: packages/observability
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-        run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env production
-
       - name: Delete sticky PR comment
         # After cleanup, there's nothing left to point at — delete the
         # sticky entirely rather than leaving a "preview cleaned up"
         # gravestone. Matches the apply-time policy of not posting a
         # comment when there's nothing to show.
-        #
-        # Runs even if a cleanup step failed: the daily sweep
-        # (observability-preview-sweep.yml) will catch whatever survived,
-        # and a comment pointing at deleted dashboards is worse than none.
-        if: ${{ !cancelled() }}
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
diff --git a/packages/observability/README.md b/packages/observability/README.md
index 7916c1c1412..ea990a5cf4c 100644
--- a/packages/observability/README.md
+++ b/packages/observability/README.md
@@ -455,11 +455,23 @@ CI runs `apply.sh --env staging` on merge to main (workflow:
 A PR that changes anything under `grafanactl/resources/dashboards/` also gets
 a live preview of just the dashboards it touches, pushed to **both** the
 staging and the production Grafana under `pr<n>-`-prefixed UIDs in a per-PR
-folder, so it coexists with the canonical copies rather than overwriting them
-(`.github/workflows/observability-preview.yml`). One sticky PR comment links
-each changed dashboard in both environments. Previews are deleted when the PR
-closes; `.github/workflows/observability-preview-sweep.yml` sweeps both
-environments daily for any the close hook missed.
+folder, so it coexists with the canonical copies rather than overwriting them.
+One sticky PR comment links each changed dashboard in both environments, and
+both previews are deleted when the PR closes.
+`.github/workflows/observability-preview-sweep.yml` sweeps both environments
+daily for any the close hook missed.
+
+The preview is split across two workflows, and **the split is a security
+boundary**. `observability-preview.yml` runs on `pull_request`, so GitHub
+loads it — and every script it calls — from the PR, where a contributor
+controls both; it therefore holds **staging credentials only**.
+`observability-preview-production.yml` runs on `workflow_run`, which GitHub
+always loads from the default branch, out of a PR's reach. It restores every
+script from the default branch and takes only `grafanactl/resources` from the
+PR, as data, with `render-preview.sh` rewriting every UID it emits so a
+crafted manifest cannot address a canonical dashboard. Never move a production
+role, a production SSM read, or a production Grafana push into the
+`pull_request` workflow.
 
 ## Operator actions
 
diff --git a/packages/observability/scripts/ci-apply-preview.sh b/packages/observability/scripts/ci-apply-preview.sh
index 4a58d6ed755..f0120cb4115 100755
--- a/packages/observability/scripts/ci-apply-preview.sh
+++ b/packages/observability/scripts/ci-apply-preview.sh
@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
-# ci-apply-preview.sh — one environment's half of observability-preview.yml's
-# apply job.
+# ci-apply-preview.sh — one environment's half of the PR preview.
+#
+# Called by observability-preview.yml for staging and by
+# observability-preview-production.yml for production. The production
+# caller runs the default branch's copy of this file, never the pull
+# request's — see that workflow's header for why.
 #
 # Fetches the apply-time secrets for $ENV_NAME from SSM, pushes the per-PR
 # preview with apply-preview.sh, and reports the result back to the workflow
@@ -10,9 +14,9 @@
 #                     apply-preview.sh); empty file when nothing was pushed
 #   empty=true|false  whether this PR's HEAD changes any dashboards
 #
-# Lives here rather than inline in the workflow so staging and production
+# Lives here rather than inline in either workflow so staging and production
 # run byte-identical logic — the environments differ only in $ENV_NAME and
-# in which AWS credentials the workflow configured before calling this.
+# in which AWS credentials the caller configured before invoking this.
 #
 # Environment (all set by the workflow step):
 #   ENV_NAME      staging | production
diff --git a/packages/observability/scripts/cleanup-preview.sh b/packages/observability/scripts/cleanup-preview.sh
index 297a46f69dd..d453d640663 100755
--- a/packages/observability/scripts/cleanup-preview.sh
+++ b/packages/observability/scripts/cleanup-preview.sh
@@ -23,8 +23,9 @@
 #     gets the old preview resources removed before the new tree lands.
 #   - ci-apply-preview.sh runs this when a PR stops changing dashboards
 #     altogether, to clear a preview an earlier push left behind.
-#   - observability-preview.yml runs this on pull_request `closed`, once
-#     per environment.
+#   - observability-preview.yml runs this for staging on pull_request
+#     `closed`, and observability-preview-production.yml runs it for
+#     production once that close event has been observed.
 #   - observability-preview-sweep.yml runs this for PRs the daily sweep
 #     has decided are stale.
 set -eo pipefail
diff --git a/packages/observability/scripts/render-preview.sh b/packages/observability/scripts/render-preview.sh
index d4c7b975312..8ef4c78d1d6 100755
--- a/packages/observability/scripts/render-preview.sh
+++ b/packages/observability/scripts/render-preview.sh
@@ -246,4 +246,26 @@ if [[ -z "$(find "$rendered/dashboards" -type f -name '*.json' 2>/dev/null)" ]];
   exit 0
 fi
 
+# Fail closed if any emitted manifest kept a canonical UID. The rewrites
+# above already guarantee the `pr<n>-` prefix on every dashboard, every
+# folder, and every dashboard's folder pointer; this check makes that
+# guarantee explicit so a future change to them cannot silently produce a
+# tree that overwrites a real dashboard. The tree is pushed to production as
+# well as staging, so losing the prefix would cost a live production
+# dashboard.
+while IFS= read -r -d '' f; do
+  uid="$(jq -r '.metadata.name // ""' "$f")"
+  if [[ "$uid" != "pr${pr_number}-"* ]]; then
+    echo "error: $f carries UID '$uid', which lacks the pr${pr_number}- prefix" >&2
+    rm -rf "$rendered"
+    exit 1
+  fi
+  folder="$(jq -r '(.metadata.annotations // {})["grafana.app/folder"] // ""' "$f")"
+  if [[ -n "$folder" && "$folder" != "pr${pr_number}-"* ]]; then
+    echo "error: $f points at folder '$folder', which lacks the pr${pr_number}- prefix" >&2
+    rm -rf "$rendered"
+    exit 1
+  fi
+done < <(find "$rendered" -type f -name '*.json' -print0)
+
 echo "$rendered"

From e25f927b91dc2b7bec868eb35122a5e48736de6c Mon Sep 17 00:00:00 2001
From: Buck Doyle <buck.doyle@cardstack.com>
Date: Wed, 23 Sep 2026 09:59:17 +0200
Subject: [PATCH 3/6] Narrow the preview IAM role and drop the two-workflow
 split

The preview workflow runs on `pull_request`. GitHub loads that workflow, and
every script it calls, from the pull request. A contributor controls the code
that runs after the job assumes the role.

The previous commit fenced the code. It moved the production push into a
`workflow_run` workflow that GitHub loads from the default branch. That
protected the credential, but it added a second workflow, and it stopped the
production preview from running before merge.

This commit removes the asset instead. The preview assumes
boxel-observability-preview, which grants two parameters for each
environment:

- /<env>/grafana/grafanactl_token
- /<env>/boxel-grafana/realm_server_url

The worst case now equals the feature itself: write `pr<n>-` dashboards.

- render-preview.sh stamps an inert placeholder into `grafana_secret`. It
  ignores an exported GRAFANA_SECRET, so a caller with wider credentials
  cannot leak the real value into a preview. Operator buttons on a preview
  dashboard return 401.
- ci-apply-preview.sh no longer reads /<env>/boxel/GRAFANA_SECRET.
- observability-preview-production.yml is deleted. One workflow applies both
  environments again, and one comment links both.
- The daily sweep uses the same narrow role.

render-preview.sh keeps the assertion that every manifest it emits carries
the `pr<n>-` prefix.

cardstack/infra adds the role. Merge that change first.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../observability-preview-production.yml      | 328 ------------------
 .../workflows/observability-preview-sweep.yml |   8 +-
 .github/workflows/observability-preview.yml   | 272 ++++++++++-----
 packages/observability/README.md              |  39 ++-
 .../observability/scripts/ci-apply-preview.sh |  20 +-
 .../observability/scripts/cleanup-preview.sh  |   5 +-
 .../observability/scripts/render-preview.sh   |  30 +-
 7 files changed, 252 insertions(+), 450 deletions(-)
 delete mode 100644 .github/workflows/observability-preview-production.yml

diff --git a/.github/workflows/observability-preview-production.yml b/.github/workflows/observability-preview-production.yml
deleted file mode 100644
index 48d78345c9b..00000000000
--- a/.github/workflows/observability-preview-production.yml
+++ /dev/null
@@ -1,328 +0,0 @@
-name: Observability — PR preview (production)
-
-# Applies the per-PR dashboard preview to the PRODUCTION Grafana, and tears
-# it down again when the PR closes.
-#
-# WHY THIS IS A SEPARATE WORKFLOW
-#
-# observability-preview.yml runs on `pull_request`. GitHub loads that file,
-# and every script it calls, from the pull request's own merge ref — a
-# contributor controls both. Giving it the production role would let any
-# same-repository PR run arbitrary code with production credentials live and
-# exfiltrate the temporary AWS credentials, the decrypted production
-# grafanactl token, and GRAFANA_SECRET. It would also defeat the main-only
-# guard in observability-apply-production.yml.
-#
-# GitHub always loads a `workflow_run` workflow from the default branch, no
-# matter which branch the triggering run used. This file is therefore out of
-# a pull request's reach, and it is the only place production credentials are
-# safe to use for a preview.
-#
-# THE TRUST BOUNDARY
-#
-# The one thing this job takes from the pull request is the contents of
-# packages/observability/grafanactl/resources — dashboard and folder
-# manifests. Those are data, not code: nothing here executes them, and
-# render-preview.sh (restored from the default branch, like every other
-# script) rewrites every UID it emits to the `pr<n>-` form, so a crafted
-# manifest cannot address a canonical production dashboard. The "Overlay"
-# step below is what enforces that split; read it before changing it.
-
-on:
-  workflow_run:
-    workflows: ["Observability — PR preview"]
-    types: [completed]
-
-permissions:
-  contents: read
-  id-token: write
-  pull-requests: write
-
-concurrency:
-  # Serialize per branch: a force-push while a previous production apply is
-  # mid-flight must not race it. Never cancel in-flight — that could leave
-  # the production Grafana half-pushed.
-  group: observability-preview-production-${{ github.event.workflow_run.head_branch }}
-  cancel-in-progress: false
-
-env:
-  AWS_REGION: us-east-1
-  # The same narrowly-scoped role observability-apply-production.yml uses
-  # (cardstack/infra#666). Permissions are ssm:GetParameter on the three
-  # paths ci-apply-preview.sh reads for production.
-  AWS_ROLE_ARN: arn:aws:iam::120317779495:role/boxel-observability-apply
-  EXPECTED_GRAFANA_HOSTNAME: dashboard.boxel.ai
-  PREVIEW_MARKER: "<!-- observability-preview -->"
-  STAGING_GRAFANA_BASE_URL: "https://dashboard-staging.stack.cards"
-  PRODUCTION_GRAFANA_BASE_URL: "https://dashboard.boxel.ai"
-
-jobs:
-  preview-production:
-    name: Apply preview to production
-    # Fork guard. `workflow_run` fires even when the triggering job itself
-    # was skipped, so this has to be re-checked here rather than inherited.
-    if: >-
-      github.event.workflow_run.head_repository.full_name == github.repository
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout the default branch
-        # Trusted code. Everything this job executes comes from here; the
-        # pull request's manifests are overlaid further down, after
-        # grafanactl is already installed.
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          ref: ${{ github.event.repository.default_branch }}
-          fetch-depth: 0
-
-      - name: Install grafanactl
-        # Runs while the workspace still holds only default-branch code, so
-        # this `uses: ./` cannot resolve to a pull-request-authored action.
-        uses: ./.github/actions/install-grafanactl
-
-      - name: Resolve the pull request
-        id: pr
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REPO: ${{ github.repository }}
-          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
-        run: |
-          set -eo pipefail
-          # Ask GitHub which PR this run belongs to rather than trusting
-          # anything the triggering run produced. `workflow_run` does carry a
-          # `pull_requests` array, but it is empty often enough that it can't
-          # be relied on.
-          pr_json="$(gh api "repos/${REPO}/commits/${HEAD_SHA}/pulls" \
-            --jq '[.[] | select(.head.sha == env.HEAD_SHA)][0] // empty')"
-
-          if [[ -z "$pr_json" ]]; then
-            echo "No pull request found for ${HEAD_SHA} — nothing to do."
-            echo "found=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          number="$(jq -r '.number' <<< "$pr_json")"
-          base_ref="$(jq -r '.base.ref' <<< "$pr_json")"
-          state="$(jq -r '.state' <<< "$pr_json")"
-          head_repo="$(jq -r '.head.repo.full_name // ""' <<< "$pr_json")"
-
-          # Second fork guard, on the PR record itself.
-          if [[ "$head_repo" != "$REPO" ]]; then
-            echo "PR #${number} comes from ${head_repo} — refusing to touch production."
-            echo "found=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          echo "PR #${number}, base ${base_ref}, state ${state}"
-          {
-            echo "found=true"
-            echo "number=${number}"
-            echo "base_ref=${base_ref}"
-            echo "state=${state}"
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Overlay the pull request's dashboard manifests
-        # THE TRUST BOUNDARY. After this step the working tree holds
-        # default-branch code everywhere except
-        # packages/observability/grafanactl/resources, which holds the pull
-        # request's manifests. HEAD is the pull request's commit so that
-        # render-preview.sh's `git diff <base>...HEAD` still sees the right
-        # changed set.
-        #
-        # Skipped for a closed PR: teardown reads live Grafana state and
-        # needs no manifests at all.
-        if: steps.pr.outputs.found == 'true' && steps.pr.outputs.state == 'open'
-        env:
-          PR_NUMBER: ${{ steps.pr.outputs.number }}
-          PR_BASE_REF: ${{ steps.pr.outputs.base_ref }}
-        run: |
-          set -eo pipefail
-          main_sha="$(git rev-parse HEAD)"
-
-          git fetch --no-tags origin "pull/${PR_NUMBER}/head"
-          git checkout --quiet --detach FETCH_HEAD
-
-          # Restore every path this job executes or reads for configuration.
-          # packages/observability covers the scripts and grafanactl/config.yaml
-          # (which names the production server URL); .github covers any action
-          # a later step might resolve through `uses: ./`.
-          git checkout --quiet "$main_sha" -- .github packages/observability
-
-          # ...then take back ONLY the manifests. This is the sole pull-request
-          # input to the production push.
-          git checkout --quiet FETCH_HEAD -- packages/observability/grafanactl/resources
-
-          git fetch --no-tags origin "$PR_BASE_REF"
-
-          echo "Overlaid manifests from PR #${PR_NUMBER} onto ${main_sha}."
-
-      - name: Pre-flight — assert config.yaml's production server URL
-        # Runs before any credential is fetched. config.yaml comes from the
-        # default branch (restored above), so this catches a config-as-code
-        # mistake rather than a hostile edit — but it is the check that makes
-        # "the token can only go to the real production Grafana" verifiable.
-        #
-        # The awk uses an in-block flag rather than awk's range syntax
-        # (`/start/,/end/`), because `/^  production:$/,/^  [a-z]/` would
-        # match only the start line — the end pattern `^  [a-z]` matches the
-        # start line itself, terminating the range immediately.
-        if: steps.pr.outputs.found == 'true'
-        working-directory: packages/observability
-        run: |
-          set -eo pipefail
-          ACTUAL="$(awk '
-            /^  production:$/      { in_prod=1; next }
-            in_prod && /^  [a-z]/  { exit }
-            in_prod && /^      server:/ { print $2; exit }
-          ' grafanactl/config.yaml)"
-          echo "config.yaml production server: $ACTUAL"
-          echo "expected:                       https://${EXPECTED_GRAFANA_HOSTNAME}"
-          if [[ "$ACTUAL" != "https://${EXPECTED_GRAFANA_HOSTNAME}" ]]; then
-            echo "::error::config.yaml's production context server URL doesn't match expected hostname." >&2
-            echo "If this is an intentional rename, update EXPECTED_GRAFANA_HOSTNAME in this workflow to match." >&2
-            exit 1
-          fi
-
-      - name: Configure AWS credentials
-        if: steps.pr.outputs.found == 'true'
-        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
-        with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Apply preview
-        id: apply
-        if: steps.pr.outputs.found == 'true' && steps.pr.outputs.state == 'open'
-        working-directory: packages/observability
-        env:
-          ENV_NAME: production
-          PR_NUMBER: ${{ steps.pr.outputs.number }}
-          PR_BASE_REF: ${{ steps.pr.outputs.base_ref }}
-        run: ./scripts/ci-apply-preview.sh
-
-      - name: Cleanup preview
-        if: steps.pr.outputs.found == 'true' && steps.pr.outputs.state == 'closed'
-        working-directory: packages/observability
-        env:
-          PR_NUMBER: ${{ steps.pr.outputs.number }}
-        run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env production
-
-      - name: Rewrite the sticky PR comment
-        # observability-preview.yml already posted a staging-only sticky.
-        # Replace it with one covering both environments. The preview UIDs and
-        # titles are environment-independent — render-preview.sh varies only
-        # the apply-time substitutions — so this job's own summary describes
-        # the staging resources just as well as the production ones.
-        if: steps.pr.outputs.found == 'true' && steps.pr.outputs.state == 'open'
-        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
-        env:
-          OUT_FILE: ${{ steps.apply.outputs.out_file }}
-          PR_NUMBER: ${{ steps.pr.outputs.number }}
-          STAGING_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
-          STAGING_RUN_URL: ${{ github.event.workflow_run.html_url }}
-        with:
-          script: |
-            const fs = require('fs');
-            const marker = process.env.PREVIEW_MARKER;
-            const prNumber = Number(process.env.PR_NUMBER);
-            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
-
-            const outFile = process.env.OUT_FILE;
-            const raw = outFile && fs.existsSync(outFile)
-              ? fs.readFileSync(outFile, 'utf8').trim()
-              : '';
-
-            if (!raw) {
-              // This HEAD changes no dashboards, so there is nothing to link
-              // and the staging job already removed any stale sticky.
-              core.info('No dashboard changes — leaving the comment alone.');
-              return;
-            }
-
-            const folders = [];
-            const dashboards = [];
-            for (const line of raw.split('\n')) {
-              // Format: `<KIND>:<uid>:<title>` (title may contain `:`).
-              const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/);
-              if (!m) continue;
-              const [, kind, uid, title] = m;
-              (kind === 'FOLDER' ? folders : dashboards).push({ uid, title });
-            }
-
-            const stagingOk = process.env.STAGING_CONCLUSION === 'success';
-            const environments = [
-              ...(stagingOk
-                ? [{ name: 'staging', baseUrl: process.env.STAGING_GRAFANA_BASE_URL }]
-                : []),
-              { name: 'production', baseUrl: process.env.PRODUCTION_GRAFANA_BASE_URL },
-            ];
-
-            const links = (path) =>
-              environments
-                .map(e => `[${e.name}](${e.baseUrl}${path})`)
-                .join(' · ');
-
-            const folderLines = folders.map(
-              f => `- **Folder:** ${f.title} — ${links(`/dashboards/f/${f.uid}/`)}`,
-            );
-            const dashboardLines = dashboards.map(
-              d => `- ${d.title} — ${links(`/d/${d.uid}/`)}`,
-            );
-
-            const warning = stagingOk
-              ? []
-              : [
-                  '',
-                  '> [!WARNING]',
-                  `> The **staging** preview did not apply — see [its run](${process.env.STAGING_RUN_URL}). Only the production links are listed.`,
-                ];
-
-            const envNames = environments.map(e => e.name).join(' and ');
-
-            const body = [
-              marker,
-              '## Grafana preview',
-              '',
-              `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in ${envNames}.`,
-              "Cross-dashboard drill-throughs still point at each environment's canonical dashboards.",
-              ...warning,
-              '',
-              ...folderLines,
-              '',
-              '**Dashboards:**',
-              ...dashboardLines,
-              '',
-              'Previews are torn down automatically when this PR is closed or merged.',
-              '',
-              `_(Run: ${runUrl})_`,
-            ].join('\n');
-
-            const comments = await github.paginate(
-              github.rest.issues.listComments,
-              {
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: prNumber,
-                per_page: 100,
-              },
-            );
-            const existing = comments.find(
-              c => c.body && c.body.startsWith(marker),
-            );
-
-            if (existing) {
-              await github.rest.issues.updateComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                comment_id: existing.id,
-                body,
-              });
-              core.info(`Updated sticky comment ${existing.id}`);
-            } else {
-              await github.rest.issues.createComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: prNumber,
-                body,
-              });
-              core.info('Created sticky comment');
-            }
diff --git a/.github/workflows/observability-preview-sweep.yml b/.github/workflows/observability-preview-sweep.yml
index 119e7d59021..3f058de79f4 100644
--- a/.github/workflows/observability-preview-sweep.yml
+++ b/.github/workflows/observability-preview-sweep.yml
@@ -54,10 +54,14 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # The dedicated preview role — it grants the grafanactl token the
+          # sweep needs to list and delete `pr<n>-` resources, and nothing
+          # else. See observability-preview.yml for why the apply role is
+          # the wrong one to reach for here.
           - env: staging
-            role_arn: arn:aws:iam::680542703984:role/boxel-observability-apply
+            role_arn: arn:aws:iam::680542703984:role/boxel-observability-preview
           - env: production
-            role_arn: arn:aws:iam::120317779495:role/boxel-observability-apply
+            role_arn: arn:aws:iam::120317779495:role/boxel-observability-preview
 
     steps:
       - name: Refuse to sweep production from a non-main ref
diff --git a/.github/workflows/observability-preview.yml b/.github/workflows/observability-preview.yml
index 58703443e78..00898936937 100644
--- a/.github/workflows/observability-preview.yml
+++ b/.github/workflows/observability-preview.yml
@@ -1,23 +1,26 @@
 name: Observability — PR preview
 
-# Spins up a per-PR preview of the dashboards a PR changes inside the staging
-# Grafana, under per-PR-prefixed UIDs (`pr<n>-*`) in a per-PR folder so the
-# preview coexists with the canonical staging copies without overwriting
-# them. On PR close the preview is torn down. A separate sweep workflow
-# (observability-preview-sweep.yml) is the safety net for closes the
-# preview-cleanup job missed.
+# Spins up a per-PR preview of the dashboards a PR changes inside BOTH the
+# staging and the production Grafana, under per-PR-prefixed UIDs (`pr<n>-*`)
+# in a per-PR folder so the preview coexists with the canonical copies
+# without overwriting them. On PR close both previews are torn down. A
+# separate sweep workflow (observability-preview-sweep.yml) is the safety net
+# for closes the preview-cleanup job missed.
 #
-# This workflow runs on `pull_request`, so GitHub loads it — and every script
-# it calls — from the pull request's own merge ref, where a contributor can
-# change both. It therefore handles STAGING CREDENTIALS ONLY. The production
-# preview lives in observability-preview-production.yml, which GitHub starts
-# with `workflow_run` and always loads from the default branch, out of a pull
-# request's reach. Never add a production role, a production SSM read, or a
-# production Grafana push to this file.
+# Both environments are previewed from one job so the PR gets a single
+# sticky comment linking each changed dashboard in both Grafanas, rather
+# than one comment per environment. Staging is applied first: a production
+# Grafana problem then costs the run its red status but not the staging
+# preview or the comment, which is posted from whatever succeeded.
 #
-# That production workflow rewrites the sticky comment below into one that
-# covers both environments. While it is still running, or when it fails, the
-# staging-only comment stands on its own.
+# The two environments render the same dashboard set under the same preview
+# UIDs — only the realm-server URL substitution differs — so the comment
+# lists each dashboard once with a link per environment.
+#
+# A preview never carries the real operator secret: render-preview.sh stamps
+# an inert placeholder into `grafana_secret`, so operator buttons on a
+# preview dashboard return 401. That is what keeps this workflow runnable
+# under a role that cannot read GRAFANA_SECRET.
 #
 # Scope: dashboards + folders only. Data sources, alert rules, and the
 # home-dashboard preference are file-provisioned at Grafana startup and
@@ -49,20 +52,28 @@ concurrency:
 
 env:
   AWS_REGION: us-east-1
-  # Reuses the apply role from cardstack/infra#665 (same role
-  # observability-apply-staging.yml uses). Permissions cover every SSM path
-  # ci-apply-preview.sh reads — see that workflow's header for the up-to-date
-  # list.
-  AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-apply
+  # The dedicated preview role, NOT boxel-observability-apply. This workflow
+  # runs on `pull_request`, so GitHub loads it — and every script it calls —
+  # from the pull request itself; whatever the assumed role can read, a
+  # contributor can read. boxel-observability-preview grants only
+  # /<env>/grafana/grafanactl_token and /<env>/boxel-grafana/realm_server_url,
+  # so the worst case equals this feature: write `pr<n>-` dashboards. The
+  # apply role additionally grants the Grafana database password and
+  # GRAFANA_SECRET (the realm-server operator bearer token) — never point
+  # this workflow at it.
+  STAGING_AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-preview
+  PRODUCTION_AWS_ROLE_ARN: arn:aws:iam::120317779495:role/boxel-observability-preview
   PREVIEW_MARKER: "<!-- observability-preview -->"
   STAGING_GRAFANA_BASE_URL: "https://dashboard-staging.stack.cards"
+  PRODUCTION_GRAFANA_BASE_URL: "https://dashboard.boxel.ai"
 
 jobs:
   preview-apply:
-    name: Apply preview to staging
-    # Fork-PR guard: this job assumes AWS and pushes to staging Grafana, so
-    # it must not run on PRs whose code we don't trust. Same pattern as
-    # observability-diff.yml and observability-apply-staging.yml.
+    name: Apply preview
+    # Fork-PR guard: this job assumes AWS and pushes to staging and
+    # production Grafana, so it must not run on PRs whose code we don't
+    # trust. Same pattern as observability-diff.yml and
+    # observability-apply-staging.yml.
     if: >-
       github.event.action != 'closed'
       && github.event.pull_request.head.repo.full_name == github.repository
@@ -90,14 +101,14 @@ jobs:
       - name: Install grafanactl
         uses: ./.github/actions/install-grafanactl
 
-      - name: Configure AWS credentials
+      - name: Configure AWS credentials (staging)
         uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
         with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          role-to-assume: ${{ env.STAGING_AWS_ROLE_ARN }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Apply preview
-        id: apply
+      - name: Apply preview to staging
+        id: apply-staging
         working-directory: packages/observability
         env:
           ENV_NAME: staging
@@ -105,26 +116,82 @@ jobs:
           PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
         run: ./scripts/ci-apply-preview.sh
 
+      # Production is applied after staging, and each environment's
+      # credentials live only in the step that uses them — the AWS creds in
+      # the runner environment and the Grafana token inside the step's own
+      # shell. Nothing is written to $GITHUB_ENV, so there's no window in
+      # which one environment's token is live while the other's push runs.
+      - name: Configure AWS credentials (production)
+        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
+        with:
+          role-to-assume: ${{ env.PRODUCTION_AWS_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Apply preview to production
+        id: apply-production
+        working-directory: packages/observability
+        env:
+          ENV_NAME: production
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
+        run: ./scripts/ci-apply-preview.sh
+
       - name: Post sticky PR comment
+        # Runs even when an apply step failed, so a production-side outage
+        # still leaves reviewers the staging links (and a note saying which
+        # environment is missing). The job is still red — the failed step
+        # decides that, not this one.
+        #
         # Pinned to SHA (matches observability-diff.yml's reasoning):
         # pull-requests: write + AWS role = non-trivial supply-chain exposure.
+        if: ${{ !cancelled() }}
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
-          OUT_FILE: ${{ steps.apply.outputs.out_file }}
+          STAGING_OUT_FILE: ${{ steps.apply-staging.outputs.out_file }}
+          STAGING_OUTCOME: ${{ steps.apply-staging.outcome }}
+          PRODUCTION_OUT_FILE: ${{ steps.apply-production.outputs.out_file }}
+          PRODUCTION_OUTCOME: ${{ steps.apply-production.outcome }}
         with:
           script: |
             const fs = require('fs');
             const marker = process.env.PREVIEW_MARKER;
-            const grafanaBase = process.env.STAGING_GRAFANA_BASE_URL;
             const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
 
-            const outFile = process.env.OUT_FILE;
-            const raw = outFile && fs.existsSync(outFile)
-              ? fs.readFileSync(outFile, 'utf8').trim()
-              : '';
+            // Parse one environment's `<KIND>:<uid>:<title>` summary (the
+            // structured stdout of apply-preview.sh). A missing file means
+            // the apply step never got far enough to write one.
+            const readSummary = (outFile) => {
+              if (!outFile || !fs.existsSync(outFile)) return null;
+              const folders = [];
+              const dashboards = [];
+              for (const line of fs.readFileSync(outFile, 'utf8').trim().split('\n')) {
+                // Title may contain `:`, so only the first two fields are
+                // delimited.
+                const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/);
+                if (!m) continue;
+                const [, kind, uid, title] = m;
+                (kind === 'FOLDER' ? folders : dashboards).push({ uid, title });
+              }
+              return { folders, dashboards };
+            };
 
-            // Find any prior preview sticky on this PR; we either update or
-            // delete it based on whether there's anything to preview now.
+            const environments = [
+              {
+                name: 'staging',
+                baseUrl: process.env.STAGING_GRAFANA_BASE_URL,
+                outcome: process.env.STAGING_OUTCOME,
+                summary: readSummary(process.env.STAGING_OUT_FILE),
+              },
+              {
+                name: 'production',
+                baseUrl: process.env.PRODUCTION_GRAFANA_BASE_URL,
+                outcome: process.env.PRODUCTION_OUTCOME,
+                summary: readSummary(process.env.PRODUCTION_OUT_FILE),
+              },
+            ];
+
+            // Find any prior preview sticky on this PR; we either update,
+            // delete, or deliberately leave it alone below.
             const comments = await github.paginate(
               github.rest.issues.listComments,
               {
@@ -138,12 +205,28 @@ jobs:
               c => c.body && c.body.startsWith(marker),
             );
 
-            if (!raw) {
-              // No dashboards changed at this HEAD. Don't post a "nothing
-              // to see" comment — if a prior push left a sticky behind,
-              // delete it so the PR's comment stream matches its current
-              // state. (Stale preview *resources* from a prior push are
-              // handled by ci-apply-preview.sh.)
+            const applied = environments.filter(
+              e => e.outcome === 'success' && e.summary && e.summary.dashboards.length,
+            );
+            const missing = environments.filter(e => e.outcome !== 'success');
+
+            if (!applied.length) {
+              if (missing.length) {
+                // At least one apply failed and nothing is left to link.
+                // Leave any prior sticky in place: preview resources from
+                // an earlier push may well still be live, and the failed
+                // step already makes the run red.
+                core.info(
+                  `No preview to link and ${missing.map(e => e.name).join(', ')} did not apply — leaving any existing comment alone.`,
+                );
+                return;
+              }
+              // Every environment applied cleanly and none had dashboards:
+              // this HEAD changes no dashboards. Don't post a "nothing to
+              // see" comment — if a prior push left a sticky behind, delete
+              // it so the PR's comment stream matches its current state.
+              // (Stale preview *resources* from a prior push are deleted by
+              // ci-apply-preview.sh.)
               if (existing) {
                 await github.rest.issues.deleteComment({
                   owner: context.repo.owner,
@@ -157,38 +240,61 @@ jobs:
               return;
             }
 
-            const folders = [];
-            const dashboards = [];
-            for (const line of raw.split('\n')) {
-              // Format: `<KIND>:<uid>:<title>` (title may contain `:`).
-              const m = line.match(/^(FOLDER|DASHBOARD):([^:]+):(.*)$/);
-              if (!m) continue;
-              const [, kind, uid, title] = m;
-              (kind === 'FOLDER' ? folders : dashboards).push({ uid, title });
-            }
+            // Both environments render the same preview UIDs, so collapse
+            // them into one row per resource carrying a link per
+            // environment. Keyed by UID and built in the order the first
+            // applied environment reported, so a partial apply still lists
+            // everything that made it somewhere.
+            const collapse = (key) => {
+              const byUid = new Map();
+              for (const env of applied) {
+                for (const { uid, title } of env.summary[key]) {
+                  if (!byUid.has(uid)) byUid.set(uid, { title, envs: [] });
+                  byUid.get(uid).envs.push(env);
+                }
+              }
+              return [...byUid.entries()].map(([uid, v]) => ({ uid, ...v }));
+            };
+
+            const links = (envs, path) =>
+              envs
+                .map(e => `[${e.name}](${e.baseUrl}${path})`)
+                .join(' · ');
 
-            const folderLines = folders.map(f =>
-              `- **Folder:** [${f.title}](${grafanaBase}/dashboards/f/${f.uid}/)`,
+            const folders = collapse('folders');
+            const dashboards = collapse('dashboards');
+
+            const folderLines = folders.map(
+              f => `- **Folder:** ${f.title} — ${links(f.envs, `/dashboards/f/${f.uid}/`)}`,
             );
-            const dashboardLines = dashboards.map(d =>
-              `- [${d.title}](${grafanaBase}/d/${d.uid}/)`,
+            const dashboardLines = dashboards.map(
+              d => `- ${d.title} — ${links(d.envs, `/d/${d.uid}/`)}`,
             );
 
+            const appliedNames = applied.map(e => e.name).join(' and ');
+
+            const warning = missing.length
+              ? [
+                  '',
+                  '> [!WARNING]',
+                  `> The **${missing.map(e => e.name).join('** and **')}** preview did not apply on this run — see the run log below.`,
+                ]
+              : [];
+
             const body = [
               marker,
               '## Grafana preview',
               '',
-              `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in the staging Grafana.`,
-              'Cross-dashboard drill-throughs still point at the canonical staging dashboards.',
+              `Preview deployed for **${dashboards.length} dashboard${dashboards.length === 1 ? '' : 's'}** in ${appliedNames}.`,
+              "Cross-dashboard drill-throughs still point at each environment's canonical dashboards.",
+              ...warning,
               '',
               ...folderLines,
               '',
               '**Dashboards:**',
               ...dashboardLines,
               '',
-              'The production preview runs next and rewrites this comment with a link for each environment.',
-              '',
-              'Preview is torn down automatically when this PR is closed or merged.',
+              'Previews are torn down automatically when this PR is closed or merged.',
               '',
               `_(Run: ${runUrl})_`,
             ].join('\n');
@@ -212,12 +318,10 @@ jobs:
             }
 
   preview-cleanup:
-    name: Cleanup preview from staging
+    name: Cleanup preview
     # Cleanup runs on the `closed` action (covers both merge and close-
-    # without-merge). Same fork-PR guard — cleanup hits staging.
-    #
-    # The production preview is torn down by
-    # observability-preview-production.yml, which this run triggers.
+    # without-merge). Same fork-PR guard — cleanup hits staging and
+    # production.
     if: >-
       github.event.action == 'closed'
       && github.event.pull_request.head.repo.full_name == github.repository
@@ -229,36 +333,40 @@ jobs:
       - name: Install grafanactl
         uses: ./.github/actions/install-grafanactl
 
-      - name: Configure AWS credentials
+      - name: Configure AWS credentials (staging)
         uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
         with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          role-to-assume: ${{ env.STAGING_AWS_ROLE_ARN }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Source GRAFANA_TOKEN
-        # Cleanup doesn't need any of the URL/secret substitution values —
-        # it lists live state and deletes by selector. Only the grafanactl
-        # token matters.
-        run: |
-          GRAFANA_TOKEN="$(aws ssm get-parameter \
-            --name /staging/grafana/grafanactl_token \
-            --with-decryption \
-            --query 'Parameter.Value' \
-            --output text)"
-          echo "::add-mask::$GRAFANA_TOKEN"
-          echo "GRAFANA_TOKEN=$GRAFANA_TOKEN" >> "$GITHUB_ENV"
-
-      - name: Cleanup preview
+      - name: Cleanup preview from staging
         working-directory: packages/observability
         env:
           PR_NUMBER: ${{ github.event.pull_request.number }}
         run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env staging
 
+      - name: Configure AWS credentials (production)
+        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
+        with:
+          role-to-assume: ${{ env.PRODUCTION_AWS_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Cleanup preview from production
+        working-directory: packages/observability
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: ./scripts/cleanup-preview.sh --pr "$PR_NUMBER" --env production
+
       - name: Delete sticky PR comment
         # After cleanup, there's nothing left to point at — delete the
         # sticky entirely rather than leaving a "preview cleaned up"
         # gravestone. Matches the apply-time policy of not posting a
         # comment when there's nothing to show.
+        #
+        # Runs even if a cleanup step failed: the daily sweep
+        # (observability-preview-sweep.yml) will catch whatever survived,
+        # and a comment pointing at deleted dashboards is worse than none.
+        if: ${{ !cancelled() }}
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
diff --git a/packages/observability/README.md b/packages/observability/README.md
index ea990a5cf4c..2534c9cadfa 100644
--- a/packages/observability/README.md
+++ b/packages/observability/README.md
@@ -461,17 +461,34 @@ both previews are deleted when the PR closes.
 `.github/workflows/observability-preview-sweep.yml` sweeps both environments
 daily for any the close hook missed.
 
-The preview is split across two workflows, and **the split is a security
-boundary**. `observability-preview.yml` runs on `pull_request`, so GitHub
-loads it — and every script it calls — from the PR, where a contributor
-controls both; it therefore holds **staging credentials only**.
-`observability-preview-production.yml` runs on `workflow_run`, which GitHub
-always loads from the default branch, out of a PR's reach. It restores every
-script from the default branch and takes only `grafanactl/resources` from the
-PR, as data, with `render-preview.sh` rewriting every UID it emits so a
-crafted manifest cannot address a canonical dashboard. Never move a production
-role, a production SSM read, or a production Grafana push into the
-`pull_request` workflow.
+**The preview runs pull-request-controlled code, so it holds a deliberately
+weak credential.** It runs on `pull_request`, and GitHub loads that workflow —
+and every script it calls — from the PR itself. Whatever the assumed role can
+read, a contributor can read. So the preview uses its own IAM role,
+`boxel-observability-preview` (cardstack/infra,
+`configs/boxel-observability-preview/`), which grants exactly two parameters
+per environment:
+
+```
+/<env>/grafana/grafanactl_token        authenticate grafanactl
+/<env>/boxel-grafana/realm_server_url  fill the realm_server template variable
+```
+
+Not `boxel-observability-apply`. That role also grants the Grafana database
+password and `GRAFANA_SECRET` — and `GRAFANA_SECRET` is the shared bearer token
+guarding the realm-server's operator endpoints (`_grafana-reindex`,
+`_grafana-add-credit`, `_grafana-upsert-realm-user-permission`,
+`_grafana-revoke-user-sessions`). Never point the preview at it.
+
+Two things keep that grant list short, so keep them true:
+
+- **Previews carry no operator secret.** `render-preview.sh` stamps an inert
+  placeholder into the `grafana_secret` constant unconditionally, so operator
+  buttons on a preview dashboard return 401. Review an operator-button change
+  against the canonical dashboards after merge.
+- **Previews cannot address a canonical dashboard.** `render-preview.sh`
+  rewrites every UID it emits and then fails closed if any manifest lacks the
+  `pr<n>-` prefix, so a bug there costs a preview rather than a live dashboard.
 
 ## Operator actions
 
diff --git a/packages/observability/scripts/ci-apply-preview.sh b/packages/observability/scripts/ci-apply-preview.sh
index f0120cb4115..4a1d1361606 100755
--- a/packages/observability/scripts/ci-apply-preview.sh
+++ b/packages/observability/scripts/ci-apply-preview.sh
@@ -1,10 +1,7 @@
 #!/usr/bin/env bash
 # ci-apply-preview.sh — one environment's half of the PR preview.
 #
-# Called by observability-preview.yml for staging and by
-# observability-preview-production.yml for production. The production
-# caller runs the default branch's copy of this file, never the pull
-# request's — see that workflow's header for why.
+# Called once per environment by observability-preview.yml.
 #
 # Fetches the apply-time secrets for $ENV_NAME from SSM, pushes the per-PR
 # preview with apply-preview.sh, and reports the result back to the workflow
@@ -43,10 +40,12 @@ esac
 
 cd "$(dirname "$0")/.."
 
-# Subset of what observability-apply-<env>.yml fetches. The preview only
-# touches dashboards + folders, so we don't need the LOKI_URL / BOXEL_DB_* /
-# WORKER_LOG_* set — those are read by apply-datasources / apply-alerting,
-# which apply-preview.sh never invokes.
+# These two parameters are the whole grant of the boxel-observability-preview
+# role, and the list is deliberately short: this script runs from the pull
+# request's own checkout, so whatever it can read, a pull request can read.
+# Adding an SSM read here means widening that role and handing the new value
+# to unreviewed code — GRAFANA_SECRET and the Grafana database password are
+# excluded for exactly that reason.
 #
 # GRAFANA_TOKEN is re-fetched by grafanactl-env.sh inside apply-preview.sh;
 # fetching it here too is what registers the `::add-mask::`, so the value is
@@ -59,12 +58,9 @@ ssm() {
 GRAFANA_TOKEN="$(ssm "/${ENV_NAME}/grafana/grafanactl_token" --with-decryption)"
 echo "::add-mask::$GRAFANA_TOKEN"
 
-GRAFANA_SECRET="$(ssm "/${ENV_NAME}/boxel/GRAFANA_SECRET" --with-decryption)"
-echo "::add-mask::$GRAFANA_SECRET"
-
 REALM_SERVER_URL="$(ssm "/${ENV_NAME}/boxel-grafana/realm_server_url")"
 
-export GRAFANA_TOKEN GRAFANA_SECRET REALM_SERVER_URL
+export GRAFANA_TOKEN REALM_SERVER_URL
 
 out_file="$(mktemp)"
 echo "out_file=$out_file" >> "$GITHUB_OUTPUT"
diff --git a/packages/observability/scripts/cleanup-preview.sh b/packages/observability/scripts/cleanup-preview.sh
index d453d640663..297a46f69dd 100755
--- a/packages/observability/scripts/cleanup-preview.sh
+++ b/packages/observability/scripts/cleanup-preview.sh
@@ -23,9 +23,8 @@
 #     gets the old preview resources removed before the new tree lands.
 #   - ci-apply-preview.sh runs this when a PR stops changing dashboards
 #     altogether, to clear a preview an earlier push left behind.
-#   - observability-preview.yml runs this for staging on pull_request
-#     `closed`, and observability-preview-production.yml runs it for
-#     production once that close event has been observed.
+#   - observability-preview.yml runs this on pull_request `closed`, once
+#     per environment.
 #   - observability-preview-sweep.yml runs this for PRs the daily sweep
 #     has decided are stale.
 set -eo pipefail
diff --git a/packages/observability/scripts/render-preview.sh b/packages/observability/scripts/render-preview.sh
index 8ef4c78d1d6..37a0e73aa90 100755
--- a/packages/observability/scripts/render-preview.sh
+++ b/packages/observability/scripts/render-preview.sh
@@ -15,8 +15,8 @@
 #   ./scripts/render-preview.sh --pr <n> --base-ref <ref> [--env <name>]
 #
 # --env defaults to staging and only affects the apply-style template
-# substitutions (REALM_SERVER_URL, GRAFANA_SECRET, __ENV__) — the preview
-# rewrites are env-independent.
+# substitutions (REALM_SERVER_URL, __ENV__) — the preview rewrites are
+# env-independent.
 #
 # Scope: dashboards + folders only. Data sources, alert rules, and the home-
 # dashboard preference are file-provisioned at Grafana startup and can't be
@@ -94,24 +94,30 @@ if [[ -z "$changed_paths" ]]; then
   exit 0
 fi
 
-# Per-env substitution defaults mirror apply.sh exactly so the rendered
-# preview matches what `apply.sh --env <env>` would push.
+# A preview never carries the real operator secret. The canonical
+# dashboards get it from apply.sh; here the `grafana_secret` constant is
+# stamped with an inert placeholder, so the operator buttons on a preview
+# dashboard (Reindex, Add Credit, Grant Permission, Delete Job) get a 401
+# instead of acting on the environment.
+#
+# This is what lets the preview workflow run under an IAM role that cannot
+# read GRAFANA_SECRET at all — which matters because that workflow runs
+# pull-request-controlled code. Assigning unconditionally (rather than
+# honouring an exported GRAFANA_SECRET) keeps it true even if a caller is
+# handed broader credentials later. Review an operator-button change
+# against the canonical dashboards after merge.
+grafana_secret="preview-operator-actions-disabled"
+
+# REALM_SERVER_URL still mirrors apply.sh, so a previewed panel resolves
+# the same endpoint the canonical copy would.
 case "$env_name" in
   local)
     realm_server_url="${REALM_SERVER_URL:-http://localhost:4201/}"
-    if [[ -n "${GRAFANA_SECRET:-}" ]]; then
-      grafana_secret="$GRAFANA_SECRET"
-    else
-      grafana_secret="shhh! it's a secret"
-    fi
     ;;
   staging | production)
     [[ -n "${REALM_SERVER_URL:-}" ]] \
       || { echo "error: REALM_SERVER_URL not set; required for --env=$env_name (CI fetches it from /${env_name}/boxel-grafana/realm_server_url)" >&2; exit 1; }
-    [[ -n "${GRAFANA_SECRET:-}" ]] \
-      || { echo "error: GRAFANA_SECRET not set; required for --env=$env_name" >&2; exit 1; }
     realm_server_url="$REALM_SERVER_URL"
-    grafana_secret="$GRAFANA_SECRET"
     ;;
   *)
     usage_error "unknown env: $env_name (expected local|staging|production)" ;;

From 055a3780844db71120988c9e32cfe6f9369de0ee Mon Sep 17 00:00:00 2001
From: Buck Doyle <buck.doyle@cardstack.com>
Date: Thu, 22 Oct 2026 15:45:57 +0100
Subject: [PATCH 4/6] Give the preview the real operator secret again

A preview whose operator buttons return 401 cannot show a reviewer whether a
button change works. Those buttons are part of what the preview exists to
show.

The preview role stays narrow. It grants three parameters for each
environment, and it still excludes the Grafana database credentials:

- /<env>/grafana/grafanactl_token
- /<env>/boxel-grafana/realm_server_url
- /<env>/boxel/GRAFANA_SECRET

The operator token is weaker than it looks. apply.sh substitutes it into six
canonical dashboards as a `constant` template variable, so every Grafana
viewer can already read it from the dashboard JSON. A pull request that reads
it from CI gains nothing that a dashboard viewer does not have.

CAUTION: the buttons on a preview dashboard act on the real environment. A
button clicked on a production preview performs the production mutation.

render-preview.sh keeps the assertion that every manifest it emits carries
the `pr<n>-` prefix.

cardstack/infra adds the parameter to the role. Apply that change first.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T6hzc2RPYWnjG8xPbmYhf4
---
 .github/workflows/observability-preview.yml   | 26 +++++++-------
 .../observability/scripts/ci-apply-preview.sh | 24 +++++++++----
 .../observability/scripts/render-preview.sh   | 36 ++++++++++---------
 3 files changed, 50 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/observability-preview.yml b/.github/workflows/observability-preview.yml
index 00898936937..6ea5fd252f0 100644
--- a/.github/workflows/observability-preview.yml
+++ b/.github/workflows/observability-preview.yml
@@ -14,13 +14,12 @@ name: Observability — PR preview
 # preview or the comment, which is posted from whatever succeeded.
 #
 # The two environments render the same dashboard set under the same preview
-# UIDs — only the realm-server URL substitution differs — so the comment
-# lists each dashboard once with a link per environment.
+# UIDs — only the apply-time substitutions differ — so the comment lists
+# each dashboard once with a link per environment.
 #
-# A preview never carries the real operator secret: render-preview.sh stamps
-# an inert placeholder into `grafana_secret`, so operator buttons on a
-# preview dashboard return 401. That is what keeps this workflow runnable
-# under a role that cannot read GRAFANA_SECRET.
+# CAUTION: a preview carries the real operator secret, so the action buttons
+# on a preview dashboard act on the real environment. A button clicked on a
+# production preview performs the production mutation.
 #
 # Scope: dashboards + folders only. Data sources, alert rules, and the
 # home-dashboard preference are file-provisioned at Grafana startup and
@@ -55,12 +54,15 @@ env:
   # The dedicated preview role, NOT boxel-observability-apply. This workflow
   # runs on `pull_request`, so GitHub loads it — and every script it calls —
   # from the pull request itself; whatever the assumed role can read, a
-  # contributor can read. boxel-observability-preview grants only
-  # /<env>/grafana/grafanactl_token and /<env>/boxel-grafana/realm_server_url,
-  # so the worst case equals this feature: write `pr<n>-` dashboards. The
-  # apply role additionally grants the Grafana database password and
-  # GRAFANA_SECRET (the realm-server operator bearer token) — never point
-  # this workflow at it.
+  # contributor can read. boxel-observability-preview grants three
+  # parameters per environment and nothing else:
+  #
+  #   /<env>/grafana/grafanactl_token        authenticate grafanactl
+  #   /<env>/boxel-grafana/realm_server_url  fill a template variable
+  #   /<env>/boxel/GRAFANA_SECRET            make operator buttons work
+  #
+  # The apply role additionally grants the Grafana database credentials and
+  # the Loki / Prometheus / DB-host values. Never point this workflow at it.
   STAGING_AWS_ROLE_ARN: arn:aws:iam::680542703984:role/boxel-observability-preview
   PRODUCTION_AWS_ROLE_ARN: arn:aws:iam::120317779495:role/boxel-observability-preview
   PREVIEW_MARKER: "<!-- observability-preview -->"
diff --git a/packages/observability/scripts/ci-apply-preview.sh b/packages/observability/scripts/ci-apply-preview.sh
index 4a1d1361606..b4677c78540 100755
--- a/packages/observability/scripts/ci-apply-preview.sh
+++ b/packages/observability/scripts/ci-apply-preview.sh
@@ -40,12 +40,19 @@ esac
 
 cd "$(dirname "$0")/.."
 
-# These two parameters are the whole grant of the boxel-observability-preview
-# role, and the list is deliberately short: this script runs from the pull
-# request's own checkout, so whatever it can read, a pull request can read.
-# Adding an SSM read here means widening that role and handing the new value
-# to unreviewed code — GRAFANA_SECRET and the Grafana database password are
-# excluded for exactly that reason.
+# These three parameters are the whole grant of the
+# boxel-observability-preview role, and the list stays short on purpose:
+# this script runs from the pull request's own checkout, so whatever it can
+# read, a pull request can read. Adding an SSM read here means widening that
+# role and handing the new value to unreviewed code. The Grafana database
+# credentials are excluded for exactly that reason.
+#
+# GRAFANA_SECRET is included because the operator buttons on a preview
+# dashboard are part of what a reviewer needs to see working. It is a weaker
+# secret than it looks: apply.sh substitutes it into six canonical dashboards
+# as a `constant` template variable, so every Grafana viewer can already read
+# it out of the dashboard JSON. Withholding it here would not keep it from
+# anyone who can reach the preview.
 #
 # GRAFANA_TOKEN is re-fetched by grafanactl-env.sh inside apply-preview.sh;
 # fetching it here too is what registers the `::add-mask::`, so the value is
@@ -58,9 +65,12 @@ ssm() {
 GRAFANA_TOKEN="$(ssm "/${ENV_NAME}/grafana/grafanactl_token" --with-decryption)"
 echo "::add-mask::$GRAFANA_TOKEN"
 
+GRAFANA_SECRET="$(ssm "/${ENV_NAME}/boxel/GRAFANA_SECRET" --with-decryption)"
+echo "::add-mask::$GRAFANA_SECRET"
+
 REALM_SERVER_URL="$(ssm "/${ENV_NAME}/boxel-grafana/realm_server_url")"
 
-export GRAFANA_TOKEN REALM_SERVER_URL
+export GRAFANA_TOKEN GRAFANA_SECRET REALM_SERVER_URL
 
 out_file="$(mktemp)"
 echo "out_file=$out_file" >> "$GITHUB_OUTPUT"
diff --git a/packages/observability/scripts/render-preview.sh b/packages/observability/scripts/render-preview.sh
index 37a0e73aa90..6ca28530b34 100755
--- a/packages/observability/scripts/render-preview.sh
+++ b/packages/observability/scripts/render-preview.sh
@@ -15,8 +15,8 @@
 #   ./scripts/render-preview.sh --pr <n> --base-ref <ref> [--env <name>]
 #
 # --env defaults to staging and only affects the apply-style template
-# substitutions (REALM_SERVER_URL, __ENV__) — the preview rewrites are
-# env-independent.
+# substitutions (REALM_SERVER_URL, GRAFANA_SECRET, __ENV__) — the preview
+# rewrites are env-independent.
 #
 # Scope: dashboards + folders only. Data sources, alert rules, and the home-
 # dashboard preference are file-provisioned at Grafana startup and can't be
@@ -94,30 +94,32 @@ if [[ -z "$changed_paths" ]]; then
   exit 0
 fi
 
-# A preview never carries the real operator secret. The canonical
-# dashboards get it from apply.sh; here the `grafana_secret` constant is
-# stamped with an inert placeholder, so the operator buttons on a preview
-# dashboard (Reindex, Add Credit, Grant Permission, Delete Job) get a 401
-# instead of acting on the environment.
+# Per-env substitution defaults mirror apply.sh exactly so the rendered
+# preview matches what `apply.sh --env <env>` would push. The real
+# GRAFANA_SECRET goes in, so the operator buttons on a preview dashboard
+# (Reindex, Add Credit, Grant Permission, Delete Job) behave as they will
+# once the change merges — a preview whose buttons 401 cannot show whether
+# a button change works.
 #
-# This is what lets the preview workflow run under an IAM role that cannot
-# read GRAFANA_SECRET at all — which matters because that workflow runs
-# pull-request-controlled code. Assigning unconditionally (rather than
-# honouring an exported GRAFANA_SECRET) keeps it true even if a caller is
-# handed broader credentials later. Review an operator-button change
-# against the canonical dashboards after merge.
-grafana_secret="preview-operator-actions-disabled"
-
-# REALM_SERVER_URL still mirrors apply.sh, so a previewed panel resolves
-# the same endpoint the canonical copy would.
+# CAUTION: those buttons act on the real environment. A button clicked on a
+# production preview performs the production mutation; it is not a
+# rehearsal. Prefer the staging preview for exercising one.
 case "$env_name" in
   local)
     realm_server_url="${REALM_SERVER_URL:-http://localhost:4201/}"
+    if [[ -n "${GRAFANA_SECRET:-}" ]]; then
+      grafana_secret="$GRAFANA_SECRET"
+    else
+      grafana_secret="shhh! it's a secret"
+    fi
     ;;
   staging | production)
     [[ -n "${REALM_SERVER_URL:-}" ]] \
       || { echo "error: REALM_SERVER_URL not set; required for --env=$env_name (CI fetches it from /${env_name}/boxel-grafana/realm_server_url)" >&2; exit 1; }
+    [[ -n "${GRAFANA_SECRET:-}" ]] \
+      || { echo "error: GRAFANA_SECRET not set; required for --env=$env_name" >&2; exit 1; }
     realm_server_url="$REALM_SERVER_URL"
+    grafana_secret="$GRAFANA_SECRET"
     ;;
   *)
     usage_error "unknown env: $env_name (expected local|staging|production)" ;;

From e11a2cd3010de4867c9b145ddeb2d622d054abf0 Mon Sep 17 00:00:00 2001
From: Buck Doyle <buck.doyle@cardstack.com>
Date: Fri, 23 Oct 2026 12:33:41 +0100
Subject: [PATCH 5/6] Point the preview at a scoped Grafana service account

The preview used /<env>/grafana/grafanactl_token. That token belongs to an
Admin service account, and nothing in Grafana scopes it to `pr<n>-` UIDs, so
a pull request holding it could overwrite or delete any canonical dashboard.
It could also read GRAFANA_SECRET without a grant, because apply.sh
substitutes the live secret into six canonical dashboards and
cleanup-preview.sh pulls every dashboard on each close.

The preview now authenticates as `observability-preview`: Viewer, plus Admin
on the `Previews` folder and nothing else.

- grafanactl-env.sh takes GRAFANA_TOKEN_SSM_PATH. It still defaults to the
  Admin parameter, so the canonical tooling is unchanged.
- apply-preview.sh and cleanup-preview.sh set that path to
  /<env>/grafana/preview_grafanactl_token. Both are preview-only entry
  points, so every caller inherits it, including the daily sweep.
- render-preview.sh nests each per-PR folder under `Previews`. The scoped
  account can create a folder only inside that parent, so a root-level
  preview folder is refused.
- The UID assertion now reads the manifest kind. A dashboard must sit in this
  pull request's own folder; a preview folder's only permitted parent is the
  shared `Previews` one.

Measured end to end against staging with the scoped credential alone: the
apply pushed one folder and five dashboards into `Previews`, the canonical
Overview dashboard stayed at its existing version, and the cleanup removed
all six with no errors.

A pull request can still read every dashboard, and so can still read
GRAFANA_SECRET. That half closes when operator calls move to a Grafana
datasource proxy. The README and the config comments say so rather than
implying the preview is fully fenced.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T6hzc2RPYWnjG8xPbmYhf4
---
 .../workflows/observability-preview-sweep.yml |  2 +-
 .github/workflows/observability-preview.yml   | 12 +++-
 packages/observability/README.md              | 56 +++++++++++++------
 .../observability/scripts/apply-preview.sh    |  5 ++
 .../observability/scripts/ci-apply-preview.sh |  8 ++-
 .../observability/scripts/cleanup-preview.sh  |  4 ++
 .../observability/scripts/grafanactl-env.sh   | 13 +++--
 .../observability/scripts/render-preview.sh   | 42 ++++++++++++--
 8 files changed, 112 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/observability-preview-sweep.yml b/.github/workflows/observability-preview-sweep.yml
index 3f058de79f4..3b5c8658e09 100644
--- a/.github/workflows/observability-preview-sweep.yml
+++ b/.github/workflows/observability-preview-sweep.yml
@@ -98,7 +98,7 @@ jobs:
           ENV_NAME: ${{ matrix.env }}
         run: |
           GRAFANA_TOKEN="$(aws ssm get-parameter \
-            --name "/${ENV_NAME}/grafana/grafanactl_token" \
+            --name "/${ENV_NAME}/grafana/preview_grafanactl_token" \
             --with-decryption \
             --query 'Parameter.Value' \
             --output text)"
diff --git a/.github/workflows/observability-preview.yml b/.github/workflows/observability-preview.yml
index 6ea5fd252f0..f2dfd7790f4 100644
--- a/.github/workflows/observability-preview.yml
+++ b/.github/workflows/observability-preview.yml
@@ -57,9 +57,15 @@ env:
   # contributor can read. boxel-observability-preview grants three
   # parameters per environment and nothing else:
   #
-  #   /<env>/grafana/grafanactl_token        authenticate grafanactl
-  #   /<env>/boxel-grafana/realm_server_url  fill a template variable
-  #   /<env>/boxel/GRAFANA_SECRET            make operator buttons work
+  #   /<env>/grafana/preview_grafanactl_token  scoped Grafana account
+  #   /<env>/boxel-grafana/realm_server_url    fill a template variable
+  #   /<env>/boxel/GRAFANA_SECRET              make operator buttons work
+  #
+  # The Grafana token belongs to `observability-preview`: Viewer, plus Admin
+  # on the `Previews` folder. So a pull request cannot overwrite or delete a
+  # canonical dashboard. It can still read them, and therefore read
+  # GRAFANA_SECRET out of the six that carry it — that half closes when
+  # operator calls move to a Grafana datasource proxy.
   #
   # The apply role additionally grants the Grafana database credentials and
   # the Loki / Prometheus / DB-host values. Never point this workflow at it.
diff --git a/packages/observability/README.md b/packages/observability/README.md
index d337564b025..d6682814a3b 100644
--- a/packages/observability/README.md
+++ b/packages/observability/README.md
@@ -482,29 +482,53 @@ weak credential.** It runs on `pull_request`, and GitHub loads that workflow —
 and every script it calls — from the PR itself. Whatever the assumed role can
 read, a contributor can read. So the preview uses its own IAM role,
 `boxel-observability-preview` (cardstack/infra,
-`configs/boxel-observability-preview/`), which grants exactly two parameters
-per environment:
+`configs/boxel-observability-preview/`), granting three parameters per
+environment:
 
 ```
-/<env>/grafana/grafanactl_token        authenticate grafanactl
-/<env>/boxel-grafana/realm_server_url  fill the realm_server template variable
+/<env>/grafana/preview_grafanactl_token  scoped Grafana service account
+/<env>/boxel-grafana/realm_server_url    fill the realm_server variable
+/<env>/boxel/GRAFANA_SECRET              make the operator buttons work
 ```
 
-Not `boxel-observability-apply`. That role also grants the Grafana database
-password and `GRAFANA_SECRET` — and `GRAFANA_SECRET` is the shared bearer token
-guarding the realm-server's operator endpoints (`_grafana-reindex`,
-`_grafana-add-credit`, `_grafana-upsert-realm-user-permission`,
-`_grafana-revoke-user-sessions`). Never point the preview at it.
+Not `boxel-observability-apply`. That role grants the Grafana database
+credentials and, worse, the **Admin** `grafanactl_token`. Nothing in Grafana
+scopes that token to `pr<n>-` UIDs, so a PR holding it could overwrite or
+delete any canonical dashboard — and could read `GRAFANA_SECRET` without being
+granted it, since `apply.sh` substitutes the live secret into six canonical
+dashboards and `cleanup-preview.sh` pulls every dashboard on each PR close.
 
-Two things keep that grant list short, so keep them true:
+The preview token therefore belongs to a separate Grafana service account,
+`observability-preview` (cardstack/infra, `configs/grafana/base/main.tf`):
+**Viewer, plus Admin on the `Previews` folder and nothing else.** Every per-PR
+folder is nested inside `Previews`, which is what makes that single grant
+sufficient — a root-level preview folder is refused.
+
+What a PR can still do, stated plainly:
+
+- write and delete dashboards and folders **inside `Previews`**
+- read every dashboard, and so read `GRAFANA_SECRET` out of the six carrying it
+- use `GRAFANA_SECRET` to call the realm-server operator endpoints
+
+It **cannot** overwrite or delete a canonical dashboard. That was the
+irreversible risk and it is closed. The read half stays open until operator
+calls move to a Grafana datasource proxy and the secret leaves dashboard
+bodies; the preview grant should be dropped when that lands.
+
+`role = "None"` would close the read half too, and Grafana enforces the folder
+scope correctly under it — but grafanactl drives the App Platform APIs, which
+403 for an account with no basic role however its folder permissions are set.
+Push, pull and delete all fail, so the preview tooling cannot run under it.
+Only the legacy REST API honours folder scoping.
+
+Two mechanisms keep the preview inside its lane, so keep them true:
 
-- **Previews carry no operator secret.** `render-preview.sh` stamps an inert
-  placeholder into the `grafana_secret` constant unconditionally, so operator
-  buttons on a preview dashboard return 401. Review an operator-button change
-  against the canonical dashboards after merge.
 - **Previews cannot address a canonical dashboard.** `render-preview.sh`
-  rewrites every UID it emits and then fails closed if any manifest lacks the
-  `pr<n>-` prefix, so a bug there costs a preview rather than a live dashboard.
+  rewrites every UID it emits, then fails closed unless each dashboard sits in
+  this PR's own folder and each preview folder's only parent is `Previews`.
+- **Operator buttons act on the real environment.** A preview carries the real
+  `GRAFANA_SECRET`, so a button clicked on a production preview performs the
+  production mutation. Prefer the staging preview for exercising one.
 
 ## Operator actions
 
diff --git a/packages/observability/scripts/apply-preview.sh b/packages/observability/scripts/apply-preview.sh
index 69d533d7a00..5d5fdf772ff 100755
--- a/packages/observability/scripts/apply-preview.sh
+++ b/packages/observability/scripts/apply-preview.sh
@@ -60,6 +60,11 @@ done
 
 cd "$(dirname "$0")/.."
 
+# Previews authenticate as the scoped `observability-preview` account, never
+# the Admin one the canonical apply uses. See cardstack/infra,
+# configs/grafana/base/main.tf.
+export GRAFANA_TOKEN_SSM_PATH="${GRAFANA_TOKEN_SSM_PATH:-/${env_name}/grafana/preview_grafanactl_token}"
+
 # shellcheck source=./grafanactl-env.sh
 source ./scripts/grafanactl-env.sh "$env_name"
 
diff --git a/packages/observability/scripts/ci-apply-preview.sh b/packages/observability/scripts/ci-apply-preview.sh
index b4677c78540..f1404f15ec0 100755
--- a/packages/observability/scripts/ci-apply-preview.sh
+++ b/packages/observability/scripts/ci-apply-preview.sh
@@ -54,6 +54,12 @@ cd "$(dirname "$0")/.."
 # it out of the dashboard JSON. Withholding it here would not keep it from
 # anyone who can reach the preview.
 #
+# The Grafana token is the `observability-preview` service account, not the
+# Admin `grafanactl_token` the canonical apply uses. It is Viewer plus Admin
+# on the `Previews` folder, so a pull request holding it cannot overwrite or
+# delete a canonical dashboard. It can still read them, which is why
+# GRAFANA_SECRET below is not meaningfully protected by withholding it.
+#
 # GRAFANA_TOKEN is re-fetched by grafanactl-env.sh inside apply-preview.sh;
 # fetching it here too is what registers the `::add-mask::`, so the value is
 # redacted if grafanactl ever echoes it on a failure path.
@@ -62,7 +68,7 @@ ssm() {
     --query 'Parameter.Value' --output text
 }
 
-GRAFANA_TOKEN="$(ssm "/${ENV_NAME}/grafana/grafanactl_token" --with-decryption)"
+GRAFANA_TOKEN="$(ssm "/${ENV_NAME}/grafana/preview_grafanactl_token" --with-decryption)"
 echo "::add-mask::$GRAFANA_TOKEN"
 
 GRAFANA_SECRET="$(ssm "/${ENV_NAME}/boxel/GRAFANA_SECRET" --with-decryption)"
diff --git a/packages/observability/scripts/cleanup-preview.sh b/packages/observability/scripts/cleanup-preview.sh
index 297a46f69dd..9a728fa4f54 100755
--- a/packages/observability/scripts/cleanup-preview.sh
+++ b/packages/observability/scripts/cleanup-preview.sh
@@ -56,6 +56,10 @@ done
 
 cd "$(dirname "$0")/.."
 
+# Same scoped account as apply-preview.sh — teardown needs no more reach
+# than the push did.
+export GRAFANA_TOKEN_SSM_PATH="${GRAFANA_TOKEN_SSM_PATH:-/${env_name}/grafana/preview_grafanactl_token}"
+
 # shellcheck source=./grafanactl-env.sh
 source ./scripts/grafanactl-env.sh "$env_name"
 
diff --git a/packages/observability/scripts/grafanactl-env.sh b/packages/observability/scripts/grafanactl-env.sh
index 3b8a7a9df67..a8396afc32d 100755
--- a/packages/observability/scripts/grafanactl-env.sh
+++ b/packages/observability/scripts/grafanactl-env.sh
@@ -7,8 +7,12 @@
 #   source ./scripts/grafanactl-env.sh production
 #
 # Local is a no-op (admin/admin in-config is sufficient).
-# Staging and production fetch the token from SSM at /<env>/grafana/grafanactl_token,
-# which is populated by the cardstack/infra Terraform.
+# Staging and production fetch the token from SSM. The default parameter is
+# /<env>/grafana/grafanactl_token — the Admin service account that applies
+# the canonical dashboards. Set GRAFANA_TOKEN_SSM_PATH to override it: the
+# preview tooling points at /<env>/grafana/preview_grafanactl_token, a
+# Viewer account holding Admin on the `Previews` folder only. Both are
+# populated by the cardstack/infra Terraform.
 #
 # Requires AWS credentials with ssm:GetParameter on /<env>/grafana/* — the same
 # creds devs already use for `/terraform-plan workspace=grafana/<env>`.
@@ -36,12 +40,13 @@ _grafanactl_env_main() {
     staging | production)
       # Capture stderr so we can show it on failure.
       if ! token_val="$(aws ssm get-parameter \
-          --name "/${env_name}/grafana/grafanactl_token" \
+          --name "${GRAFANA_TOKEN_SSM_PATH:-/${env_name}/grafana/grafanactl_token}" \
           --with-decryption \
           --query 'Parameter.Value' \
           --output text 2>&1)"; then
         fetch_err="$token_val"
-        printf 'error: failed to fetch /%s/grafana/grafanactl_token from SSM:\n' "$env_name" >&2
+        printf 'error: failed to fetch %s from SSM:\n' \
+          "${GRAFANA_TOKEN_SSM_PATH:-/${env_name}/grafana/grafanactl_token}" >&2
         printf '%s\n' "$fetch_err" >&2
         printf 'hint: confirm AWS creds active for the right account and ssm:GetParameter is granted.\n' >&2
         return 1
diff --git a/packages/observability/scripts/render-preview.sh b/packages/observability/scripts/render-preview.sh
index 6ca28530b34..56e7fa0135c 100755
--- a/packages/observability/scripts/render-preview.sh
+++ b/packages/observability/scripts/render-preview.sh
@@ -104,9 +104,17 @@ fi
 # CAUTION: those buttons act on the real environment. A button clicked on a
 # production preview performs the production mutation; it is not a
 # rehearsal. Prefer the staging preview for exercising one.
+# UID of the shared parent folder that every per-PR preview folder nests
+# under. Fixed, because cardstack/infra creates it with this UID and grants
+# the preview service account Admin on it by name.
+preview_parent_folder="previews"
+
 case "$env_name" in
   local)
     realm_server_url="${REALM_SERVER_URL:-http://localhost:4201/}"
+    # The docker-compose Grafana has no `Previews` folder, and pushing a
+    # child of a missing parent fails.
+    preview_parent_folder=""
     if [[ -n "${GRAFANA_SECRET:-}" ]]; then
       grafana_secret="$GRAFANA_SECRET"
     else
@@ -229,8 +237,14 @@ done < <(find "$rendered/dashboards" -type f -name '*.json' -print0)
 find "$rendered/dashboards" -type d -empty -delete 2>/dev/null || true
 
 # Step 3 — folders: keep only those referenced by a kept dashboard, rewrite
-# UID + title. Drop the rest (so the push doesn't create unused per-PR
-# folders for unchanged areas).
+# UID + title, and nest them under the shared `Previews` parent. Drop the
+# rest (so the push doesn't create unused per-PR folders for unchanged
+# areas).
+#
+# The nesting is what makes the scoped credential sufficient. The
+# `observability-preview` service account is Viewer plus Admin on `Previews`
+# only, so it can create a folder inside that parent and nowhere else; a
+# root-level preview folder is refused.
 while IFS= read -r -d '' f; do
   orig_uid="$(jq -r '.metadata.name // ""' "$f")"
   if [[ -z "$orig_uid" ]] || [[ "$referenced_folders" != *"${orig_uid}"$'\n'* ]]; then
@@ -238,9 +252,15 @@ while IFS= read -r -d '' f; do
     continue
   fi
   new_uid="$(prefix_uid "$pr_number" "$orig_uid")"
-  jq --arg uid "$new_uid" --arg prefix "Preview – PR #${pr_number}: " '
+  jq --arg uid "$new_uid" \
+     --arg prefix "Preview – PR #${pr_number}: " \
+     --arg parent "$preview_parent_folder" '
     .metadata.name = $uid
     | (if .spec.title then .spec.title = ($prefix + .spec.title) else . end)
+    | (if $parent != "" then
+         .metadata.annotations =
+           ((.metadata.annotations // {}) + {"grafana.app/folder": $parent})
+       else . end)
   ' "$f" > "$f.tmp"
   mv "$f.tmp" "$f"
 done < <(find "$rendered/folders" -type f -name '*.json' -print0)
@@ -269,8 +289,20 @@ while IFS= read -r -d '' f; do
     exit 1
   fi
   folder="$(jq -r '(.metadata.annotations // {})["grafana.app/folder"] // ""' "$f")"
-  if [[ -n "$folder" && "$folder" != "pr${pr_number}-"* ]]; then
-    echo "error: $f points at folder '$folder', which lacks the pr${pr_number}- prefix" >&2
+  kind="$(jq -r '.kind // ""' "$f")"
+  # A preview FOLDER is allowed exactly one parent: the shared `Previews`
+  # folder, which has no `pr<n>-` prefix because it is not per-PR. Everything
+  # else — every dashboard — must sit in this PR's own folder, so that a
+  # manifest cannot drop a dashboard into a canonical folder.
+  allowed_parent=""
+  if [[ "$kind" == "Folder" ]]; then
+    allowed_parent="$preview_parent_folder"
+  fi
+  if [[ -n "$folder" \
+        && "$folder" != "pr${pr_number}-"* \
+        && ( -z "$allowed_parent" || "$folder" != "$allowed_parent" ) ]]; then
+    echo "error: $f points at folder '$folder', which is neither this PR's" \
+         "folder nor the shared preview parent" >&2
     rm -rf "$rendered"
     exit 1
   fi

From dbf918e32c658d22ad398936b7093127786a6f04 Mon Sep 17 00:00:00 2001
From: Buck Doyle <buck.doyle@cardstack.com>
Date: Fri, 23 Oct 2026 13:11:30 +0100
Subject: [PATCH 6/6] Say what the sweep's non-main guard actually covers

A review pointed out that the comment claimed more than the step delivers.
`workflow_dispatch` runs the chosen ref's copy of this workflow, so it runs
the chosen ref's copy of the guard too. The guard cannot stop a rewritten
workflow.

It does stop an operator dispatching a stale but honest branch, which is a
real mistake, so the step stays. The comment now says that, and points at the
control that does limit a rewritten copy: the scoped Grafana account the
sweep assumes.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T6hzc2RPYWnjG8xPbmYhf4
---
 .github/workflows/observability-preview-sweep.yml | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/observability-preview-sweep.yml b/.github/workflows/observability-preview-sweep.yml
index 3b5c8658e09..1297866c2bc 100644
--- a/.github/workflows/observability-preview-sweep.yml
+++ b/.github/workflows/observability-preview-sweep.yml
@@ -65,11 +65,16 @@ jobs:
 
     steps:
       - name: Refuse to sweep production from a non-main ref
-        # `schedule` always runs the default branch, but
-        # `workflow_dispatch` lets an operator choose any ref — including
-        # one whose copy of this workflow was rewritten to do something
-        # else with the production role. Mirrors the guard in
-        # observability-apply-production.yml.
+        # `schedule` always runs the default branch; `workflow_dispatch` lets
+        # an operator choose any ref. This guard catches dispatching a stale
+        # but honest branch, which is a real mistake — nothing more.
+        #
+        # It cannot stop a ref whose copy of this workflow was rewritten,
+        # because workflow_dispatch runs the chosen ref's copy of the guard
+        # too. What limits the damage there is the credential, not this step:
+        # the sweep assumes boxel-observability-preview, whose Grafana account
+        # is Viewer plus Admin on the `Previews` folder, so a rewritten copy
+        # still cannot touch a canonical dashboard.
         if: matrix.env == 'production'
         env:
           REF: ${{ github.ref }}