ci(canary): Open one issue per failed canary test application

.github/workflows/canary.yml

@@ -14,7 +14,6 @@ env:
 
 permissions:
   contents: read
-  issues: write
 
 jobs:
   job_e2e_prepare:
@@ -122,13 +121,30 @@ jobs:
         timeout-minutes: 15
         run: ${{ matrix.assert-command || 'pnpm test:assert' }}
 
-      - name: Create Issue
-        if: failure() && github.event_name == 'schedule'
-        uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5
+  job_report_failures:
+    name: Report canary failures
+    needs: [job_e2e_prepare, job_e2e_tests]
+    if: always() && github.event_name == 'schedule' && needs.job_e2e_tests.result == 'failure'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      # To find the failed jobs of this run
+      actions: read
+      issues: write
+    steps:
+      - name: Check out current commit
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+          persist-credentials: false
+      - name: Open or update an issue per failed test application
+        uses: actions/github-script@v9
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          TITLE: ${{ matrix.label || matrix.test-application }} Test Failed
+          CANARY_MATRIX: ${{ needs.job_e2e_prepare.outputs.matrix }}
         with:
-          filename: .github/CANARY_FAILURE_TEMPLATE.md
-          update_existing: true
+          script: |
+            const { default: run } = await import(
+              `${process.env.GITHUB_WORKSPACE}/scripts/report-canary-failures.mjs`
+            );
+            await run({ github, context, core, matrix: JSON.parse(process.env.CANARY_MATRIX) });

scripts/report-canary-failures.mjs

@@ -0,0 +1,116 @@
+const ISSUE_LABELS = ['Type: Tests', 'Waiting for: Product Owner'];
+const FAILED_CONCLUSIONS = ['failure', 'timed_out'];
+
+export function issueTitle(testApplication) {
+  return `Canary tests failed: ${testApplication}`;
+}
+
+// Must match the `name` of `job_e2e_tests` in .github/workflows/canary.yml
+function jobName(include) {
+  return `E2E ${include.label || include['test-application']} Test`;
+}
+
+/**
+ * Groups the failed canary jobs of a run by test application.
+ *
+ * @param {{ include: Array<Record<string, string>> }} matrix
+ * @param {Array<{ name: string; conclusion: string | null; html_url: string | null }>} jobs
+ * @returns {Map<string, Array<{ label: string; url: string | null }>>}
+ */
+export function getFailedVariantsByApp(matrix, jobs) {
+  const failedJobs = new Map(
+    jobs.filter(job => FAILED_CONCLUSIONS.includes(job.conclusion)).map(job => [job.name, job.html_url]),
+  );
+
+  const failedVariantsByApp = new Map();
+  for (const include of matrix.include) {
+    const name = jobName(include);
+    if (!failedJobs.has(name)) {
+      continue;
+    }
+
+    const app = include['test-application'];
+    const variants = failedVariantsByApp.get(app) || [];
+    variants.push({ label: include.label || app, url: failedJobs.get(name) });
+    failedVariantsByApp.set(app, variants);
+  }
+
+  return failedVariantsByApp;
+}
+
+function renderVariants(variants) {
+  return variants.map(({ label, url }) => (url ? `- [${label}](${url})` : `- ${label}`)).join('\n');
+}
+
+function renderIssueBody(app, variants, runUrl) {
+  return `The canary tests for \`${app}\` failed: ${runUrl}
+
+Failed variants:
+${renderVariants(variants)}
+
+Canary tests run the \`canaryVariants\` of the test application against upstream prereleases. Until this issue is
+closed, new failures of \`${app}\` are posted as comments here.`;
+}
+
+function renderComment(variants, runUrl) {
+  return `Still failing: ${runUrl}
+
+${renderVariants(variants)}`;
+}
+
+/**
+ * Opens one issue per failed test application, or comments on the open issue for that application.
+ */
+export default async function run({ github, context, core, matrix }) {
+  const { owner, repo } = context.repo;
+  const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
+
+  const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
+    owner,
+    repo,
+    run_id: context.runId,
+    filter: 'latest',
+    per_page: 100,
+  });
+
+  const failedVariantsByApp = getFailedVariantsByApp(matrix, jobs);
+  if (failedVariantsByApp.size === 0) {
+    core.info('No failed canary tests found.');
+    return;
+  }
+
+  const openIssues = await github.paginate(github.rest.issues.listForRepo, {
+    owner,
+    repo,
+    state: 'open',
+    per_page: 100,
+  });
+  const openIssuesByTitle = new Map(
+    openIssues.filter(issue => !issue.pull_request).map(issue => [issue.title, issue.number]),
+  );
+
+  for (const [app, variants] of failedVariantsByApp) {
+    const title = issueTitle(app);
+    const existingIssueNumber = openIssuesByTitle.get(title);
+
+    if (existingIssueNumber) {
+      await github.rest.issues.createComment({
+        owner,
+        repo,
+        issue_number: existingIssueNumber,
+        body: renderComment(variants, runUrl),
+      });
+      core.info(`${app}: commented on #${existingIssueNumber}`);
+      continue;
+    }
+
+    const { data: issue } = await github.rest.issues.create({
+      owner,
+      repo,
+      title,
+      labels: ISSUE_LABELS,
+      body: renderIssueBody(app, variants, runUrl),
+    });
+    core.info(`${app}: opened #${issue.number}`);
+  }
+}

scripts/report-canary-failures.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it, vi } from 'vitest';
+import run, { getFailedVariantsByApp, issueTitle } from './report-canary-failures.mjs';
+
+const matrix = {
+  include: [
+    { 'test-application': 'nextjs-16', label: 'nextjs-16 (canary, turbopack)' },
+    { 'test-application': 'nextjs-16', label: 'nextjs-16 (canary, webpack)' },
+    { 'test-application': 'nuxt-4', label: 'nuxt-4 (canary)' },
+    { 'test-application': 'vue-3' },
+  ],
+};
+
+const jobs = [
+  { name: 'Prepare E2E Canary tests', conclusion: 'success', html_url: 'https://jobs/prepare' },
+  { name: 'E2E nextjs-16 (canary, turbopack) Test', conclusion: 'failure', html_url: 'https://jobs/1' },
+  { name: 'E2E nextjs-16 (canary, webpack) Test', conclusion: 'timed_out', html_url: 'https://jobs/2' },
+  { name: 'E2E nuxt-4 (canary) Test', conclusion: 'success', html_url: 'https://jobs/3' },
+  { name: 'E2E vue-3 Test', conclusion: 'failure', html_url: 'https://jobs/4' },
+];
+
+function createGithub(openIssues: Array<{ number: number; title: string; pull_request?: object }>) {
+  const listJobsForWorkflowRun = vi.fn();
+  const listForRepo = vi.fn();
+  const createComment = vi.fn();
+  const create = vi.fn().mockResolvedValue({ data: { number: 123 } });
+
+  return {
+    rest: {
+      actions: { listJobsForWorkflowRun },
+      issues: { listForRepo, createComment, create },
+    },
+    paginate: vi.fn(async (method: unknown) => (method === listJobsForWorkflowRun ? jobs : openIssues)),
+  };
+}
+
+const context = { repo: { owner: 'getsentry', repo: 'sentry-javascript' }, serverUrl: 'https://github.com', runId: 42 };
+const core = { info: vi.fn() };
+const runUrl = 'https://github.com/getsentry/sentry-javascript/actions/runs/42';
+
+describe('getFailedVariantsByApp', () => {
+  it('groups failed and timed out jobs by test application', () => {
+    expect(getFailedVariantsByApp(matrix, jobs)).toEqual(
+      new Map([
+        [
+          'nextjs-16',
+          [
+            { label: 'nextjs-16 (canary, turbopack)', url: 'https://jobs/1' },
+            { label: 'nextjs-16 (canary, webpack)', url: 'https://jobs/2' },
+          ],
+        ],
+        ['vue-3', [{ label: 'vue-3', url: 'https://jobs/4' }]],
+      ]),
+    );
+  });
+});
+
+describe('run', () => {
+  it('opens one issue per failed test application', async () => {
+    const github = createGithub([]);
+
+    await run({ github, context, core, matrix });
+
+    expect(github.rest.issues.create).toHaveBeenCalledTimes(2);
+    expect(github.rest.issues.create).toHaveBeenCalledWith({
+      owner: 'getsentry',
+      repo: 'sentry-javascript',
+      title: issueTitle('nextjs-16'),
+      labels: ['Type: Tests', 'Waiting for: Product Owner'],
+      body: expect.stringContaining(runUrl),
+    });
+    expect(github.rest.issues.create).toHaveBeenCalledWith(expect.objectContaining({ title: issueTitle('vue-3') }));
+    expect(github.rest.issues.createComment).not.toHaveBeenCalled();
+  });
+
+  it('comments on the open issue of a test application instead of opening another one', async () => {
+    const github = createGithub([
+      { number: 7, title: issueTitle('nextjs-16') },
+      { number: 8, title: issueTitle('vue-3'), pull_request: {} },
+    ]);
+
+    await run({ github, context, core, matrix });
+
+    expect(github.rest.issues.createComment).toHaveBeenCalledTimes(1);
+    expect(github.rest.issues.createComment).toHaveBeenCalledWith({
+      owner: 'getsentry',
+      repo: 'sentry-javascript',
+      issue_number: 7,
+      body: expect.stringContaining(runUrl),
+    });
+    expect(github.rest.issues.create).toHaveBeenCalledTimes(1);
+    expect(github.rest.issues.create).toHaveBeenCalledWith(expect.objectContaining({ title: issueTitle('vue-3') }));
+  });
+
+  it('does nothing when no canary test failed', async () => {
+    const github = createGithub([]);
+
+    await run({ github, context, core, matrix: { include: [matrix.include[2]] } });
+
+    expect(github.paginate).toHaveBeenCalledTimes(1);
+    expect(github.rest.issues.create).not.toHaveBeenCalled();
+    expect(github.rest.issues.createComment).not.toHaveBeenCalled();
+  });
+});