From f0b84bf8ac98632ae2eb680ef98500b888b6fa62 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:24:03 +0000 Subject: [PATCH] ci(canary): Open one issue per failed canary test application A report job opens an issue for each test application with failed canary variants. If an issue for the application is still open, it comments on it with the link to the failed run instead. Co-Authored-By: Francesco Gringl-Novy --- .github/CANARY_FAILURE_TEMPLATE.md | 6 -- .github/workflows/canary.yml | 34 ++++++-- scripts/report-canary-failures.mjs | 116 +++++++++++++++++++++++++ scripts/report-canary-failures.test.ts | 103 ++++++++++++++++++++++ 4 files changed, 244 insertions(+), 15 deletions(-) delete mode 100644 .github/CANARY_FAILURE_TEMPLATE.md create mode 100644 scripts/report-canary-failures.mjs create mode 100644 scripts/report-canary-failures.test.ts diff --git a/.github/CANARY_FAILURE_TEMPLATE.md b/.github/CANARY_FAILURE_TEMPLATE.md deleted file mode 100644 index 9e05fcfc44ae..000000000000 --- a/.github/CANARY_FAILURE_TEMPLATE.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: '{{ env.TITLE }}' -labels: 'Type: Tests, Waiting for: Product Owner' ---- - -Canary tests failed: {{ env.RUN_LINK }} diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 337c22acd36f..b9980222b755 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -14,7 +14,6 @@ env: permissions: contents: read - issues: write jobs: job_e2e_prepare: @@ -121,13 +120,30 @@ jobs: timeout-minutes: 15 run: ${{ matrix.assert-command || 'pnpm test:assert' }} - - name: Create Issue - if: failure() && github.event_name == 'schedule' - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 + job_report_failures: + name: Report canary failures + needs: [job_e2e_prepare, job_e2e_tests] + if: always() && github.event_name == 'schedule' && needs.job_e2e_tests.result == 'failure' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + # To find the failed jobs of this run + actions: read + issues: write + steps: + - name: Check out current commit + uses: actions/checkout@v7 + with: + ref: ${{ env.HEAD_COMMIT }} + persist-credentials: false + - name: Open or update an issue per failed test application + uses: actions/github-script@v9 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - TITLE: ${{ matrix.label || matrix.test-application }} Test Failed + CANARY_MATRIX: ${{ needs.job_e2e_prepare.outputs.matrix }} with: - filename: .github/CANARY_FAILURE_TEMPLATE.md - update_existing: true + script: | + const { default: run } = await import( + `${process.env.GITHUB_WORKSPACE}/scripts/report-canary-failures.mjs` + ); + await run({ github, context, core, matrix: JSON.parse(process.env.CANARY_MATRIX) }); diff --git a/scripts/report-canary-failures.mjs b/scripts/report-canary-failures.mjs new file mode 100644 index 000000000000..7d1461b222f1 --- /dev/null +++ b/scripts/report-canary-failures.mjs @@ -0,0 +1,116 @@ +const ISSUE_LABELS = ['Type: Tests', 'Waiting for: Product Owner']; +const FAILED_CONCLUSIONS = ['failure', 'timed_out']; + +export function issueTitle(testApplication) { + return `Canary tests failed: ${testApplication}`; +} + +// Must match the `name` of `job_e2e_tests` in .github/workflows/canary.yml +function jobName(include) { + return `E2E ${include.label || include['test-application']} Test`; +} + +/** + * Groups the failed canary jobs of a run by test application. + * + * @param {{ include: Array> }} matrix + * @param {Array<{ name: string; conclusion: string | null; html_url: string | null }>} jobs + * @returns {Map>} + */ +export function getFailedVariantsByApp(matrix, jobs) { + const failedJobs = new Map( + jobs.filter(job => FAILED_CONCLUSIONS.includes(job.conclusion)).map(job => [job.name, job.html_url]), + ); + + const failedVariantsByApp = new Map(); + for (const include of matrix.include) { + const name = jobName(include); + if (!failedJobs.has(name)) { + continue; + } + + const app = include['test-application']; + const variants = failedVariantsByApp.get(app) || []; + variants.push({ label: include.label || app, url: failedJobs.get(name) }); + failedVariantsByApp.set(app, variants); + } + + return failedVariantsByApp; +} + +function renderVariants(variants) { + return variants.map(({ label, url }) => (url ? `- [${label}](${url})` : `- ${label}`)).join('\n'); +} + +function renderIssueBody(app, variants, runUrl) { + return `The canary tests for \`${app}\` failed: ${runUrl} + +Failed variants: +${renderVariants(variants)} + +Canary tests run the \`canaryVariants\` of the test application against upstream prereleases. Until this issue is +closed, new failures of \`${app}\` are posted as comments here.`; +} + +function renderComment(variants, runUrl) { + return `Still failing: ${runUrl} + +${renderVariants(variants)}`; +} + +/** + * Opens one issue per failed test application, or comments on the open issue for that application. + */ +export default async function run({ github, context, core, matrix }) { + const { owner, repo } = context.repo; + const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner, + repo, + run_id: context.runId, + filter: 'latest', + per_page: 100, + }); + + const failedVariantsByApp = getFailedVariantsByApp(matrix, jobs); + if (failedVariantsByApp.size === 0) { + core.info('No failed canary tests found.'); + return; + } + + const openIssues = await github.paginate(github.rest.issues.listForRepo, { + owner, + repo, + state: 'open', + per_page: 100, + }); + const openIssuesByTitle = new Map( + openIssues.filter(issue => !issue.pull_request).map(issue => [issue.title, issue.number]), + ); + + for (const [app, variants] of failedVariantsByApp) { + const title = issueTitle(app); + const existingIssueNumber = openIssuesByTitle.get(title); + + if (existingIssueNumber) { + await github.rest.issues.createComment({ + owner, + repo, + issue_number: existingIssueNumber, + body: renderComment(variants, runUrl), + }); + core.info(`${app}: commented on #${existingIssueNumber}`); + continue; + } + + const { data: issue } = await github.rest.issues.create({ + owner, + repo, + title, + labels: ISSUE_LABELS, + body: renderIssueBody(app, variants, runUrl), + }); + core.info(`${app}: opened #${issue.number}`); + } +} diff --git a/scripts/report-canary-failures.test.ts b/scripts/report-canary-failures.test.ts new file mode 100644 index 000000000000..b58aa04a0ca5 --- /dev/null +++ b/scripts/report-canary-failures.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, vi } from 'vitest'; +import run, { getFailedVariantsByApp, issueTitle } from './report-canary-failures.mjs'; + +const matrix = { + include: [ + { 'test-application': 'nextjs-16', label: 'nextjs-16 (canary, turbopack)' }, + { 'test-application': 'nextjs-16', label: 'nextjs-16 (canary, webpack)' }, + { 'test-application': 'nuxt-4', label: 'nuxt-4 (canary)' }, + { 'test-application': 'vue-3' }, + ], +}; + +const jobs = [ + { name: 'Prepare E2E Canary tests', conclusion: 'success', html_url: 'https://jobs/prepare' }, + { name: 'E2E nextjs-16 (canary, turbopack) Test', conclusion: 'failure', html_url: 'https://jobs/1' }, + { name: 'E2E nextjs-16 (canary, webpack) Test', conclusion: 'timed_out', html_url: 'https://jobs/2' }, + { name: 'E2E nuxt-4 (canary) Test', conclusion: 'success', html_url: 'https://jobs/3' }, + { name: 'E2E vue-3 Test', conclusion: 'failure', html_url: 'https://jobs/4' }, +]; + +function createGithub(openIssues: Array<{ number: number; title: string; pull_request?: object }>) { + const listJobsForWorkflowRun = vi.fn(); + const listForRepo = vi.fn(); + const createComment = vi.fn(); + const create = vi.fn().mockResolvedValue({ data: { number: 123 } }); + + return { + rest: { + actions: { listJobsForWorkflowRun }, + issues: { listForRepo, createComment, create }, + }, + paginate: vi.fn(async (method: unknown) => (method === listJobsForWorkflowRun ? jobs : openIssues)), + }; +} + +const context = { repo: { owner: 'getsentry', repo: 'sentry-javascript' }, serverUrl: 'https://github.com', runId: 42 }; +const core = { info: vi.fn() }; +const runUrl = 'https://github.com/getsentry/sentry-javascript/actions/runs/42'; + +describe('getFailedVariantsByApp', () => { + it('groups failed and timed out jobs by test application', () => { + expect(getFailedVariantsByApp(matrix, jobs)).toEqual( + new Map([ + [ + 'nextjs-16', + [ + { label: 'nextjs-16 (canary, turbopack)', url: 'https://jobs/1' }, + { label: 'nextjs-16 (canary, webpack)', url: 'https://jobs/2' }, + ], + ], + ['vue-3', [{ label: 'vue-3', url: 'https://jobs/4' }]], + ]), + ); + }); +}); + +describe('run', () => { + it('opens one issue per failed test application', async () => { + const github = createGithub([]); + + await run({ github, context, core, matrix }); + + expect(github.rest.issues.create).toHaveBeenCalledTimes(2); + expect(github.rest.issues.create).toHaveBeenCalledWith({ + owner: 'getsentry', + repo: 'sentry-javascript', + title: issueTitle('nextjs-16'), + labels: ['Type: Tests', 'Waiting for: Product Owner'], + body: expect.stringContaining(runUrl), + }); + expect(github.rest.issues.create).toHaveBeenCalledWith(expect.objectContaining({ title: issueTitle('vue-3') })); + expect(github.rest.issues.createComment).not.toHaveBeenCalled(); + }); + + it('comments on the open issue of a test application instead of opening another one', async () => { + const github = createGithub([ + { number: 7, title: issueTitle('nextjs-16') }, + { number: 8, title: issueTitle('vue-3'), pull_request: {} }, + ]); + + await run({ github, context, core, matrix }); + + expect(github.rest.issues.createComment).toHaveBeenCalledTimes(1); + expect(github.rest.issues.createComment).toHaveBeenCalledWith({ + owner: 'getsentry', + repo: 'sentry-javascript', + issue_number: 7, + body: expect.stringContaining(runUrl), + }); + expect(github.rest.issues.create).toHaveBeenCalledTimes(1); + expect(github.rest.issues.create).toHaveBeenCalledWith(expect.objectContaining({ title: issueTitle('vue-3') })); + }); + + it('does nothing when no canary test failed', async () => { + const github = createGithub([]); + + await run({ github, context, core, matrix: { include: [matrix.include[2]] } }); + + expect(github.paginate).toHaveBeenCalledTimes(1); + expect(github.rest.issues.create).not.toHaveBeenCalled(); + expect(github.rest.issues.createComment).not.toHaveBeenCalled(); + }); +});