diff --git a/.github/CANARY_FAILURE_TEMPLATE.md b/.github/CANARY_FAILURE_TEMPLATE.md deleted file mode 100644 index 9e05fcfc44ae..000000000000 --- a/.github/CANARY_FAILURE_TEMPLATE.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: '{{ env.TITLE }}' -labels: 'Type: Tests, Waiting for: Product Owner' ---- - -Canary tests failed: {{ env.RUN_LINK }} diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 62cc55309d3a..82c9302754dd 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -14,7 +14,6 @@ env: permissions: contents: read - issues: write jobs: job_e2e_prepare: @@ -122,13 +121,30 @@ jobs: timeout-minutes: 15 run: ${{ matrix.assert-command || 'pnpm test:assert' }} - - name: Create Issue - if: failure() && github.event_name == 'schedule' - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 + job_report_failures: + name: Report canary failures + needs: [job_e2e_prepare, job_e2e_tests] + if: always() && github.event_name == 'schedule' && needs.job_e2e_tests.result == 'failure' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + # To find the failed jobs of this run + actions: read + issues: write + steps: + - name: Check out current commit + uses: actions/checkout@v7 + with: + ref: ${{ env.HEAD_COMMIT }} + persist-credentials: false + - name: Open or update an issue per failed test application + uses: actions/github-script@v9 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - TITLE: ${{ matrix.label || matrix.test-application }} Test Failed + CANARY_MATRIX: ${{ needs.job_e2e_prepare.outputs.matrix }} with: - filename: .github/CANARY_FAILURE_TEMPLATE.md - update_existing: true + script: | + const { default: run } = await import( + `${process.env.GITHUB_WORKSPACE}/scripts/report-canary-failures.mjs` + ); + await run({ github, context, core, matrix: JSON.parse(process.env.CANARY_MATRIX) }); diff --git a/scripts/report-canary-failures.mjs b/scripts/report-canary-failures.mjs new file mode 100644 index 000000000000..7d1461b222f1 --- /dev/null +++ b/scripts/report-canary-failures.mjs @@ -0,0 +1,116 @@ +const ISSUE_LABELS = ['Type: Tests', 'Waiting for: Product Owner']; +const FAILED_CONCLUSIONS = ['failure', 'timed_out']; + +export function issueTitle(testApplication) { + return `Canary tests failed: ${testApplication}`; +} + +// Must match the `name` of `job_e2e_tests` in .github/workflows/canary.yml +function jobName(include) { + return `E2E ${include.label || include['test-application']} Test`; +} + +/** + * Groups the failed canary jobs of a run by test application. + * + * @param {{ include: Array> }} matrix + * @param {Array<{ name: string; conclusion: string | null; html_url: string | null }>} jobs + * @returns {Map>} + */ +export function getFailedVariantsByApp(matrix, jobs) { + const failedJobs = new Map( + jobs.filter(job => FAILED_CONCLUSIONS.includes(job.conclusion)).map(job => [job.name, job.html_url]), + ); + + const failedVariantsByApp = new Map(); + for (const include of matrix.include) { + const name = jobName(include); + if (!failedJobs.has(name)) { + continue; + } + + const app = include['test-application']; + const variants = failedVariantsByApp.get(app) || []; + variants.push({ label: include.label || app, url: failedJobs.get(name) }); + failedVariantsByApp.set(app, variants); + } + + return failedVariantsByApp; +} + +function renderVariants(variants) { + return variants.map(({ label, url }) => (url ? `- [${label}](${url})` : `- ${label}`)).join('\n'); +} + +function renderIssueBody(app, variants, runUrl) { + return `The canary tests for \`${app}\` failed: ${runUrl} + +Failed variants: +${renderVariants(variants)} + +Canary tests run the \`canaryVariants\` of the test application against upstream prereleases. Until this issue is +closed, new failures of \`${app}\` are posted as comments here.`; +} + +function renderComment(variants, runUrl) { + return `Still failing: ${runUrl} + +${renderVariants(variants)}`; +} + +/** + * Opens one issue per failed test application, or comments on the open issue for that application. + */ +export default async function run({ github, context, core, matrix }) { + const { owner, repo } = context.repo; + const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner, + repo, + run_id: context.runId, + filter: 'latest', + per_page: 100, + }); + + const failedVariantsByApp = getFailedVariantsByApp(matrix, jobs); + if (failedVariantsByApp.size === 0) { + core.info('No failed canary tests found.'); + return; + } + + const openIssues = await github.paginate(github.rest.issues.listForRepo, { + owner, + repo, + state: 'open', + per_page: 100, + }); + const openIssuesByTitle = new Map( + openIssues.filter(issue => !issue.pull_request).map(issue => [issue.title, issue.number]), + ); + + for (const [app, variants] of failedVariantsByApp) { + const title = issueTitle(app); + const existingIssueNumber = openIssuesByTitle.get(title); + + if (existingIssueNumber) { + await github.rest.issues.createComment({ + owner, + repo, + issue_number: existingIssueNumber, + body: renderComment(variants, runUrl), + }); + core.info(`${app}: commented on #${existingIssueNumber}`); + continue; + } + + const { data: issue } = await github.rest.issues.create({ + owner, + repo, + title, + labels: ISSUE_LABELS, + body: renderIssueBody(app, variants, runUrl), + }); + core.info(`${app}: opened #${issue.number}`); + } +} diff --git a/scripts/report-canary-failures.test.ts b/scripts/report-canary-failures.test.ts new file mode 100644 index 000000000000..b58aa04a0ca5 --- /dev/null +++ b/scripts/report-canary-failures.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, vi } from 'vitest'; +import run, { getFailedVariantsByApp, issueTitle } from './report-canary-failures.mjs'; + +const matrix = { + include: [ + { 'test-application': 'nextjs-16', label: 'nextjs-16 (canary, turbopack)' }, + { 'test-application': 'nextjs-16', label: 'nextjs-16 (canary, webpack)' }, + { 'test-application': 'nuxt-4', label: 'nuxt-4 (canary)' }, + { 'test-application': 'vue-3' }, + ], +}; + +const jobs = [ + { name: 'Prepare E2E Canary tests', conclusion: 'success', html_url: 'https://jobs/prepare' }, + { name: 'E2E nextjs-16 (canary, turbopack) Test', conclusion: 'failure', html_url: 'https://jobs/1' }, + { name: 'E2E nextjs-16 (canary, webpack) Test', conclusion: 'timed_out', html_url: 'https://jobs/2' }, + { name: 'E2E nuxt-4 (canary) Test', conclusion: 'success', html_url: 'https://jobs/3' }, + { name: 'E2E vue-3 Test', conclusion: 'failure', html_url: 'https://jobs/4' }, +]; + +function createGithub(openIssues: Array<{ number: number; title: string; pull_request?: object }>) { + const listJobsForWorkflowRun = vi.fn(); + const listForRepo = vi.fn(); + const createComment = vi.fn(); + const create = vi.fn().mockResolvedValue({ data: { number: 123 } }); + + return { + rest: { + actions: { listJobsForWorkflowRun }, + issues: { listForRepo, createComment, create }, + }, + paginate: vi.fn(async (method: unknown) => (method === listJobsForWorkflowRun ? jobs : openIssues)), + }; +} + +const context = { repo: { owner: 'getsentry', repo: 'sentry-javascript' }, serverUrl: 'https://github.com', runId: 42 }; +const core = { info: vi.fn() }; +const runUrl = 'https://github.com/getsentry/sentry-javascript/actions/runs/42'; + +describe('getFailedVariantsByApp', () => { + it('groups failed and timed out jobs by test application', () => { + expect(getFailedVariantsByApp(matrix, jobs)).toEqual( + new Map([ + [ + 'nextjs-16', + [ + { label: 'nextjs-16 (canary, turbopack)', url: 'https://jobs/1' }, + { label: 'nextjs-16 (canary, webpack)', url: 'https://jobs/2' }, + ], + ], + ['vue-3', [{ label: 'vue-3', url: 'https://jobs/4' }]], + ]), + ); + }); +}); + +describe('run', () => { + it('opens one issue per failed test application', async () => { + const github = createGithub([]); + + await run({ github, context, core, matrix }); + + expect(github.rest.issues.create).toHaveBeenCalledTimes(2); + expect(github.rest.issues.create).toHaveBeenCalledWith({ + owner: 'getsentry', + repo: 'sentry-javascript', + title: issueTitle('nextjs-16'), + labels: ['Type: Tests', 'Waiting for: Product Owner'], + body: expect.stringContaining(runUrl), + }); + expect(github.rest.issues.create).toHaveBeenCalledWith(expect.objectContaining({ title: issueTitle('vue-3') })); + expect(github.rest.issues.createComment).not.toHaveBeenCalled(); + }); + + it('comments on the open issue of a test application instead of opening another one', async () => { + const github = createGithub([ + { number: 7, title: issueTitle('nextjs-16') }, + { number: 8, title: issueTitle('vue-3'), pull_request: {} }, + ]); + + await run({ github, context, core, matrix }); + + expect(github.rest.issues.createComment).toHaveBeenCalledTimes(1); + expect(github.rest.issues.createComment).toHaveBeenCalledWith({ + owner: 'getsentry', + repo: 'sentry-javascript', + issue_number: 7, + body: expect.stringContaining(runUrl), + }); + expect(github.rest.issues.create).toHaveBeenCalledTimes(1); + expect(github.rest.issues.create).toHaveBeenCalledWith(expect.objectContaining({ title: issueTitle('vue-3') })); + }); + + it('does nothing when no canary test failed', async () => { + const github = createGithub([]); + + await run({ github, context, core, matrix: { include: [matrix.include[2]] } }); + + expect(github.paginate).toHaveBeenCalledTimes(1); + expect(github.rest.issues.create).not.toHaveBeenCalled(); + expect(github.rest.issues.createComment).not.toHaveBeenCalled(); + }); +});