diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dbbfd3..b589e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to `@testingbot/cli` are documented here. Releases are publi ### Added +- Job summary and annotations in CI: in GitHub Actions a results table is appended to `$GITHUB_STEP_SUMMARY` and every failed flow or run gets an `::error` annotation; in Azure Pipelines the table is attached to the build and failures are logged as errors. For `maestro`, `espresso`, `xcuitest` and `status`. `TESTINGBOT_CI_SUMMARY=0` turns it off. - `testingbot maestro --cancel-previous`: before starting, cancels your queued or running runs of the same repository and branch or pull request from an earlier commit (`cancelPrevious` on the run API). Runs on the same commit are left alone. - `testingbot maestro` detects the repository URL, branch, commit and pull request in GitHub Actions, GitLab CI, Bitbucket Pipelines, Azure Pipelines and CircleCI and sends them as run metadata (`repoUrl` is new), so the dashboard can link to the commit and PR on any Git host. On a GitHub pull request the PR's head commit is used, not the merge commit. Explicit flags win. - `testingbot espresso` and `testingbot xcuitest` take the same CI/CD metadata as `maestro`: `--branch`, `--pr-url`, `-m/--metadata` and the CI environment detection, through one shared code path. diff --git a/README.md b/README.md index 9733de3..06b4fd8 100644 --- a/README.md +++ b/README.md @@ -698,6 +698,10 @@ testingbot xcuitest app.ipa app-test.zip \ `testingbot maestro --cancel-previous` uses this context to cancel the runs of a previous push: your queued or running Maestro runs with the same repository and the same pull request (or branch) but a different commit. Runs on the same commit, such as the iOS and Android jobs of one pipeline, keep running. +### CI Job Summary and Annotations + +In GitHub Actions the results are added to the job summary (`$GITHUB_STEP_SUMMARY`) as a table per device with links to TestingBot, and each failed flow or run gets an `::error` annotation, shown on the workflow run and the pull request. In Azure Pipelines the same table is attached to the build (Extensions tab) and failures are logged as errors. Annotations are left out with `--json`, since stdout then holds the JSON document. Set `TESTINGBOT_CI_SUMMARY=0` to turn this off. + ### Real-time Progress By default, the CLI shows real-time progress updates including: diff --git a/src/cli.ts b/src/cli.ts index 0c57523..dce1498 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -22,6 +22,7 @@ import MaestroOptions, { } from './models/maestro_options'; import Maestro from './providers/maestro'; import { ciRepoMetadata } from './utils/ci_env'; +import { publishCiSummary } from './utils/ci_summary'; import Login from './providers/login'; import { CREDENTIALS_FILE, @@ -357,11 +358,15 @@ function applyMaestroCloudAliases(args: MaestroCloudAliasArgs): string[] { async function finishCommand( output: JsonOutput, jsonOptions: JsonOutputOptions, + { ciSummary = true }: { ciSummary?: boolean } = {}, ): Promise { const written = await writeJsonOutput(output, jsonOptions); if (written && !jsonOptions.json) { logger.info(`JSON results written to ${written}`); } + if (ciSummary) { + await publishCiSummary(output, { stdoutReserved: jsonOptions.json }); + } process.exitCode = resolveExitCode(output, jsonOptions); } @@ -1462,7 +1467,10 @@ withFlags( }), ); const result = await maestro.artifacts(args.id); - await finishCommand(maestro.toJsonOutput(result), jsonOptions); + // The run's own command already published its summary. + await finishCommand(maestro.toJsonOutput(result), jsonOptions, { + ciSummary: false, + }); } catch (err) { await failCommand('maestro', 'Artifacts', err, jsonOptions); } diff --git a/src/utils/ci_summary.ts b/src/utils/ci_summary.ts new file mode 100644 index 0000000..9ae8606 --- /dev/null +++ b/src/utils/ci_summary.ts @@ -0,0 +1,224 @@ +/** + * The results in the CI system's own UI, not only in the log: + * + * - GitHub Actions: a markdown table appended to $GITHUB_STEP_SUMMARY (the job + * summary page), plus an `::error` annotation per failed run or flow, which + * GitHub shows on the run and the pull request. + * - Azure Pipelines: the same markdown attached to the build's Extensions tab + * (`##vso[task.uploadsummary]`) and a `##vso[task.logissue]` per failure. + * + * Annotations are workflow commands printed on stdout, so they are left out + * when --json owns stdout. Set TESTINGBOT_CI_SUMMARY=0 to turn all of it off. + * Nothing here may fail the command: errors are logged at debug level. + */ + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import logger from '../logger'; +import type { JsonFlowResult, JsonOutput, JsonRunResult } from './json_output'; + +type Env = NodeJS.ProcessEnv; + +const PROVIDER_LABEL: Record = { + maestro: 'Maestro', + espresso: 'Espresso', + xcuitest: 'XCUITest', +}; + +const MAX_FAILURE_ROWS = 50; +const MAX_ANNOTATIONS = 10; // GitHub shows at most 10 error annotations per step + +interface Failure { + title: string; + message: string; + url?: string; +} + +/** Flows whose verdict counts: the last attempt of each. */ +function countedFlows(run: JsonRunResult): JsonFlowResult[] { + return (run.flows ?? []).filter((flow) => flow.latest); +} + +function deviceLabel(run: JsonRunResult): string { + const { name, version, platform } = run.device; + return version ? `${name} (${platform} ${version})` : name; +} + +function failuresOf(output: JsonOutput): Failure[] { + const failures: Failure[] = []; + for (const run of output.runs) { + const flows = countedFlows(run); + const failedFlows = flows.filter((flow) => !flow.passed); + if (failedFlows.length > 0) { + for (const flow of failedFlows) { + failures.push({ + title: `${flow.name} on ${deviceLabel(run)}`, + message: flow.errors[0] ?? `Flow ${flow.status.toLowerCase()}`, + url: run.url, + }); + } + } else if (!run.passed) { + failures.push({ + title: `Run ${run.id} on ${deviceLabel(run)}`, + message: `Run ${run.status.toLowerCase()}`, + url: run.url, + }); + } + } + return failures; +} + +/** One line, no table-breaking pipes, bounded length. */ +function cell(text: string, max = 200): string { + const flat = text.replace(/\s+/g, ' ').replace(/\|/g, '\\|').trim(); + return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat; +} + +function link(text: string, url?: string): string { + return url ? `[${text}](${url})` : text; +} + +export function renderMarkdown(output: JsonOutput): string { + const label = PROVIDER_LABEL[output.provider]; + const failedRuns = output.runs.filter((run) => !run.passed).length; + const verdict = output.success + ? '✅ passed' + : `❌ ${failedRuns} of ${output.runs.length} run(s) failed`; + + const lines = [`### TestingBot ${label}: ${verdict}`, '']; + if (output.url) lines.push(`[Open in TestingBot](${output.url})`, ''); + + const hasFlows = output.runs.some((run) => run.flows?.length); + lines.push( + hasFlows + ? '| | Device | Flows passed | Run |' + : '| | Device | Result | Run |', + '|---|---|---|---|', + ); + for (const run of output.runs) { + const flows = countedFlows(run); + const result = hasFlows + ? `${flows.filter((flow) => flow.passed).length} / ${flows.length}` + : run.passed + ? 'passed' + : run.status.toLowerCase(); + lines.push( + `| ${run.passed ? '✅' : '❌'} | ${cell(deviceLabel(run))} | ${result} | ${link(`#${run.id}`, run.url)} |`, + ); + } + + const failures = failuresOf(output); + if (failures.length > 0) { + lines.push( + '', + `Failures (${failures.length})`, + '', + '| Test | Error |', + '|---|---|', + ); + for (const failure of failures.slice(0, MAX_FAILURE_ROWS)) { + lines.push( + `| ${link(cell(failure.title, 120), failure.url)} | ${cell(failure.message)} |`, + ); + } + if (failures.length > MAX_FAILURE_ROWS) { + lines.push( + '', + `…and ${failures.length - MAX_FAILURE_ROWS} more, see TestingBot.`, + ); + } + lines.push('', ''); + } + return `${lines.join('\n')}\n`; +} + +/** GitHub workflow-command escaping for the message and for properties. */ +function ghData(text: string): string { + return text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); +} +function ghProperty(text: string): string { + return ghData(text).replace(/:/g, '%3A').replace(/,/g, '%2C'); +} + +/** Azure logging commands are one line; `]` and `;` end the properties. */ +function azureData(text: string): string { + return text.replace(/\r?\n/g, ' '); +} + +function annotationMessage(failure: Failure): string { + return failure.url ? `${failure.message}\n${failure.url}` : failure.message; +} + +export function githubAnnotations(output: JsonOutput): string[] { + const failures = failuresOf(output); + const lines = failures + .slice(0, MAX_ANNOTATIONS) + .map( + (failure) => + `::error title=${ghProperty(`TestingBot: ${failure.title}`)}::${ghData(annotationMessage(failure))}`, + ); + if (failures.length > MAX_ANNOTATIONS) { + lines.push( + `::error title=TestingBot::${ghData(`${failures.length - MAX_ANNOTATIONS} more failures, see the job summary.`)}`, + ); + } + return lines; +} + +export function azureAnnotations(output: JsonOutput): string[] { + return failuresOf(output) + .slice(0, MAX_ANNOTATIONS) + .map( + (failure) => + `##vso[task.logissue type=error]${azureData(`TestingBot: ${failure.title}: ${annotationMessage(failure)}`)}`, + ); +} + +export interface CiSummaryOptions { + /** --json: stdout is the JSON document, so no workflow commands. */ + stdoutReserved?: boolean; + env?: Env; + write?: (line: string) => void; +} + +/** Publishes the job summary and annotations for a finished test run. */ +export async function publishCiSummary( + output: JsonOutput, + options: CiSummaryOptions = {}, +): Promise { + const env = options.env ?? process.env; + if (env.TESTINGBOT_CI_SUMMARY === '0') return; + if (output.outcome !== 'passed' && output.outcome !== 'failed') return; + if (output.runs.length === 0) return; + const write = + options.write ?? ((line: string) => process.stdout.write(`${line}\n`)); + const commands = !options.stdoutReserved; + + try { + if (env.GITHUB_ACTIONS === 'true') { + if (env.GITHUB_STEP_SUMMARY) { + await fs.promises.appendFile( + env.GITHUB_STEP_SUMMARY, + renderMarkdown(output), + ); + } + if (commands) githubAnnotations(output).forEach(write); + } else if (env.TF_BUILD?.toLowerCase() === 'true') { + if (commands) { + const dir = env.AGENT_TEMPDIRECTORY || os.tmpdir(); + const file = path.join( + dir, + `testingbot-${output.provider}-${output.appId ?? Date.now()}.md`, + ); + await fs.promises.writeFile(file, renderMarkdown(output)); + write(`##vso[task.uploadsummary]${file}`); + azureAnnotations(output).forEach(write); + } + } + } catch (error) { + logger.debug( + `Could not publish the CI summary: ${error instanceof Error ? error.message : error}`, + ); + } +} diff --git a/tests/cli.test.ts b/tests/cli.test.ts index d79e37f..da5badf 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -19,6 +19,9 @@ jest.mock('./../src/providers/maestro'); jest.mock('./../src/utils/ci_env', () => ({ ciRepoMetadata: jest.fn(() => ({})), })); +jest.mock('./../src/utils/ci_summary', () => ({ + publishCiSummary: jest.fn(), +})); jest.mock('./../src/providers/account', () => ({ ...jest.requireActual('./../src/providers/account'), fetchAccount: jest.fn(), diff --git a/tests/utils/ci_summary.test.ts b/tests/utils/ci_summary.test.ts new file mode 100644 index 0000000..e54c464 --- /dev/null +++ b/tests/utils/ci_summary.test.ts @@ -0,0 +1,194 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + azureAnnotations, + githubAnnotations, + publishCiSummary, + renderMarkdown, +} from '../../src/utils/ci_summary'; +import type { JsonOutput } from '../../src/utils/json_output'; + +jest.mock('../../src/logger'); + +const flow = ( + id: number, + name: string, + passed: boolean, + extra: { attempt?: number; latest?: boolean; errors?: string[] } = {}, +) => ({ + id, + runId: 10, + name, + status: passed ? 'DONE' : 'FAILED', + passed, + attempt: extra.attempt ?? 1, + latest: extra.latest ?? true, + errors: extra.errors ?? [], +}); + +const maestro: JsonOutput = { + provider: 'maestro', + outcome: 'failed', + success: false, + appId: 77, + url: 'https://testingbot.com/members/maestro/77', + runs: [ + { + id: 10, + status: 'DONE', + passed: false, + device: { name: 'Pixel 8', platform: 'Android', version: '14' }, + url: 'https://testingbot.com/members/maestro/77/runs/10', + flows: [ + flow(1, 'login', true), + flow(2, 'checkout', false, { + errors: ['Assertion is false: "Pay | now" is visible\nat step 4'], + }), + // A failed first attempt that a retry fixed does not count. + flow(3, 'search', false, { latest: false }), + flow(4, 'search', true, { attempt: 2 }), + ], + }, + ], +}; + +const espresso: JsonOutput = { + provider: 'espresso', + outcome: 'passed', + success: true, + runs: [ + { + id: 5, + status: 'DONE', + passed: true, + device: { name: 'Pixel 6', platform: 'Android' }, + }, + ], +}; + +describe('renderMarkdown', () => { + test('a Maestro run: flows passed per device and the failing flows', () => { + const md = renderMarkdown(maestro); + expect(md).toContain('### TestingBot Maestro: ❌ 1 of 1 run(s) failed'); + expect(md).toContain( + '[Open in TestingBot](https://testingbot.com/members/maestro/77)', + ); + expect(md).toContain( + '| ❌ | Pixel 8 (Android 14) | 2 / 3 | [#10](https://testingbot.com/members/maestro/77/runs/10) |', + ); + expect(md).toContain('Failures (1)'); + expect(md).toContain( + '| [checkout on Pixel 8 (Android 14)](https://testingbot.com/members/maestro/77/runs/10) | Assertion is false: "Pay \\| now" is visible at step 4 |', + ); + expect(md).not.toContain('search on'); + }); + + test('a passed Espresso run without flows', () => { + const md = renderMarkdown(espresso); + expect(md).toContain('### TestingBot Espresso: ✅ passed'); + expect(md).toContain('| ✅ | Pixel 6 | passed | #5 |'); + expect(md).not.toContain('Failures'); + }); +}); + +describe('annotations', () => { + test('GitHub: one escaped ::error per failed flow', () => { + expect(githubAnnotations(maestro)).toEqual([ + '::error title=TestingBot%3A checkout on Pixel 8 (Android 14)::Assertion is false: "Pay | now" is visible%0Aat step 4%0Ahttps://testingbot.com/members/maestro/77/runs/10', + ]); + }); + + test('Azure: one single-line logissue per failed flow', () => { + expect(azureAnnotations(maestro)).toEqual([ + '##vso[task.logissue type=error]TestingBot: checkout on Pixel 8 (Android 14): Assertion is false: "Pay | now" is visible at step 4 https://testingbot.com/members/maestro/77/runs/10', + ]); + }); + + test('a failed run without flows is annotated as a run', () => { + const failed: JsonOutput = { + ...espresso, + outcome: 'failed', + success: false, + runs: [{ ...espresso.runs[0], passed: false, status: 'FAILED' }], + }; + expect(githubAnnotations(failed)).toEqual([ + '::error title=TestingBot%3A Run 5 on Pixel 6::Run failed', + ]); + }); +}); + +describe('publishCiSummary', () => { + let dir: string; + let lines: string[]; + const write = (line: string) => lines.push(line); + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-summary-')); + lines = []; + }); + afterEach(() => fs.rmSync(dir, { recursive: true, force: true })); + + test('GitHub Actions: appends to the step summary and annotates', async () => { + const summary = path.join(dir, 'summary.md'); + fs.writeFileSync(summary, 'earlier step\n'); + await publishCiSummary(maestro, { + env: { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: summary }, + write, + }); + const md = fs.readFileSync(summary, 'utf8'); + expect(md.startsWith('earlier step\n### TestingBot Maestro')).toBe(true); + expect(lines).toHaveLength(1); + expect(lines[0]).toMatch(/^::error /); + }); + + test('--json keeps stdout clean but still writes the summary file', async () => { + const summary = path.join(dir, 'summary.md'); + await publishCiSummary(maestro, { + env: { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: summary }, + stdoutReserved: true, + write, + }); + expect(fs.existsSync(summary)).toBe(true); + expect(lines).toEqual([]); + }); + + test('Azure Pipelines: uploads the markdown and logs issues', async () => { + await publishCiSummary(maestro, { + env: { TF_BUILD: 'True', AGENT_TEMPDIRECTORY: dir }, + write, + }); + const file = path.join(dir, 'testingbot-maestro-77.md'); + expect(fs.readFileSync(file, 'utf8')).toContain('TestingBot Maestro'); + expect(lines[0]).toBe(`##vso[task.uploadsummary]${file}`); + expect(lines[1]).toMatch(/^##vso\[task\.logissue type=error\]/); + }); + + test('does nothing outside CI, when turned off, or without a verdict', async () => { + const summary = path.join(dir, 'summary.md'); + const gh = { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: summary }; + await publishCiSummary(maestro, { env: {}, write }); + await publishCiSummary(maestro, { + env: { ...gh, TESTINGBOT_CI_SUMMARY: '0' }, + write, + }); + await publishCiSummary( + { ...maestro, outcome: 'started' }, + { env: gh, write }, + ); + expect(fs.existsSync(summary)).toBe(false); + expect(lines).toEqual([]); + }); + + test('a summary that cannot be written does not throw', async () => { + await expect( + publishCiSummary(maestro, { + env: { + GITHUB_ACTIONS: 'true', + GITHUB_STEP_SUMMARY: path.join(dir, 'missing', 'summary.md'), + }, + write, + }), + ).resolves.toBeUndefined(); + }); +});