Job summary and annotations in GitHub Actions and Azure Pipelines

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to `@testingbot/cli` are documented here. Releases are publi
 
 ### Added
 
+- Job summary and annotations in CI: in GitHub Actions a results table is appended to `$GITHUB_STEP_SUMMARY` and every failed flow or run gets an `::error` annotation; in Azure Pipelines the table is attached to the build and failures are logged as errors. For `maestro`, `espresso`, `xcuitest` and `status`. `TESTINGBOT_CI_SUMMARY=0` turns it off.
 - `testingbot maestro --cancel-previous`: before starting, cancels your queued or running runs of the same repository and branch or pull request from an earlier commit (`cancelPrevious` on the run API). Runs on the same commit are left alone.
 - `testingbot maestro` detects the repository URL, branch, commit and pull request in GitHub Actions, GitLab CI, Bitbucket Pipelines, Azure Pipelines and CircleCI and sends them as run metadata (`repoUrl` is new), so the dashboard can link to the commit and PR on any Git host. On a GitHub pull request the PR's head commit is used, not the merge commit. Explicit flags win.
 - `testingbot espresso` and `testingbot xcuitest` take the same CI/CD metadata as `maestro`: `--branch`, `--pr-url`, `-m/--metadata` and the CI environment detection, through one shared code path.

README.md

@@ -698,6 +698,10 @@ testingbot xcuitest app.ipa app-test.zip \
 
 `testingbot maestro --cancel-previous` uses this context to cancel the runs of a previous push: your queued or running Maestro runs with the same repository and the same pull request (or branch) but a different commit. Runs on the same commit, such as the iOS and Android jobs of one pipeline, keep running.
 
+### CI Job Summary and Annotations
+
+In GitHub Actions the results are added to the job summary (`$GITHUB_STEP_SUMMARY`) as a table per device with links to TestingBot, and each failed flow or run gets an `::error` annotation, shown on the workflow run and the pull request. In Azure Pipelines the same table is attached to the build (Extensions tab) and failures are logged as errors. Annotations are left out with `--json`, since stdout then holds the JSON document. Set `TESTINGBOT_CI_SUMMARY=0` to turn this off.
+
 ### Real-time Progress
 
 By default, the CLI shows real-time progress updates including:

src/cli.ts

@@ -22,6 +22,7 @@ import MaestroOptions, {
 } from './models/maestro_options';
 import Maestro from './providers/maestro';
 import { ciRepoMetadata } from './utils/ci_env';
+import { publishCiSummary } from './utils/ci_summary';
 import Login from './providers/login';
 import {
   CREDENTIALS_FILE,
@@ -357,11 +358,15 @@ function applyMaestroCloudAliases(args: MaestroCloudAliasArgs): string[] {
 async function finishCommand(
   output: JsonOutput,
   jsonOptions: JsonOutputOptions,
+  { ciSummary = true }: { ciSummary?: boolean } = {},
 ): Promise<void> {
   const written = await writeJsonOutput(output, jsonOptions);
   if (written && !jsonOptions.json) {
     logger.info(`JSON results written to ${written}`);
   }
+  if (ciSummary) {
+    await publishCiSummary(output, { stdoutReserved: jsonOptions.json });
+  }
   process.exitCode = resolveExitCode(output, jsonOptions);
 }
 
@@ -1462,7 +1467,10 @@ withFlags(
         }),
       );
       const result = await maestro.artifacts(args.id);
-      await finishCommand(maestro.toJsonOutput(result), jsonOptions);
+      // The run's own command already published its summary.
+      await finishCommand(maestro.toJsonOutput(result), jsonOptions, {
+        ciSummary: false,
+      });
     } catch (err) {
       await failCommand('maestro', 'Artifacts', err, jsonOptions);
     }

src/utils/ci_summary.ts

@@ -0,0 +1,224 @@
+/**
+ * The results in the CI system's own UI, not only in the log:
+ *
+ * - GitHub Actions: a markdown table appended to $GITHUB_STEP_SUMMARY (the job
+ *   summary page), plus an `::error` annotation per failed run or flow, which
+ *   GitHub shows on the run and the pull request.
+ * - Azure Pipelines: the same markdown attached to the build's Extensions tab
+ *   (`##vso[task.uploadsummary]`) and a `##vso[task.logissue]` per failure.
+ *
+ * Annotations are workflow commands printed on stdout, so they are left out
+ * when --json owns stdout. Set TESTINGBOT_CI_SUMMARY=0 to turn all of it off.
+ * Nothing here may fail the command: errors are logged at debug level.
+ */
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import logger from '../logger';
+import type { JsonFlowResult, JsonOutput, JsonRunResult } from './json_output';
+
+type Env = NodeJS.ProcessEnv;
+
+const PROVIDER_LABEL: Record<JsonOutput['provider'], string> = {
+  maestro: 'Maestro',
+  espresso: 'Espresso',
+  xcuitest: 'XCUITest',
+};
+
+const MAX_FAILURE_ROWS = 50;
+const MAX_ANNOTATIONS = 10; // GitHub shows at most 10 error annotations per step
+
+interface Failure {
+  title: string;
+  message: string;
+  url?: string;
+}
+
+/** Flows whose verdict counts: the last attempt of each. */
+function countedFlows(run: JsonRunResult): JsonFlowResult[] {
+  return (run.flows ?? []).filter((flow) => flow.latest);
+}
+
+function deviceLabel(run: JsonRunResult): string {
+  const { name, version, platform } = run.device;
+  return version ? `${name} (${platform} ${version})` : name;
+}
+
+function failuresOf(output: JsonOutput): Failure[] {
+  const failures: Failure[] = [];
+  for (const run of output.runs) {
+    const flows = countedFlows(run);
+    const failedFlows = flows.filter((flow) => !flow.passed);
+    if (failedFlows.length > 0) {
+      for (const flow of failedFlows) {
+        failures.push({
+          title: `${flow.name} on ${deviceLabel(run)}`,
+          message: flow.errors[0] ?? `Flow ${flow.status.toLowerCase()}`,
+          url: run.url,
+        });
+      }
+    } else if (!run.passed) {
+      failures.push({
+        title: `Run ${run.id} on ${deviceLabel(run)}`,
+        message: `Run ${run.status.toLowerCase()}`,
+        url: run.url,
+      });
+    }
+  }
+  return failures;
+}
+
+/** One line, no table-breaking pipes, bounded length. */
+function cell(text: string, max = 200): string {
+  const flat = text.replace(/\s+/g, ' ').replace(/\|/g, '\\|').trim();
+  return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
+}
+
+function link(text: string, url?: string): string {
+  return url ? `[${text}](${url})` : text;
+}
+
+export function renderMarkdown(output: JsonOutput): string {
+  const label = PROVIDER_LABEL[output.provider];
+  const failedRuns = output.runs.filter((run) => !run.passed).length;
+  const verdict = output.success
+    ? '✅ passed'
+    : `❌ ${failedRuns} of ${output.runs.length} run(s) failed`;
+
+  const lines = [`### TestingBot ${label}: ${verdict}`, ''];
+  if (output.url) lines.push(`[Open in TestingBot](${output.url})`, '');
+
+  const hasFlows = output.runs.some((run) => run.flows?.length);
+  lines.push(
+    hasFlows
+      ? '| | Device | Flows passed | Run |'
+      : '| | Device | Result | Run |',
+    '|---|---|---|---|',
+  );
+  for (const run of output.runs) {
+    const flows = countedFlows(run);
+    const result = hasFlows
+      ? `${flows.filter((flow) => flow.passed).length} / ${flows.length}`
+      : run.passed
+        ? 'passed'
+        : run.status.toLowerCase();
+    lines.push(
+      `| ${run.passed ? '✅' : '❌'} | ${cell(deviceLabel(run))} | ${result} | ${link(`#${run.id}`, run.url)} |`,
+    );
+  }
+
+  const failures = failuresOf(output);
+  if (failures.length > 0) {
+    lines.push(
+      '',
+      `<details${failures.length <= 10 ? ' open' : ''}><summary>Failures (${failures.length})</summary>`,
+      '',
+      '| Test | Error |',
+      '|---|---|',
+    );
+    for (const failure of failures.slice(0, MAX_FAILURE_ROWS)) {
+      lines.push(
+        `| ${link(cell(failure.title, 120), failure.url)} | ${cell(failure.message)} |`,
+      );
+    }
+    if (failures.length > MAX_FAILURE_ROWS) {
+      lines.push(
+        '',
+        `…and ${failures.length - MAX_FAILURE_ROWS} more, see TestingBot.`,
+      );
+    }
+    lines.push('', '</details>');
+  }
+  return `${lines.join('\n')}\n`;
+}
+
+/** GitHub workflow-command escaping for the message and for properties. */
+function ghData(text: string): string {
+  return text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
+}
+function ghProperty(text: string): string {
+  return ghData(text).replace(/:/g, '%3A').replace(/,/g, '%2C');
+}
+
+/** Azure logging commands are one line; `]` and `;` end the properties. */
+function azureData(text: string): string {
+  return text.replace(/\r?\n/g, ' ');
+}
+
+function annotationMessage(failure: Failure): string {
+  return failure.url ? `${failure.message}\n${failure.url}` : failure.message;
+}
+
+export function githubAnnotations(output: JsonOutput): string[] {
+  const failures = failuresOf(output);
+  const lines = failures
+    .slice(0, MAX_ANNOTATIONS)
+    .map(
+      (failure) =>
+        `::error title=${ghProperty(`TestingBot: ${failure.title}`)}::${ghData(annotationMessage(failure))}`,
+    );
+  if (failures.length > MAX_ANNOTATIONS) {
+    lines.push(
+      `::error title=TestingBot::${ghData(`${failures.length - MAX_ANNOTATIONS} more failures, see the job summary.`)}`,
+    );
+  }
+  return lines;
+}
+
+export function azureAnnotations(output: JsonOutput): string[] {
+  return failuresOf(output)
+    .slice(0, MAX_ANNOTATIONS)
+    .map(
+      (failure) =>
+        `##vso[task.logissue type=error]${azureData(`TestingBot: ${failure.title}: ${annotationMessage(failure)}`)}`,
+    );
+}
+
+export interface CiSummaryOptions {
+  /** --json: stdout is the JSON document, so no workflow commands. */
+  stdoutReserved?: boolean;
+  env?: Env;
+  write?: (line: string) => void;
+}
+
+/** Publishes the job summary and annotations for a finished test run. */
+export async function publishCiSummary(
+  output: JsonOutput,
+  options: CiSummaryOptions = {},
+): Promise<void> {
+  const env = options.env ?? process.env;
+  if (env.TESTINGBOT_CI_SUMMARY === '0') return;
+  if (output.outcome !== 'passed' && output.outcome !== 'failed') return;
+  if (output.runs.length === 0) return;
+  const write =
+    options.write ?? ((line: string) => process.stdout.write(`${line}\n`));
+  const commands = !options.stdoutReserved;
+
+  try {
+    if (env.GITHUB_ACTIONS === 'true') {
+      if (env.GITHUB_STEP_SUMMARY) {
+        await fs.promises.appendFile(
+          env.GITHUB_STEP_SUMMARY,
+          renderMarkdown(output),
+        );
+      }
+      if (commands) githubAnnotations(output).forEach(write);
+    } else if (env.TF_BUILD?.toLowerCase() === 'true') {
+      if (commands) {
+        const dir = env.AGENT_TEMPDIRECTORY || os.tmpdir();
+        const file = path.join(
+          dir,
+          `testingbot-${output.provider}-${output.appId ?? Date.now()}.md`,
+        );
+        await fs.promises.writeFile(file, renderMarkdown(output));
+        write(`##vso[task.uploadsummary]${file}`);
+        azureAnnotations(output).forEach(write);
+      }
+    }
+  } catch (error) {
+    logger.debug(
+      `Could not publish the CI summary: ${error instanceof Error ? error.message : error}`,
+    );
+  }
+}

tests/cli.test.ts

@@ -19,6 +19,9 @@ jest.mock('./../src/providers/maestro');
 jest.mock('./../src/utils/ci_env', () => ({
   ciRepoMetadata: jest.fn(() => ({})),
 }));
+jest.mock('./../src/utils/ci_summary', () => ({
+  publishCiSummary: jest.fn(),
+}));
 jest.mock('./../src/providers/account', () => ({
   ...jest.requireActual('./../src/providers/account'),
   fetchAccount: jest.fn(),