Add deterministic AI reporter qualification gates

common/changes/@rushstack/rush-reporter/r8-machine-source-aliases_2026-09-11.json

@@ -2,7 +2,7 @@
   "changes": [
     {
       "packageName": "@rushstack/rush-reporter",
-      "comment": "Redact classified secret source aliases in machine JSON without losing owner-only file context, and qualify every machine output for producer privacy.",
+      "comment": "Redact classified secret source aliases in machine JSON without losing owner-only file context, qualify every machine output for producer privacy, and document/test late-artifact-safe AI flush reservation.",
       "type": "patch"
     }
   ],

common/changes/@rushstack/rush-reporter/r8-qualification-setup-worker_2026-09-22.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rush-reporter",
+      "comment": "Run AI qualification setup in an owned worker with bounded work and cleanup deadlines to avoid Windows CI setup timeouts leaving the corpus running.",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@rushstack/rush-reporter"
+}

common/changes/@rushstack/rush-reporter/r8-qualification-test-isolation_2026-09-10.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rush-reporter",
+      "comment": "Isolate AI qualification test mutations and join bounded worker cleanup after delayed, rejected, or timed-out corpus work.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@rushstack/rush-reporter"
+}

common/reviews/api/rush-reporter.api.md

@@ -18,7 +18,6 @@ export class AiReporter implements IReporter {
     constructor(options: IAiReporterOptions);
     // (undocumented)
     closeAsync(): Promise<void>;
-    // (undocumented)
     flushAsync(): Promise<void>;
     // (undocumented)
     initializeAsync(): Promise<void>;

libraries/reporter/README.md

@@ -51,18 +51,27 @@ these gates only produces a reusable qualification decision; it does not enable
 reporter selection. That decision also requires the separate telemetry privacy prerequisite to be accepted.
 The pre-major Rush frontend remains explicit/repository-opt-in, and `RUSH_REPORTER=legacy` remains
 authoritative.
-The Jest setup hook has a bounded 15-second allowance for the three file-backed corpus passes, matching
-the integration test setup policy. This allowance does not change any quality gate or production deadline.
+Jest setup and mutation runs use owned workers with a 30-second work budget and a separate 2-second
+termination/cleanup budget. The setup hook allows 33 seconds; tests that await two corpus sessions allow
+66 seconds. Timed-out work is terminated and its temporary directory removed before the session settles.
+These test-only allowances do not change any quality gate or production deadline.
 
 AI output reserves final-record space, including its supplied log reference, before emitting progress.
 Progress is buffered within the invocation byte limit until the primary log reservation is known, or until
 close if no log is supplied. Excess progress/details set `truncated`; the final result remains valid JSON.
-An unrendered start acknowledgement is coalesced into a known final result. Ongoing commands still expose
-buffered status at the next non-terminal event or explicit flush; watch history and every final field,
+An unrendered start acknowledgement is coalesced into a known final result. Once the log reservation is
+known, ongoing commands expose buffered status at the next non-terminal event or explicit flush; watch history and every final field,
 including the supplied log reference, are retained. No path shortening or measurement normalization is used.
 The final scope carries the command name, and standard `diagnostic.<code>.summary` keys are implicit rather
 than repeated alongside the same code. Custom summary keys are preserved.
 
+Before a log artifact arrives, AI flush and signal-flush retain bounded progress rather than spend bytes
+that a late, individually representable log reference may require. Reserving half the invocation budget is
+not sufficient for every supplied reference. Closing drains this bounded progress and emits the final
+result even if no artifact arrives; it does not invent a log path or claim full-log completeness.
+This is a limitation of eager no-artifact flushing, not permission to drop a required late artifact or
+exceed the invocation budget.
+
 AI fallback message text is emitted only for public envelopes. Its context retains the known command, and
 the usage-review action invokes that command's help (or `rush --help` when the command is unavailable).
 Qualification checks exact expected context, remediation commands/URLs, descriptions, and execution safety;

libraries/reporter/src/reporters/AiReporter.ts

@@ -378,6 +378,14 @@ export class AiReporter implements IReporter {
     }
   }
 
+  /**
+   * Flushes progress after the primary log reservation is known.
+   *
+   * @remarks
+   * Before an artifact arrives, even a bounded flush can consume space needed by a later
+   * individually representable log reference. Keep that progress bounded and deferred;
+   * closeAsync drains it with the final result even when no artifact ever arrives.
+   */
   public async flushAsync(): Promise<void> {
     if (this._logPath !== undefined) {
       this._flushPendingProgress();

libraries/reporter/src/test/AiReporterFlush.test.ts

@@ -0,0 +1,81 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import {
+  AiReporter,
+  ReporterManager,
+  type IAiFinalRecord,
+  type IReporterEventEnvelope
+} from '../index';
+
+describe('AI flush and required final-artifact reservation', () => {
+  it.each([
+    { maxBytes: 2048, logPath: `/protected/${'logs/'.repeat(250)}full.log` },
+    { maxBytes: 65536, logPath: `\\\\?\\C:\\${(`${'a'.repeat(250)}\\`).repeat(130)}full.log` }
+  ])('retains a representable late reference after signal flush with $maxBytes bytes', async ({ maxBytes, logPath }) => {
+    let output: string = '';
+    const manager = new ReporterManager();
+    manager.addReporter(new AiReporter({ maxBytes, write: (text) => { output += text; } }));
+    await manager.initializeAsync();
+    const emit = (type: IReporterEventEnvelope<unknown>['type'], payload: unknown): void => {
+      manager.emit({
+        protocolVersion: { major: 1, minor: 1 },
+        sessionId: 'flush-reservation',
+        source: { packageName: '@microsoft/rush-lib', packageVersion: '5.179.0' },
+        privacy: 'public',
+        type,
+        payload
+      });
+    };
+    emit('commandStarted', { commandName: 'build' });
+    for (let iterationId: number = 0; iterationId < 1000; iterationId++) {
+      emit('watchCycleCompleted', { iterationId, succeeded: true });
+    }
+    await manager.signalFlushAsync();
+    expect(output).toBe('');
+    emit('artifactAvailable', { role: 'log', path: logPath, complete: true });
+    emit('commandResult', { succeeded: false, exitCode: 1 });
+    await manager.closeAsync();
+
+    const final: IAiFinalRecord = JSON.parse(output.trimEnd().split('\n').at(-1)!);
+    expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(maxBytes);
+    expect(final.log).toEqual({ path: logPath, complete: true });
+    expect(final).toMatchObject({ kind: 'ai.final', result: 'failed', exitCode: 1, truncated: true });
+    expect(final.diagnostics.length).toBeLessThanOrEqual(20);
+  });
+
+  it.each([512, 65536])('drains on close without inventing a missing artifact at %s bytes', async (maxBytes) => {
+    let output: string = '';
+    const reporter = new AiReporter({ maxBytes, write: (text) => { output += text; } });
+    const event = (type: IReporterEventEnvelope<unknown>['type'], payload: unknown): IReporterEventEnvelope<unknown> => ({
+      protocolVersion: { major: 1, minor: 1 },
+      eventId: type,
+      sessionId: 'no-artifact',
+      sequence: 1,
+      timestamp: '2026-09-11T00:00:00.000Z',
+      source: { packageName: '@microsoft/rush-lib', packageVersion: '5.179.0' },
+      privacy: 'public',
+      required: true,
+      type,
+      payload
+    });
+    reporter.report(event('commandStarted', { commandName: 'build' }));
+    for (let iterationId: number = 0; iterationId < 1000; iterationId++) {
+      reporter.report(event('watchCycleCompleted', { iterationId, succeeded: true }));
+    }
+    await reporter.flushAsync();
+    expect(output).toBe('');
+    reporter.report(event('commandResult', { succeeded: false, exitCode: 1 }));
+    await reporter.closeAsync();
+
+    expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(maxBytes);
+    expect(output.endsWith('\n')).toBe(true);
+    const final: IAiFinalRecord = JSON.parse(output.trimEnd().split('\n').at(-1)!);
+    expect(final).toMatchObject({ kind: 'ai.final', result: 'failed', exitCode: 1, truncated: true });
+    expect(final.log).toBeUndefined();
+    const closedOutput: string = output;
+    await reporter.flushAsync();
+    await reporter.closeAsync();
+    expect(output).toBe(closedOutput);
+  });
+});

libraries/reporter/src/test/AiReporterQualification.test.ts

@@ -9,7 +9,6 @@ import {
   evaluateAiReporterQualification,
   formatAiReporterQualificationFailures,
   getQualifiedAiReporterDecision,
-  runAiReporterQualificationCorpusAsync,
   type IAiDiagnostic,
   type IAiReporterQualificationCaseResult,
   type IAiReporterQualificationGateResult,
@@ -22,6 +21,7 @@ import {
 import {
   AiQualificationTestSession,
   QUALIFICATION_CLEANUP_TIMEOUT_MS,
+  QUALIFICATION_SESSION_TIMEOUT_MS,
   QUALIFICATION_TEST_TIMEOUT_MS
 } from './helpers/AiQualificationTestSession';
 import type { AiQualificationMutation } from './helpers/AiQualificationWorker';
@@ -52,10 +52,10 @@ describe('AI reporter deterministic qualification corpus', () => {
     }
   }, QUALIFICATION_CLEANUP_TIMEOUT_MS + 1000);
 
-  // Three file-backed corpus passes can exceed Jest's default setup allowance on Windows CI.
+  // Own setup work as well as mutations so a timed-out hook cannot leave the corpus running.
   beforeAll(async () => {
-    qualification = await runAiReporterQualificationCorpusAsync();
-  }, 15000);
+    qualification = (await startSession('none').resultAsync()).qualification;
+  }, QUALIFICATION_SESSION_TIMEOUT_MS);
 
   it('passes every blocking gate with machine-readable safe results', () => {
     if (!qualification.passed) {
@@ -164,18 +164,22 @@ describe('AI reporter deterministic qualification corpus', () => {
     );
   });
 
-  it('fails with an actionable case list when the AI reporter omits its log reference', async () => {
-    const { qualification: failed } = await startSession('missing-log').resultAsync();
-    expect(failed.passed).toBe(false);
-    expect(formatAiReporterQualificationFailures(failed)).toContain(
-      'full-log: actual=0.00, required=>= 100%; cases='
-    );
-    expect(
-      failed.cases.every(({ failures }) =>
-        failures.includes('full log path, permissions, completeness, or correlation invalid')
-      )
-    ).toBe(true);
-  });
+  it(
+    'fails with an actionable case list when the AI reporter omits its log reference',
+    async () => {
+      const { qualification: failed } = await startSession('missing-log').resultAsync();
+      expect(failed.passed).toBe(false);
+      expect(formatAiReporterQualificationFailures(failed)).toContain(
+        'full-log: actual=0.00, required=>= 100%; cases='
+      );
+      expect(
+        failed.cases.every(({ failures }) =>
+          failures.includes('full log path, permissions, completeness, or correlation invalid')
+        )
+      ).toBe(true);
+    },
+    QUALIFICATION_TEST_TIMEOUT_MS
+  );
 
   it(
     'fails qualification when a renderer substitutes unrelated remediation',
@@ -212,11 +216,16 @@ describe('AI reporter deterministic qualification corpus', () => {
     QUALIFICATION_TEST_TIMEOUT_MS
   );
 
-  it.each(['timeout', 'reject', 'cancel'] as const)(
-    'joins %s mutation work before the next negative corpus',
-    async (failure) => {
+  it.each([
+    { mutation: 'none', failure: 'timeout' },
+    { mutation: 'missing-log', failure: 'timeout' },
+    { mutation: 'missing-log', failure: 'reject' },
+    { mutation: 'missing-log', failure: 'cancel' }
+  ] as const)(
+    'joins $failure $mutation work before the next negative corpus',
+    async ({ mutation, failure }) => {
       const report: typeof AiReporter.prototype.report = AiReporter.prototype.report;
-      const interrupted: AiQualificationTestSession = startSession('missing-log', {
+      const interrupted: AiQualificationTestSession = startSession(mutation, {
         waitForRelease: true,
         timeoutAfterReadyMs: failure === 'timeout' ? 50 : undefined
       });

libraries/reporter/src/test/helpers/AiQualificationTestSession.ts

@@ -13,9 +13,12 @@ import type {
   IAiQualificationWorkerResult
 } from './AiQualificationWorker';
 
-export const QUALIFICATION_TEST_TIMEOUT_MS: number = 15000;
 export const QUALIFICATION_CLEANUP_TIMEOUT_MS: number = 2000;
-const WORK_TIMEOUT_MS: number = 12000;
+const WORK_TIMEOUT_MS: number = 30000;
+export const QUALIFICATION_SESSION_TIMEOUT_MS: number =
+  WORK_TIMEOUT_MS + QUALIFICATION_CLEANUP_TIMEOUT_MS + 1000;
+// Isolation tests can await two complete corpus sessions sequentially.
+export const QUALIFICATION_TEST_TIMEOUT_MS: number = 2 * QUALIFICATION_SESSION_TIMEOUT_MS;
 
 type WorkerOutcome =
   | { success: true; result: IAiQualificationWorkerResult }

libraries/reporter/src/test/helpers/AiQualificationWorker.ts

@@ -9,7 +9,7 @@ import { runAiReporterQualificationCorpusAsync } from '../../qualification/AiRep
 import type { IAiReporterQualificationResult } from '../../qualification/AiReporterQualification';
 import type { IReporterEventEnvelope } from '../../events/IReporterEventEnvelope';
 
-export type AiQualificationMutation = 'missing-log' | 'unrelated-remediation' | 'capture-bytes';
+export type AiQualificationMutation = 'none' | 'missing-log' | 'unrelated-remediation' | 'capture-bytes';
 
 export interface IAiQualificationWorkerRequest {
   mutation: AiQualificationMutation;
@@ -48,7 +48,7 @@ if (request.mutation === 'capture-bytes') {
     }
     return byteLength(...args);
   };
-} else {
+} else if (request.mutation !== 'none') {
   AiReporter.prototype.report = function (event: IReporterEventEnvelope<unknown>): void {
     if (request.mutation === 'missing-log' && event.type === 'artifactAvailable') {
       return;