diff --git a/common/changes/@rushstack/rush-reporter/r8-machine-source-aliases_2026-09-11.json b/common/changes/@rushstack/rush-reporter/r8-machine-source-aliases_2026-09-11.json index 716c72fded..55a739983b 100644 --- a/common/changes/@rushstack/rush-reporter/r8-machine-source-aliases_2026-09-11.json +++ b/common/changes/@rushstack/rush-reporter/r8-machine-source-aliases_2026-09-11.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@rushstack/rush-reporter", - "comment": "Redact classified secret source aliases in machine JSON without losing owner-only file context, and qualify every machine output for producer privacy.", + "comment": "Redact classified secret source aliases in machine JSON without losing owner-only file context, qualify every machine output for producer privacy, and document/test late-artifact-safe AI flush reservation.", "type": "patch" } ], diff --git a/common/changes/@rushstack/rush-reporter/r8-qualification-setup-worker_2026-09-22.json b/common/changes/@rushstack/rush-reporter/r8-qualification-setup-worker_2026-09-22.json new file mode 100644 index 0000000000..ba66f3c188 --- /dev/null +++ b/common/changes/@rushstack/rush-reporter/r8-qualification-setup-worker_2026-09-22.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-reporter", + "comment": "Run AI qualification setup in an owned worker with bounded work and cleanup deadlines to avoid Windows CI setup timeouts leaving the corpus running.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-reporter" +} diff --git a/common/changes/@rushstack/rush-reporter/r8-qualification-test-isolation_2026-09-10.json b/common/changes/@rushstack/rush-reporter/r8-qualification-test-isolation_2026-09-10.json new file mode 100644 index 0000000000..1538143daa --- /dev/null +++ b/common/changes/@rushstack/rush-reporter/r8-qualification-test-isolation_2026-09-10.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-reporter", + "comment": "Isolate AI qualification test mutations and join bounded worker cleanup after delayed, rejected, or timed-out corpus work.", + "type": "none" + } + ], + "packageName": "@rushstack/rush-reporter" +} diff --git a/common/reviews/api/rush-reporter.api.md b/common/reviews/api/rush-reporter.api.md index 7ff97888d0..53894c245d 100644 --- a/common/reviews/api/rush-reporter.api.md +++ b/common/reviews/api/rush-reporter.api.md @@ -18,7 +18,6 @@ export class AiReporter implements IReporter { constructor(options: IAiReporterOptions); // (undocumented) closeAsync(): Promise; - // (undocumented) flushAsync(): Promise; // (undocumented) initializeAsync(): Promise; diff --git a/libraries/reporter/README.md b/libraries/reporter/README.md index 5b58a21045..493c189756 100644 --- a/libraries/reporter/README.md +++ b/libraries/reporter/README.md @@ -51,18 +51,27 @@ these gates only produces a reusable qualification decision; it does not enable reporter selection. That decision also requires the separate telemetry privacy prerequisite to be accepted. The pre-major Rush frontend remains explicit/repository-opt-in, and `RUSH_REPORTER=legacy` remains authoritative. -The Jest setup hook has a bounded 15-second allowance for the three file-backed corpus passes, matching -the integration test setup policy. This allowance does not change any quality gate or production deadline. +Jest setup and mutation runs use owned workers with a 30-second work budget and a separate 2-second +termination/cleanup budget. The setup hook allows 33 seconds; tests that await two corpus sessions allow +66 seconds. Timed-out work is terminated and its temporary directory removed before the session settles. +These test-only allowances do not change any quality gate or production deadline. AI output reserves final-record space, including its supplied log reference, before emitting progress. Progress is buffered within the invocation byte limit until the primary log reservation is known, or until close if no log is supplied. Excess progress/details set `truncated`; the final result remains valid JSON. -An unrendered start acknowledgement is coalesced into a known final result. Ongoing commands still expose -buffered status at the next non-terminal event or explicit flush; watch history and every final field, +An unrendered start acknowledgement is coalesced into a known final result. Once the log reservation is +known, ongoing commands expose buffered status at the next non-terminal event or explicit flush; watch history and every final field, including the supplied log reference, are retained. No path shortening or measurement normalization is used. The final scope carries the command name, and standard `diagnostic..summary` keys are implicit rather than repeated alongside the same code. Custom summary keys are preserved. +Before a log artifact arrives, AI flush and signal-flush retain bounded progress rather than spend bytes +that a late, individually representable log reference may require. Reserving half the invocation budget is +not sufficient for every supplied reference. Closing drains this bounded progress and emits the final +result even if no artifact arrives; it does not invent a log path or claim full-log completeness. +This is a limitation of eager no-artifact flushing, not permission to drop a required late artifact or +exceed the invocation budget. + AI fallback message text is emitted only for public envelopes. Its context retains the known command, and the usage-review action invokes that command's help (or `rush --help` when the command is unavailable). Qualification checks exact expected context, remediation commands/URLs, descriptions, and execution safety; diff --git a/libraries/reporter/src/reporters/AiReporter.ts b/libraries/reporter/src/reporters/AiReporter.ts index a3c0995caf..2ea5e2a682 100644 --- a/libraries/reporter/src/reporters/AiReporter.ts +++ b/libraries/reporter/src/reporters/AiReporter.ts @@ -378,6 +378,14 @@ export class AiReporter implements IReporter { } } + /** + * Flushes progress after the primary log reservation is known. + * + * @remarks + * Before an artifact arrives, even a bounded flush can consume space needed by a later + * individually representable log reference. Keep that progress bounded and deferred; + * closeAsync drains it with the final result even when no artifact ever arrives. + */ public async flushAsync(): Promise { if (this._logPath !== undefined) { this._flushPendingProgress(); diff --git a/libraries/reporter/src/test/AiReporterFlush.test.ts b/libraries/reporter/src/test/AiReporterFlush.test.ts new file mode 100644 index 0000000000..ec034ea76b --- /dev/null +++ b/libraries/reporter/src/test/AiReporterFlush.test.ts @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { + AiReporter, + ReporterManager, + type IAiFinalRecord, + type IReporterEventEnvelope +} from '../index'; + +describe('AI flush and required final-artifact reservation', () => { + it.each([ + { maxBytes: 2048, logPath: `/protected/${'logs/'.repeat(250)}full.log` }, + { maxBytes: 65536, logPath: `\\\\?\\C:\\${(`${'a'.repeat(250)}\\`).repeat(130)}full.log` } + ])('retains a representable late reference after signal flush with $maxBytes bytes', async ({ maxBytes, logPath }) => { + let output: string = ''; + const manager = new ReporterManager(); + manager.addReporter(new AiReporter({ maxBytes, write: (text) => { output += text; } })); + await manager.initializeAsync(); + const emit = (type: IReporterEventEnvelope['type'], payload: unknown): void => { + manager.emit({ + protocolVersion: { major: 1, minor: 1 }, + sessionId: 'flush-reservation', + source: { packageName: '@microsoft/rush-lib', packageVersion: '5.179.0' }, + privacy: 'public', + type, + payload + }); + }; + emit('commandStarted', { commandName: 'build' }); + for (let iterationId: number = 0; iterationId < 1000; iterationId++) { + emit('watchCycleCompleted', { iterationId, succeeded: true }); + } + await manager.signalFlushAsync(); + expect(output).toBe(''); + emit('artifactAvailable', { role: 'log', path: logPath, complete: true }); + emit('commandResult', { succeeded: false, exitCode: 1 }); + await manager.closeAsync(); + + const final: IAiFinalRecord = JSON.parse(output.trimEnd().split('\n').at(-1)!); + expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(maxBytes); + expect(final.log).toEqual({ path: logPath, complete: true }); + expect(final).toMatchObject({ kind: 'ai.final', result: 'failed', exitCode: 1, truncated: true }); + expect(final.diagnostics.length).toBeLessThanOrEqual(20); + }); + + it.each([512, 65536])('drains on close without inventing a missing artifact at %s bytes', async (maxBytes) => { + let output: string = ''; + const reporter = new AiReporter({ maxBytes, write: (text) => { output += text; } }); + const event = (type: IReporterEventEnvelope['type'], payload: unknown): IReporterEventEnvelope => ({ + protocolVersion: { major: 1, minor: 1 }, + eventId: type, + sessionId: 'no-artifact', + sequence: 1, + timestamp: '2026-09-11T00:00:00.000Z', + source: { packageName: '@microsoft/rush-lib', packageVersion: '5.179.0' }, + privacy: 'public', + required: true, + type, + payload + }); + reporter.report(event('commandStarted', { commandName: 'build' })); + for (let iterationId: number = 0; iterationId < 1000; iterationId++) { + reporter.report(event('watchCycleCompleted', { iterationId, succeeded: true })); + } + await reporter.flushAsync(); + expect(output).toBe(''); + reporter.report(event('commandResult', { succeeded: false, exitCode: 1 })); + await reporter.closeAsync(); + + expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(maxBytes); + expect(output.endsWith('\n')).toBe(true); + const final: IAiFinalRecord = JSON.parse(output.trimEnd().split('\n').at(-1)!); + expect(final).toMatchObject({ kind: 'ai.final', result: 'failed', exitCode: 1, truncated: true }); + expect(final.log).toBeUndefined(); + const closedOutput: string = output; + await reporter.flushAsync(); + await reporter.closeAsync(); + expect(output).toBe(closedOutput); + }); +}); diff --git a/libraries/reporter/src/test/AiReporterQualification.test.ts b/libraries/reporter/src/test/AiReporterQualification.test.ts index 0cc011c5fd..443a955661 100644 --- a/libraries/reporter/src/test/AiReporterQualification.test.ts +++ b/libraries/reporter/src/test/AiReporterQualification.test.ts @@ -9,7 +9,6 @@ import { evaluateAiReporterQualification, formatAiReporterQualificationFailures, getQualifiedAiReporterDecision, - runAiReporterQualificationCorpusAsync, type IAiDiagnostic, type IAiReporterQualificationCaseResult, type IAiReporterQualificationGateResult, @@ -22,6 +21,7 @@ import { import { AiQualificationTestSession, QUALIFICATION_CLEANUP_TIMEOUT_MS, + QUALIFICATION_SESSION_TIMEOUT_MS, QUALIFICATION_TEST_TIMEOUT_MS } from './helpers/AiQualificationTestSession'; import type { AiQualificationMutation } from './helpers/AiQualificationWorker'; @@ -52,10 +52,10 @@ describe('AI reporter deterministic qualification corpus', () => { } }, QUALIFICATION_CLEANUP_TIMEOUT_MS + 1000); - // Three file-backed corpus passes can exceed Jest's default setup allowance on Windows CI. + // Own setup work as well as mutations so a timed-out hook cannot leave the corpus running. beforeAll(async () => { - qualification = await runAiReporterQualificationCorpusAsync(); - }, 15000); + qualification = (await startSession('none').resultAsync()).qualification; + }, QUALIFICATION_SESSION_TIMEOUT_MS); it('passes every blocking gate with machine-readable safe results', () => { if (!qualification.passed) { @@ -164,18 +164,22 @@ describe('AI reporter deterministic qualification corpus', () => { ); }); - it('fails with an actionable case list when the AI reporter omits its log reference', async () => { - const { qualification: failed } = await startSession('missing-log').resultAsync(); - expect(failed.passed).toBe(false); - expect(formatAiReporterQualificationFailures(failed)).toContain( - 'full-log: actual=0.00, required=>= 100%; cases=' - ); - expect( - failed.cases.every(({ failures }) => - failures.includes('full log path, permissions, completeness, or correlation invalid') - ) - ).toBe(true); - }); + it( + 'fails with an actionable case list when the AI reporter omits its log reference', + async () => { + const { qualification: failed } = await startSession('missing-log').resultAsync(); + expect(failed.passed).toBe(false); + expect(formatAiReporterQualificationFailures(failed)).toContain( + 'full-log: actual=0.00, required=>= 100%; cases=' + ); + expect( + failed.cases.every(({ failures }) => + failures.includes('full log path, permissions, completeness, or correlation invalid') + ) + ).toBe(true); + }, + QUALIFICATION_TEST_TIMEOUT_MS + ); it( 'fails qualification when a renderer substitutes unrelated remediation', @@ -212,11 +216,16 @@ describe('AI reporter deterministic qualification corpus', () => { QUALIFICATION_TEST_TIMEOUT_MS ); - it.each(['timeout', 'reject', 'cancel'] as const)( - 'joins %s mutation work before the next negative corpus', - async (failure) => { + it.each([ + { mutation: 'none', failure: 'timeout' }, + { mutation: 'missing-log', failure: 'timeout' }, + { mutation: 'missing-log', failure: 'reject' }, + { mutation: 'missing-log', failure: 'cancel' } + ] as const)( + 'joins $failure $mutation work before the next negative corpus', + async ({ mutation, failure }) => { const report: typeof AiReporter.prototype.report = AiReporter.prototype.report; - const interrupted: AiQualificationTestSession = startSession('missing-log', { + const interrupted: AiQualificationTestSession = startSession(mutation, { waitForRelease: true, timeoutAfterReadyMs: failure === 'timeout' ? 50 : undefined }); diff --git a/libraries/reporter/src/test/helpers/AiQualificationTestSession.ts b/libraries/reporter/src/test/helpers/AiQualificationTestSession.ts index 31b694d59c..d65b5be58d 100644 --- a/libraries/reporter/src/test/helpers/AiQualificationTestSession.ts +++ b/libraries/reporter/src/test/helpers/AiQualificationTestSession.ts @@ -13,9 +13,12 @@ import type { IAiQualificationWorkerResult } from './AiQualificationWorker'; -export const QUALIFICATION_TEST_TIMEOUT_MS: number = 15000; export const QUALIFICATION_CLEANUP_TIMEOUT_MS: number = 2000; -const WORK_TIMEOUT_MS: number = 12000; +const WORK_TIMEOUT_MS: number = 30000; +export const QUALIFICATION_SESSION_TIMEOUT_MS: number = + WORK_TIMEOUT_MS + QUALIFICATION_CLEANUP_TIMEOUT_MS + 1000; +// Isolation tests can await two complete corpus sessions sequentially. +export const QUALIFICATION_TEST_TIMEOUT_MS: number = 2 * QUALIFICATION_SESSION_TIMEOUT_MS; type WorkerOutcome = | { success: true; result: IAiQualificationWorkerResult } diff --git a/libraries/reporter/src/test/helpers/AiQualificationWorker.ts b/libraries/reporter/src/test/helpers/AiQualificationWorker.ts index 639fdc97b2..6984207754 100644 --- a/libraries/reporter/src/test/helpers/AiQualificationWorker.ts +++ b/libraries/reporter/src/test/helpers/AiQualificationWorker.ts @@ -9,7 +9,7 @@ import { runAiReporterQualificationCorpusAsync } from '../../qualification/AiRep import type { IAiReporterQualificationResult } from '../../qualification/AiReporterQualification'; import type { IReporterEventEnvelope } from '../../events/IReporterEventEnvelope'; -export type AiQualificationMutation = 'missing-log' | 'unrelated-remediation' | 'capture-bytes'; +export type AiQualificationMutation = 'none' | 'missing-log' | 'unrelated-remediation' | 'capture-bytes'; export interface IAiQualificationWorkerRequest { mutation: AiQualificationMutation; @@ -48,7 +48,7 @@ if (request.mutation === 'capture-bytes') { } return byteLength(...args); }; -} else { +} else if (request.mutation !== 'none') { AiReporter.prototype.report = function (event: IReporterEventEnvelope): void { if (request.mutation === 'missing-log' && event.type === 'artifactAvailable') { return;