From 5c9e7205a8376f20139898d9b81d8c9c9904affa Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 16:49:19 -0700 Subject: [PATCH 1/5] Recover bounded capture deadlines and preserve terminal session loss --- .changeset/eval-capture-deadlines.md | 5 + .../integrations/core/src/facade/index.ts | 1 + .../core/src/facade/screenshot-transport.ts | 44 ++- .../core/src/facade/stdio-server.ts | 21 ++ .../integrations/core/src/facade/tools.ts | 123 ++++++-- .../tests/facade-screenshot-transport.test.ts | 66 +++++ .../core/tests/facade-tools.test.ts | 263 +++++++++++++++++- packages/sdk-ts/src/batch.ts | 32 ++- packages/sdk-ts/src/cdpClient.ts | 13 +- packages/sdk-ts/src/index.ts | 16 +- packages/sdk-ts/src/rpcClient.ts | 21 +- packages/sdk-ts/src/stagehand.ts | 49 +++- .../tests/experimentalBatchDeadline.test.ts | 149 ++++++++++ 13 files changed, 756 insertions(+), 47 deletions(-) create mode 100644 .changeset/eval-capture-deadlines.md create mode 100644 packages/sdk-ts/tests/experimentalBatchDeadline.test.ts diff --git a/.changeset/eval-capture-deadlines.md b/.changeset/eval-capture-deadlines.md new file mode 100644 index 000000000..94edd0cd9 --- /dev/null +++ b/.changeset/eval-capture-deadlines.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Bound experimental batch and RPC deadlines so callers can stop waiting without replaying actions or accepting late capture state. diff --git a/packages/integrations/core/src/facade/index.ts b/packages/integrations/core/src/facade/index.ts index c78f8df1f..9a9271d8b 100644 --- a/packages/integrations/core/src/facade/index.ts +++ b/packages/integrations/core/src/facade/index.ts @@ -37,6 +37,7 @@ export { } from "./contract.js"; export { StagehandFacadeTools, + StagehandFacadeSessionLostError, type StagehandFacadeRunReport, type StagehandFacadeToolsOptions, } from "./tools.js"; diff --git a/packages/integrations/core/src/facade/screenshot-transport.ts b/packages/integrations/core/src/facade/screenshot-transport.ts index 065bff1db..91c1e0e37 100644 --- a/packages/integrations/core/src/facade/screenshot-transport.ts +++ b/packages/integrations/core/src/facade/screenshot-transport.ts @@ -27,24 +27,64 @@ export function screenshotBase64BudgetFromArgs(args: string[]): number | undefin return budget; } +/** + * Model APIs reject images with a side longer than this. Anthropic allows + * 8000 px for a lone image but only 2000 px once a request carries many + * images, which every multi-step agent conversation does. Full-page captures + * of long pages exceed both and killed whole runs with a 400, so oversized + * captures fall back to the viewport like over-budget ones do. + */ +export const MAX_SCREENSHOT_SIDE_PX = 2000; + export async function captureScreenshotWithinBase64Budget( capture: CaptureScreenshot, requested: ScreenshotOptions, maxBase64Bytes: number, + maxSidePx = MAX_SCREENSHOT_SIDE_PX, ): Promise { const attempts = screenshotAttempts(requested); for (const [index, options] of attempts.entries()) { const image = await capture(options); - if (Buffer.byteLength(image.data, "utf8") <= maxBase64Bytes) { + const size = imageDimensions(image); + const tooLarge = size !== undefined && Math.max(size.width, size.height) > maxSidePx; + if (!tooLarge && Buffer.byteLength(image.data, "utf8") <= maxBase64Bytes) { return { image, options, adjusted: index > 0 || !sameOptions(options, requested) }; } } throw new Error( - `Screenshot exceeds the ${maxBase64Bytes}-byte MCP transport budget after compressed viewport retries.`, + `Screenshot exceeds the ${maxBase64Bytes}-byte MCP transport budget or the ${maxSidePx}px side limit after compressed viewport retries.`, ); } +/** Reads width/height from a PNG or JPEG header; undefined when unparseable. */ +export function imageDimensions( + image: StagehandFacadeScreenshot, +): { width: number; height: number } | undefined { + const bytes = Buffer.from(image.data, "base64"); + if (image.mimeType === "image/png") { + if (bytes.length < 24 || bytes.toString("ascii", 1, 4) !== "PNG") return undefined; + return { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) }; + } + // JPEG: walk the marker segments to the first SOFn (C0–CF except C4, C8, CC). + if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return undefined; + let offset = 2; + while (offset + 9 <= bytes.length) { + if (bytes[offset] !== 0xff) return undefined; + const marker = bytes[offset + 1]!; + if (marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd7) || marker === 0x01) { + offset += 2; + continue; + } + const length = bytes.readUInt16BE(offset + 2); + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { + return { height: bytes.readUInt16BE(offset + 5), width: bytes.readUInt16BE(offset + 7) }; + } + offset += 2 + length; + } + return undefined; +} + function screenshotAttempts(requested: ScreenshotOptions): ScreenshotOptions[] { const initial: ScreenshotOptions = { fullPage: requested.fullPage ?? false, diff --git a/packages/integrations/core/src/facade/stdio-server.ts b/packages/integrations/core/src/facade/stdio-server.ts index 167fa56d2..8cf2fac58 100644 --- a/packages/integrations/core/src/facade/stdio-server.ts +++ b/packages/integrations/core/src/facade/stdio-server.ts @@ -17,6 +17,7 @@ import { facadeSurfaceFromArgs, facadeToolsFor, SESSION_INFO_TOOL_NAME, + SESSION_LOST_TELEMETRY_PREFIX, ScreenshotInputSchema, SnapshotInputSchema, } from "./contract.js"; @@ -136,11 +137,31 @@ async function createResources(): Promise { config.browser.type === "browserbase" ? await browserbase.launch(config.browser.launchOptions) : await localBrowser.launch(config.browser.launchOptions); + const launchedAt = Date.now(); try { const stagehand = await Stagehand.create({ browser, ...config.stagehand }); const tools = new StagehandFacadeTools(stagehand, { onRunReport: (report) => process.stderr.write(`stagehand_playwright_compat ${JSON.stringify(report)}\n`), + // The browser is not recreated on purpose: a fresh session would silently + // change the evidence trail mid-task. Tools keep answering with the + // terminal error and the host decides what to do with the run. + // sessionAgeMs against the configured session timeout tells a Browserbase + // TIMED_OUT apart from a remote close. + onSessionLost: (loss) => + process.stderr.write( + `${SESSION_LOST_TELEMETRY_PREFIX}${JSON.stringify({ + ...loss, + cause: sanitizeErrorMessage(loss.cause), + provider: browser.provider, + ...(browser.sessionId && { sessionId: browser.sessionId }), + sessionAgeMs: Date.now() - launchedAt, + ...(config.browser.type === "browserbase" && + typeof config.browser.launchOptions.timeout === "number" && { + sessionTimeoutMs: config.browser.launchOptions.timeout * 1000, + }), + })}\n`, + ), }); return { browser, stagehand, tools }; } catch (error) { diff --git a/packages/integrations/core/src/facade/tools.ts b/packages/integrations/core/src/facade/tools.ts index 8341fa8d4..2a090b43f 100644 --- a/packages/integrations/core/src/facade/tools.ts +++ b/packages/integrations/core/src/facade/tools.ts @@ -3,10 +3,12 @@ import path from "node:path"; import type { ExperimentalBatchCallback, Page, Stagehand } from "@browserbasehq/stagehand"; import { sanitizeErrorMessage } from "../harness/redact.js"; import { + browserSessionLostError, NAVIGATED_SNAPSHOT_ERROR, NO_HYDRATED_SNAPSHOT_ERROR, RefActionSchema, staleSnapshotIdError, + type FacadeSessionLoss, type RefAction, } from "./contract.js"; import { createPlaywrightCompatRuntime, type PlaywrightCompatTelemetry } from "./runtime.js"; @@ -39,6 +41,8 @@ export type StagehandFacadeToolsOptions = { artifactRoot?: string; /** Observes every completed `run` batch (including ones whose code threw). */ onRunReport?: (report: StagehandFacadeRunReport) => void; + /** Fires once, the first time a call proves the browser session is gone. */ + onSessionLost?: (loss: FacadeSessionLoss) => void; /** * Keep a hidden about:blank tab open for the whole session (default true). * Chrome exits when its last tab closes, so a renderer crash on the agent's @@ -51,7 +55,21 @@ export type StagehandFacadeToolsOptions = { keeperPage?: boolean; }; +/** Every facade tool returns this once the browser session is gone. */ +export class StagehandFacadeSessionLostError extends Error { + override readonly name = "StagehandFacadeSessionLostError"; + constructor(readonly loss: FacadeSessionLoss) { + super(browserSessionLostError(loss.cause)); + } +} + const RUN_BATCH_TIMEOUT_MS = 60_000; +/** + * snapshot/screenshot RPCs have no executor-side deadline. Calls are serialized, + * so one that never answers must release the queue at a bounded deadline. + * Repeated capture deadlines latch terminal loss; no underlying RPC is retried. + */ +const PAGE_CAPTURE_DEADLINE_MS = 120_000; export type StagehandFacadeScreenshot = { data: string; @@ -129,12 +147,23 @@ type RunInput = { hiddenPageIds?: string[] }; export class StagehandFacadeTools { private readonly snapshotsByPage = new Map(); private queue: Promise = Promise.resolve(); + private loss: FacadeSessionLoss | undefined; private keeper: Promise | undefined; + // Consecutive capture-deadline timeouts; reset by any successful tool call. + // Three consecutive failures end this facade; this does not prove transport loss. + private consecutiveDeadlines = 0; + private static readonly MAX_CONSECUTIVE_DEADLINES = 3; + constructor( private readonly stagehand: Stagehand, private readonly options: StagehandFacadeToolsOptions = {}, ) {} + /** Set once a call has proven the browser session is gone; never cleared. */ + get sessionLoss(): FacadeSessionLoss | undefined { + return this.loss; + } + snapshot(options: { includeIframes?: boolean } = {}): Promise { return this.enqueue("snapshot", () => this.snapshotNow(options)); } @@ -155,11 +184,18 @@ export class StagehandFacadeTools { private async snapshotNow(options: { includeIframes?: boolean }): Promise { const page = await this.activePage(); - const snapshot = await page.snapshot({ includeIframes: options.includeIframes ?? true }); - this.snapshotsByPage.set(page.pageId, { - url: await page.url(), - xpathById: { ...snapshot.xpathMap }, - }); + // Failed captures invalidate the preceding snapshot too. A late response + // must never replace the IDs installed by a subsequent successful capture. + this.snapshotsByPage.delete(page.pageId); + const { snapshot, url } = await withDeadline( + (async () => { + const snapshot = await page.snapshot({ includeIframes: options.includeIframes ?? true }); + return { snapshot, url: await page.url() }; + })(), + PAGE_CAPTURE_DEADLINE_MS, + "page.snapshot", + ); + this.snapshotsByPage.set(page.pageId, { url, xpathById: { ...snapshot.xpathMap } }); return snapshot.formattedTree; } @@ -173,11 +209,15 @@ export class StagehandFacadeTools { // CDP only accepts quality for jpeg, and only as an integer. const quality = type === "jpeg" && options.quality !== undefined ? Math.round(options.quality) : undefined; - const bytes = await page.screenshot({ - type, - ...(options.fullPage === undefined ? {} : { fullPage: options.fullPage }), - ...(quality === undefined ? {} : { quality }), - }); + const bytes = await withDeadline( + page.screenshot({ + type, + ...(options.fullPage === undefined ? {} : { fullPage: options.fullPage }), + ...(quality === undefined ? {} : { quality }), + }), + PAGE_CAPTURE_DEADLINE_MS, + "page.screenshot", + ); return { data: Buffer.from(bytes).toString("base64"), mimeType: type === "jpeg" ? "image/jpeg" : "image/png", @@ -200,10 +240,12 @@ export class StagehandFacadeTools { if (!xpath) throw new Error(staleSnapshotIdError(action.id)); return { ...action, selector: `xpath=${xpath}` }; }); + // The callback can already have dispatched earlier actions when one fails. + // Never replay an action batch after a partial failure. const result = await this.stagehand.experimentalBatch( actionRunner, { actions: hydrated }, - { page, timeout: 60_000 }, + { page, timeout: RUN_BATCH_TIMEOUT_MS }, ); return { completed: result?.completed ?? hydrated.length, url: await page.url() }; } @@ -322,12 +364,35 @@ export class StagehandFacadeTools { return keeper.pageId; } - private enqueue(_tool: string, operation: () => Promise): Promise { - const execute = async (): Promise => { - await this.ensureKeeperPage(); - return operation(); + private enqueue(tool: string, operation: () => Promise): Promise { + const guarded = async (): Promise => { + if (this.loss) throw new StagehandFacadeSessionLostError(this.loss); + try { + await this.ensureKeeperPage(); + const value = await operation(); + this.consecutiveDeadlines = 0; // a real response proves the session is alive + return value; + } catch (error) { + // Permit two consecutive capture timeouts. The deadline does not cancel + // the underlying RPC, and recovery never replays that capture or an action. + if (error instanceof FacadeDeadlineError) { + this.consecutiveDeadlines += 1; + if (this.consecutiveDeadlines < StagehandFacadeTools.MAX_CONSECUTIVE_DEADLINES) { + throw error; + } + const cause = `executor unresponsive: ${this.consecutiveDeadlines} consecutive capture timeouts (last: ${error.message})`; + this.loss = { cause, tool, at: new Date().toISOString() }; + this.options.onSessionLost?.(this.loss); + throw new StagehandFacadeSessionLostError(this.loss); + } + const cause = sessionLossCause(error); + if (cause === undefined) throw error; + this.loss = { cause, tool, at: new Date().toISOString() }; + this.options.onSessionLost?.(this.loss); + throw new StagehandFacadeSessionLostError(this.loss); + } }; - const result = this.queue.then(execute, execute); + const result = this.queue.then(guarded, guarded); this.queue = result.then( () => undefined, () => undefined, @@ -336,6 +401,32 @@ export class StagehandFacadeTools { } } +class FacadeDeadlineError extends Error { + override readonly name = "FacadeDeadlineError"; + constructor(operation: string, timeoutMs: number) { + super(`${operation} received no response within ${timeoutMs}ms`); + } +} + +function withDeadline(promise: Promise, timeoutMs: number, operation: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new FacadeDeadlineError(operation, timeoutMs)), + timeoutMs, + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error: unknown) => { + clearTimeout(timer); + reject(error); + }, + ); + }); +} + /** * Maps an error to the reason the browser session is unusable, or undefined * when it is an ordinary tool failure the agent can act on. A batch that hit diff --git a/packages/integrations/core/tests/facade-screenshot-transport.test.ts b/packages/integrations/core/tests/facade-screenshot-transport.test.ts index 109aec417..f35a69cbd 100644 --- a/packages/integrations/core/tests/facade-screenshot-transport.test.ts +++ b/packages/integrations/core/tests/facade-screenshot-transport.test.ts @@ -3,6 +3,7 @@ import { captureScreenshotWithinBase64Budget, screenshotBase64BudgetFromArgs, type ScreenshotOptions, + imageDimensions, } from "../src/facade/screenshot-transport.js"; describe("facade screenshot transport", () => { @@ -95,3 +96,68 @@ describe("facade screenshot transport", () => { expect(capture).toHaveBeenCalledTimes(3); }); }); + +function png(width: number, height: number): string { + const b = Buffer.alloc(24); + b.write("\x89PNG\r\n\x1a\n", 0, "binary"); + b.writeUInt32BE(13, 8); + b.write("IHDR", 12, "ascii"); + b.writeUInt32BE(width, 16); + b.writeUInt32BE(height, 20); + return b.toString("base64"); +} +function jpeg(width: number, height: number): string { + // SOI, APP0 (empty), SOF0 with the given size. + const b = Buffer.from([ + 0xff, + 0xd8, + 0xff, + 0xe0, + 0x00, + 0x02, + 0xff, + 0xc0, + 0x00, + 0x0b, + 0x08, + (height >> 8) & 0xff, + height & 0xff, + (width >> 8) & 0xff, + width & 0xff, + 0x01, + 0x01, + 0x11, + 0x00, + ]); + return b.toString("base64"); +} + +describe("screenshot dimension guard", () => { + it("parses PNG and JPEG headers", () => { + expect(imageDimensions({ data: png(1288, 9400), mimeType: "image/png" })).toStrictEqual({ + width: 1288, + height: 9400, + }); + expect(imageDimensions({ data: jpeg(640, 480), mimeType: "image/jpeg" })).toStrictEqual({ + width: 640, + height: 480, + }); + }); + + it("falls back to the viewport when a full-page capture exceeds the side limit", async () => { + const calls: Array<{ fullPage?: boolean }> = []; + const result = await captureScreenshotWithinBase64Budget( + async (options) => { + calls.push(options); + return options.fullPage + ? { data: png(1288, 2400), mimeType: "image/png" as const } + : { data: jpeg(1288, 711), mimeType: "image/jpeg" as const }; + }, + { fullPage: true, type: "png" }, + 10_000_000, + ); + expect(calls[0]?.fullPage).toBe(true); + expect(result.options.fullPage).toBe(false); + expect(result.adjusted).toBe(true); + }); +}); diff --git a/packages/integrations/core/tests/facade-tools.test.ts b/packages/integrations/core/tests/facade-tools.test.ts index c32a4b491..7b1445853 100644 --- a/packages/integrations/core/tests/facade-tools.test.ts +++ b/packages/integrations/core/tests/facade-tools.test.ts @@ -1,13 +1,13 @@ import fsp from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { type Stagehand } from "@browserbasehq/stagehand"; +import { CDPConnectionClosedError, type Stagehand } from "@browserbasehq/stagehand"; import { afterEach, describe, expect, it, vi } from "vitest"; import { BROWSER_SESSION_LOST_ERROR_PREFIX, type FacadeSessionLoss, } from "../src/facade/contract.js"; -import { StagehandFacadeTools, type StagehandFacadeRunReport } from "../src/facade/tools.js"; +import { StagehandFacadeSessionLostError, StagehandFacadeTools, type StagehandFacadeRunReport } from "../src/facade/tools.js"; type FakePage = ReturnType; @@ -320,3 +320,262 @@ describe("StagehandFacadeTools keeper tab", () => { expect(context.newPage).not.toHaveBeenCalled(); }); }); + +describe("StagehandFacadeTools session loss", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + function batchTimeoutError() { + const error = new Error("stagehand.experimentalBatch() received no response within 75000ms"); + error.name = "StagehandBatchTimeoutError"; + Object.assign(error, { timeout: 60_000, clientTimeout: 75_000 }); + return error; + } + + it("turns a batch client deadline into the terminal error and stays dead", async () => { + const page = createFakePage(); + const { stagehand, experimentalBatch, context } = createFakeStagehand(page); + experimentalBatch.mockRejectedValueOnce(batchTimeoutError()); + const losses: FacadeSessionLoss[] = []; + const tools = new StagehandFacadeTools(stagehand, { + onSessionLost: (loss) => losses.push(loss), + }); + + const first = tools.run("await page.getByRole('button', { name: 'Search now' }).click();"); + await expect(first).rejects.toBeInstanceOf(StagehandFacadeSessionLostError); + await expect(first).rejects.toThrow( + "Browser session lost (batch received no response within 75000ms). The task cannot continue; report your final result now.", + ); + expect(losses).toEqual([ + { cause: "batch received no response within 75000ms", tool: "run", at: expect.any(String) }, + ]); + expect(tools.sessionLoss).toBe(losses[0]); + + // Every later call gets the same terminal answer without touching the browser. + const callsBefore = context.activePage.mock.calls.length; + await expect(tools.snapshot()).rejects.toThrow(BROWSER_SESSION_LOST_ERROR_PREFIX); + await expect(tools.run("return 1;")).rejects.toThrow(BROWSER_SESSION_LOST_ERROR_PREFIX); + await expect(tools.screenshot()).rejects.toThrow(BROWSER_SESSION_LOST_ERROR_PREFIX); + expect(context.activePage.mock.calls.length).toBe(callsBefore); + expect(experimentalBatch).toHaveBeenCalledTimes(1); + expect(losses).toHaveLength(1); + }); + + it("treats a closed RPC/CDP transport as session loss", async () => { + const page = createFakePage(); + const { stagehand, context } = createFakeStagehand(page); + context.activePage.mockRejectedValueOnce(new Error("RPC client is closed")); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.snapshot()).rejects.toThrow("Browser session lost (RPC client closed)."); + expect(tools.sessionLoss?.tool).toBe("snapshot"); + }); + + it("preserves socket diagnostics when a CDP error arrives before close", async () => { + // The SDK error-before-close path wraps the socket failure as cause, + // without a close code in the outer message. + const error = new CDPConnectionClosedError({ + cause: new TypeError("WebSocket failed", { + cause: Object.assign(new Error("other side closed"), { code: "UND_ERR_SOCKET" }), + }), + }); + const { stagehand, context } = createFakeStagehand(createFakePage()); + context.activePage.mockRejectedValueOnce(error); + const onSessionLost = vi.fn(); + const tools = new StagehandFacadeTools(stagehand, { onSessionLost }); + const cause = + "CDP connection closed; caused by TypeError: WebSocket failed; caused by Error [UND_ERR_SOCKET]: other side closed"; + + await expect(tools.snapshot()).rejects.toThrow(cause); + await expect(tools.snapshot()).rejects.toThrow(cause); + expect(tools.sessionLoss?.cause).toBe(cause); + expect(onSessionLost).toHaveBeenCalledOnce(); + expect(context.activePage).toHaveBeenCalledOnce(); + }); + + it("redacts credentials in CDP close reasons and nested socket causes before emitting loss", async () => { + const socketError = new TypeError( + "wss://browser.example/session?signingKey=url-secret&apiKey=api-secret&token=token-secret " + + "sk-abcdef1234567890 bb_live_abcd1234567890 Bearer bearer-secret-value", + ); + const error = new CDPConnectionClosedError({ + code: 1006, + reason: "https://browser.example/?key=reason-secret", + cause: socketError, + }); + const { stagehand, context } = createFakeStagehand(createFakePage()); + context.activePage.mockRejectedValueOnce(error); + const onSessionLost = vi.fn(); + const tools = new StagehandFacadeTools(stagehand, { onSessionLost }); + + const failure = await tools.snapshot().catch((error: unknown) => error); + expect(failure).toBeInstanceOf(StagehandFacadeSessionLostError); + const output = JSON.stringify({ + message: (failure as Error).message, + loss: tools.sessionLoss, + callback: onSessionLost.mock.calls, + }); + for (const secret of [ + "url-secret", + "api-secret", + "token-secret", + "reason-secret", + "sk-abcdef1234567890", + "bb_live_abcd1234567890", + "bearer-secret-value", + ]) { + expect(output).not.toContain(secret); + } + expect(tools.sessionLoss?.cause).toContain("close code 1006"); + expect(tools.sessionLoss?.cause).toContain( + "caused by TypeError: wss://browser.example/session", + ); + expect(tools.sessionLoss?.cause).toContain( + "signingKey=[redacted]&apiKey=[redacted]&token=[redacted]", + ); + }); + + it("handles empty and cyclic CDP causes without losing the error type", async () => { + const socketError = new TypeError(); + const error = new CDPConnectionClosedError({ cause: socketError }); + socketError.cause = error; + const { stagehand, context } = createFakeStagehand(createFakePage()); + context.activePage.mockRejectedValueOnce(error); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.snapshot()).rejects.toThrow("CDP connection closed; caused by TypeError"); + expect(tools.sessionLoss?.cause).toBe("CDP connection closed; caused by TypeError"); + }); + + it("does not treat an executor-side batch timeout or agent code errors as session loss", async () => { + const page = createFakePage(); + const { stagehand, experimentalBatch } = createFakeStagehand(page); + experimentalBatch.mockRejectedValueOnce( + new Error("Stagehand callback batch timed out after 60000ms"), + ); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.run("return 1;")).rejects.toThrow("callback batch timed out"); + expect(tools.sessionLoss).toBeUndefined(); + await expect(tools.run("throw new Error('RPC client is closed');")).rejects.toThrow( + "RPC client is closed", + ); + expect(tools.sessionLoss).toBeUndefined(); + await expect(tools.run("return 2;")).resolves.toBe(2); + }); + + it("resets capture timeout count after a successful operation", async () => { + vi.useFakeTimers(); + const page = createFakePage(); + page.snapshot.mockImplementation(() => new Promise(() => undefined)); + const { stagehand } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + const timeout = async () => { + const rejection = expect(tools.snapshot()).rejects.toThrow("received no response"); + await vi.advanceTimersByTimeAsync(120_000); + await rejection; + }; + await timeout(); + await timeout(); + await expect(tools.run("return 1;")).resolves.toBe(1); + await timeout(); + await timeout(); + expect(tools.sessionLoss).toBeUndefined(); + }); + + it("ignores a late timed-out snapshot after newer IDs have been installed", async () => { + vi.useFakeTimers(); + const world = createFakeWorld(); + const page = createFakePage("https://example.com", world); + let resolveLate!: (value: { formattedTree: string; xpathMap: Record }) => void; + page.snapshot.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveLate = resolve; + }), + ); + const { stagehand } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + const rejection = expect(tools.snapshot()).rejects.toThrow("received no response"); + await vi.advanceTimersByTimeAsync(120_000); + await rejection; + world.snapshot = { formattedTree: "new", xpathMap: { "0-2": "/new/button" } }; + await expect(tools.snapshot()).resolves.toBe("new"); + resolveLate({ formattedTree: "old", xpathMap: { "0-1": "/old/button" } }); + await Promise.resolve(); + await expect(tools.runActions([{ op: "click", id: "0-2" }])).resolves.toMatchObject({ + completed: 1, + }); + expect(page.locator).toHaveBeenLastCalledWith("xpath=/new/button"); + await expect(tools.runActions([{ op: "click", id: "0-1" }])).rejects.toThrow(); + }); + + it("latches terminal loss once and never dispatches work already queued behind it", async () => { + const page = createFakePage(); + const { stagehand, experimentalBatch } = createFakeStagehand(page); + experimentalBatch.mockRejectedValueOnce(batchTimeoutError()); + const onSessionLost = vi.fn(); + const tools = new StagehandFacadeTools(stagehand, { onSessionLost }); + const outcomes = await Promise.allSettled([ + tools.run("return 1;"), + tools.snapshot(), + tools.screenshot(), + tools.run("return 2;"), + ]); + expect( + outcomes.every( + (result) => + result.status === "rejected" && result.reason instanceof StagehandFacadeSessionLostError, + ), + ).toBe(true); + expect(onSessionLost).toHaveBeenCalledOnce(); + expect(experimentalBatch).toHaveBeenCalledOnce(); + expect(page.snapshot).not.toHaveBeenCalled(); + expect(page.screenshot).not.toHaveBeenCalled(); + }); + + it("does not replay successful actions when a later action fails", async () => { + const world = createFakeWorld(); + world.snapshot = { formattedTree: "actions", xpathMap: { "0-1": "/first", "0-2": "/second" } }; + world.clickErrors["xpath=/second"] = new Error("Node does not have a layout object"); + const page = createFakePage("https://example.com", world); + const { stagehand, experimentalBatch } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + await tools.snapshot(); + await expect( + tools.runActions([ + { op: "click", id: "0-1" }, + { op: "click", id: "0-2" }, + ]), + ).rejects.toThrow("layout object"); + expect(experimentalBatch).toHaveBeenCalledOnce(); + expect(world.locators.filter((locator) => locator.selector === "xpath=/first")).toHaveLength(1); + expect(world.locators[0]?.click).toHaveBeenCalledOnce(); + expect(page.waitForTimeout).not.toHaveBeenCalled(); + }); + + it("treats a single capture deadline as recoverable, escalating to session loss only after repeated consecutive timeouts", async () => { + vi.useFakeTimers(); + const page = createFakePage(); + page.snapshot.mockImplementation(() => new Promise(() => undefined)); + const { stagehand } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + + for (let i = 0; i < 2; i++) { + const pending = tools.snapshot(); + const rejection = expect(pending).rejects.toThrow( + "page.snapshot received no response within 120000ms", + ); + await vi.advanceTimersByTimeAsync(120_000); + await rejection; + expect(tools.sessionLoss).toBeUndefined(); + } + + const pending = tools.snapshot(); + const rejection = expect(pending).rejects.toThrow("Browser session lost"); + await vi.advanceTimersByTimeAsync(120_000); + await rejection; + expect(tools.sessionLoss?.cause).toContain("consecutive capture timeouts"); + }); +}); diff --git a/packages/sdk-ts/src/batch.ts b/packages/sdk-ts/src/batch.ts index c63307336..403ddef2c 100644 --- a/packages/sdk-ts/src/batch.ts +++ b/packages/sdk-ts/src/batch.ts @@ -15,10 +15,40 @@ import type { Page } from "./page.js"; export type ExperimentalBatchOptions = { /** Page exposed as `batch.page`. AI operations still default to the active page. */ page?: Page; - /** Overall callback deadline in milliseconds. */ + /** Overall callback deadline in milliseconds, enforced by the browser-side executor. */ timeout?: number; + /** + * Local deadline for the whole round trip in milliseconds. Defaults to + * `timeout + CALLBACK_BATCH_CLIENT_GRACE_MS`. It fires when the executor never + * answers at all (stalled navigation, hung service worker), which the + * browser-side `timeout` cannot cover. + */ + clientTimeoutMs?: number; }; +/** Slack the client grants the executor to report its own `timeout` before giving up locally. */ +export const CALLBACK_BATCH_CLIENT_GRACE_MS = 15_000; + +/** + * The batch round trip exceeded its client-side deadline. The executor may + * still be running the callback; the browser session should be treated as + * unresponsive rather than retried blindly. + */ +export class StagehandBatchTimeoutError extends Error { + readonly timeout: number; + readonly clientTimeout: number; + + constructor(details: { timeout: number; clientTimeout: number }, options?: ErrorOptions) { + super( + `stagehand.experimentalBatch() received no response within ${details.clientTimeout}ms (callback timeout ${details.timeout}ms)`, + options, + ); + this.name = "StagehandBatchTimeoutError"; + this.timeout = details.timeout; + this.clientTimeout = details.clientTimeout; + } +} + export type ExperimentalBatchBrowserContext = Omit< BrowserContext, "close" | "rpcClient" | "clipboardRef" diff --git a/packages/sdk-ts/src/cdpClient.ts b/packages/sdk-ts/src/cdpClient.ts index 487ca813e..2fdddabb5 100644 --- a/packages/sdk-ts/src/cdpClient.ts +++ b/packages/sdk-ts/src/cdpClient.ts @@ -173,8 +173,12 @@ const InstalledExtensionsResultSchema = z.looseObject({ const STAGEHAND_EXTENSION_NAME = "Stagehand Runtime"; export class CDPConnectionClosedError extends Error { - constructor(options?: ErrorOptions) { - super("CDP connection closed", options); + constructor(options?: ErrorOptions & { code?: number; reason?: string }) { + const detail = + options?.code !== undefined + ? ` (close code ${options.code}${options.reason ? `: ${options.reason}` : ""})` + : ""; + super(`CDP connection closed${detail}`, options); this.name = "CDPConnectionClosedError"; } } @@ -203,10 +207,11 @@ export class CDPClient { }); }); - this.socket.addEventListener("close", () => { + this.socket.addEventListener("close", (event) => { if (this.closed) return; this.closed = true; - const reason = new CDPConnectionClosedError(); + const { code, reason: closeReason } = event as Event & { code?: number; reason?: string }; + const reason = new CDPConnectionClosedError({ code, reason: closeReason }); this.rejectPending(reason); this.onclose?.(reason); }); diff --git a/packages/sdk-ts/src/index.ts b/packages/sdk-ts/src/index.ts index 5e3b28382..8c699ee11 100644 --- a/packages/sdk-ts/src/index.ts +++ b/packages/sdk-ts/src/index.ts @@ -38,14 +38,18 @@ export { type ResponseServerAddr, } from "./response.js"; export { WebMCPInvocation, WebMCPTool } from "./webmcp.js"; +export { CDPConnectionClosedError } from "./cdpClient.js"; +export { RPCResponseTimeoutError } from "./rpcClient.js"; export type { InitScriptSource } from "./pageScripts.js"; export { Stagehand, type ExtractResult } from "./stagehand.js"; -export type { - ExperimentalBatchCallback, - ExperimentalBatchBrowserContext, - ExperimentalBatchContext, - ExperimentalBatchExtractOptions, - ExperimentalBatchOptions, +export { + CALLBACK_BATCH_CLIENT_GRACE_MS, + StagehandBatchTimeoutError, + type ExperimentalBatchCallback, + type ExperimentalBatchBrowserContext, + type ExperimentalBatchContext, + type ExperimentalBatchExtractOptions, + type ExperimentalBatchOptions, } from "./batch.js"; export { browserbase, localBrowser } from "./browser/factories.js"; export type { diff --git a/packages/sdk-ts/src/rpcClient.ts b/packages/sdk-ts/src/rpcClient.ts index dee85b525..9e08c8a60 100644 --- a/packages/sdk-ts/src/rpcClient.ts +++ b/packages/sdk-ts/src/rpcClient.ts @@ -54,8 +54,22 @@ type RegisteredRequestHandler = { type RPCSendOptions = { signal?: AbortSignal; + /** Replaces the method's derived response deadline for this one request. */ + responseTimeoutMs?: number; }; +export class RPCResponseTimeoutError extends Error { + readonly method: string; + readonly timeoutMs: number; + + constructor(method: string, timeoutMs: number) { + super(`RPC response timed out: ${method}`, { cause: { method, timeoutMs } }); + this.name = "RPCResponseTimeoutError"; + this.method = method; + this.timeoutMs = timeoutMs; + } +} + const TRACER = trace.getTracer("@browserbasehq/stagehand"); const W3C_TRACE_CONTEXT_PROPAGATOR = new W3CTraceContextPropagator(); const MAX_PENDING_NOTIFICATIONS = 100; @@ -190,7 +204,8 @@ export class RPCClient { ...getTraceContextFields(requestContext), }); span.setAttribute("jsonrpc.request.id", String(request.id)); - const responseTimeoutMs = rpcResponseTimeoutMs(method.name, parsedParams); + const responseTimeoutMs = + options.responseTimeoutMs ?? rpcResponseTimeoutMs(method.name, parsedParams); const timeoutController = responseTimeoutMs === undefined ? undefined : new AbortController(); const signal = @@ -201,9 +216,7 @@ export class RPCClient { timeoutController && responseTimeoutMs !== undefined ? setTimeout(() => { timeoutController.abort( - new Error(`RPC response timed out: ${method.name}`, { - cause: { method: method.name, timeoutMs: responseTimeoutMs }, - }), + new RPCResponseTimeoutError(method.name, responseTimeoutMs), ); }, responseTimeoutMs) : undefined; diff --git a/packages/sdk-ts/src/stagehand.ts b/packages/sdk-ts/src/stagehand.ts index f5a10d4fe..c008f8221 100644 --- a/packages/sdk-ts/src/stagehand.ts +++ b/packages/sdk-ts/src/stagehand.ts @@ -1,4 +1,4 @@ -import { RPCClient } from "./rpcClient.js"; +import { RPCClient, RPCResponseTimeoutError } from "./rpcClient.js"; import { DefaultExtractDataSchema, MAX_CALLBACK_BATCH_TIMEOUT_MS, @@ -45,7 +45,12 @@ import { } from "./browser/factories.js"; import { attachStagehandBrowserContext, detachStagehandBrowserContext } from "./browser/index.js"; import { withStagehandInitDeadline } from "./timeouts.js"; -import type { ExperimentalBatchCallback, ExperimentalBatchOptions } from "./batch.js"; +import { + CALLBACK_BATCH_CLIENT_GRACE_MS, + StagehandBatchTimeoutError, + type ExperimentalBatchCallback, + type ExperimentalBatchOptions, +} from "./batch.js"; type ProtocolExtractResult = import("@browserbasehq/stagehand-protocol/types").ExtractResult; @@ -61,6 +66,9 @@ const isZodSchema = (value: unknown): value is z.ZodType => "safeParse" in value && typeof value.safeParse === "function"; +// setTimeout treats larger delays as 1ms; MAX_CALLBACK_BATCH_TIMEOUT_MS leaves 10s below this. +const MAX_TIMER_DELAY_MS = 2_147_483_647; + const nativeFunctionSourcePattern = /^\s*function(?:\s+[^()]*)?\([^)]*\)\s*\{\s*\[native code\]\s*\}\s*$/; @@ -154,18 +162,35 @@ export class Stagehand { if (nativeFunctionSourcePattern.test(callbackSource)) { throw new TypeError("stagehand.experimentalBatch() callback must be serializable JavaScript"); } + const clientTimeout = + options.clientTimeoutMs ?? + Math.min(timeout + CALLBACK_BATCH_CLIENT_GRACE_MS, MAX_TIMER_DELAY_MS); + if (!Number.isInteger(clientTimeout) || clientTimeout <= 0 || clientTimeout > 2_147_483_647) { + throw new RangeError( + "stagehand.experimentalBatch() clientTimeoutMs must be an integer between 1 and 2147483647", + ); + } - const result: CallbackBatchResult = await this.connectedRpcClient.send( - StagehandMethods.stagehandCallbackBatch, - { - callbackSource, - ...(parsedInput === undefined ? {} : { input: parsedInput }), - options: { - ...(options.page ? { pageId: options.page.pageId } : {}), - timeout, + let result: CallbackBatchResult; + try { + result = await this.connectedRpcClient.send( + StagehandMethods.stagehandCallbackBatch, + { + callbackSource, + ...(parsedInput === undefined ? {} : { input: parsedInput }), + options: { + ...(options.page ? { pageId: options.page.pageId } : {}), + timeout, + }, }, - }, - ); + { responseTimeoutMs: clientTimeout }, + ); + } catch (error) { + if (error instanceof RPCResponseTimeoutError) { + throw new StagehandBatchTimeoutError({ timeout, clientTimeout }, { cause: error }); + } + throw error; + } return result.value as Awaited; } diff --git a/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts b/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts new file mode 100644 index 000000000..fb3b7ea7d --- /dev/null +++ b/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts @@ -0,0 +1,149 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { JSONRPCMessage } from "@browserbasehq/stagehand-protocol/json-rpc/types"; +import { StagehandMethods } from "@browserbasehq/stagehand-protocol/schema-registry"; +import { MAX_CALLBACK_BATCH_TIMEOUT_MS } from "@browserbasehq/stagehand-protocol/schemas"; +import { + BrowserContext, + CALLBACK_BATCH_CLIENT_GRACE_MS, + Stagehand, + StagehandBatchTimeoutError, +} from "../src/index.js"; +import { RPCClient, RPCResponseTimeoutError, type CDPTransport } from "../src/rpcClient.js"; +import { + attachStagehandBrowserContext, + claimStagehandBrowserHandle, + createStagehandBrowserHandle, +} from "../src/browser/index.js"; + +/** A transport that accepts requests and never answers them. */ +class SilentCDPTransport implements CDPTransport { + readonly serviceWorker = { + targetId: "worker-target", + url: "chrome-extension://stagehand/service-worker.js", + title: "Stagehand", + extensionId: "stagehand", + }; + onmessage?: (message: unknown) => void | Promise; + onclose?: (reason?: Error) => void; + onerror?: (error: Error) => void; + readonly sent: JSONRPCMessage[] = []; + + async send(message: JSONRPCMessage): Promise { + this.sent.push(message); + } + + close(): void {} +} + +function createStagehand(client: RPCClient): Stagehand { + const browser = createStagehandBrowserHandle({ + provider: "local", + origin: "connected", + attachment: {}, + close: () => {}, + }); + claimStagehandBrowserHandle(browser); + attachStagehandBrowserContext(browser, new BrowserContext(client, () => browser.close())); + const stagehand = Object.create(Stagehand.prototype) as Stagehand; + Object.assign(stagehand, { browserHandle: browser }); + stagehand.rpcClient = client; + stagehand.isInitialized = true; + return stagehand; +} + +describe("experimentalBatch client deadline", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("rejects with a typed error when the executor never answers", async () => { + vi.useFakeTimers(); + const transport = new SilentCDPTransport(); + const client = new RPCClient(transport); + const stagehand = createStagehand(client); + + try { + const pending = stagehand.experimentalBatch(async () => "never", undefined, { + timeout: 60_000, + }); + const rejection = expect(pending).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(StagehandBatchTimeoutError); + const typed = error as StagehandBatchTimeoutError; + expect(typed.timeout).toBe(60_000); + expect(typed.clientTimeout).toBe(60_000 + CALLBACK_BATCH_CLIENT_GRACE_MS); + expect(typed.cause).toBeInstanceOf(RPCResponseTimeoutError); + return true; + }); + + await vi.advanceTimersByTimeAsync(60_000 + CALLBACK_BATCH_CLIENT_GRACE_MS - 1); + expect(client.pending.size).toBe(1); + await vi.advanceTimersByTimeAsync(1); + await rejection; + expect(client.pending.size).toBe(0); + expect(transport.sent).toHaveLength(1); + expect((transport.sent[0] as { method: string }).method).toBe( + StagehandMethods.stagehandCallbackBatch.name, + ); + } finally { + client.close(); + } + }); + + it("lets callers shorten the round-trip deadline below the executor timeout", async () => { + vi.useFakeTimers(); + const client = new RPCClient(new SilentCDPTransport()); + const stagehand = createStagehand(client); + + try { + const pending = stagehand.experimentalBatch(async () => "never", undefined, { + timeout: 60_000, + clientTimeoutMs: 5_000, + }); + const rejection = expect(pending).rejects.toMatchObject({ + name: "StagehandBatchTimeoutError", + timeout: 60_000, + clientTimeout: 5_000, + }); + await vi.advanceTimersByTimeAsync(5_000); + await rejection; + } finally { + client.close(); + } + }); + + it("keeps the maximum executor timeout within the timer limit", async () => { + vi.useFakeTimers(); + const client = new RPCClient(new SilentCDPTransport()); + const stagehand = createStagehand(client); + + try { + const pending = stagehand.experimentalBatch(async () => "never", undefined, { + timeout: MAX_CALLBACK_BATCH_TIMEOUT_MS, + }); + const rejection = expect(pending).rejects.toMatchObject({ clientTimeout: 2_147_483_647 }); + // A delay above the limit would have fired immediately; the request must still be pending. + await vi.advanceTimersByTimeAsync(1_000); + expect(client.pending.size).toBe(1); + await vi.advanceTimersByTimeAsync(2_147_483_647); + await rejection; + } finally { + client.close(); + } + }); + + it.each([0, -1, 0.5, 2_147_483_648, NaN, Infinity])( + "validates clientTimeoutMs %s before sending", + async (clientTimeoutMs) => { + const client = new RPCClient(new SilentCDPTransport()); + const stagehand = createStagehand(client); + try { + await expect( + stagehand.experimentalBatch(async () => undefined, undefined, { clientTimeoutMs }), + ).rejects.toThrow(RangeError); + expect(client.pending.size).toBe(0); + } finally { + client.close(); + } + }, + ); +}); From 41af704a18b5896062d6fc08cdffb541e306e929 Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 16:59:03 -0700 Subject: [PATCH 2/5] style(core): format capture regression imports --- packages/integrations/core/tests/facade-tools.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/integrations/core/tests/facade-tools.test.ts b/packages/integrations/core/tests/facade-tools.test.ts index 7b1445853..cd1e0545a 100644 --- a/packages/integrations/core/tests/facade-tools.test.ts +++ b/packages/integrations/core/tests/facade-tools.test.ts @@ -7,7 +7,11 @@ import { BROWSER_SESSION_LOST_ERROR_PREFIX, type FacadeSessionLoss, } from "../src/facade/contract.js"; -import { StagehandFacadeSessionLostError, StagehandFacadeTools, type StagehandFacadeRunReport } from "../src/facade/tools.js"; +import { + StagehandFacadeSessionLostError, + StagehandFacadeTools, + type StagehandFacadeRunReport, +} from "../src/facade/tools.js"; type FakePage = ReturnType; From e043e1efbfe5aa1f9472768f521234cda86cc988 Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 17:12:28 -0700 Subject: [PATCH 3/5] fix(runtime): preserve terminal diagnostics and public error boundaries --- packages/evals/core/contracts/tool.ts | 5 ++ .../evals/core/tools/browserSessionLoss.ts | 16 ++++ .../core/browserSessionLossTelemetry.test.ts | 70 +++++++++++++++ .../core/src/facade/screenshot-transport.ts | 3 +- .../core/src/facade/stdio-server.ts | 6 +- .../integrations/core/src/facade/tools.ts | 13 ++- .../tests/facade-screenshot-transport.test.ts | 19 ++++ .../tests/facade-stdio-session-age.test.ts | 86 +++++++++++++++++++ .../core/tests/facade-tools.test.ts | 39 +++++++++ packages/sdk-ts/src/index.ts | 2 +- packages/sdk-ts/src/rpcClient.ts | 13 +-- packages/sdk-ts/src/rpcErrors.ts | 13 +++ packages/sdk-ts/src/stagehand.ts | 3 +- .../tests/experimentalBatchDeadline.test.ts | 8 +- 14 files changed, 275 insertions(+), 21 deletions(-) create mode 100644 packages/evals/tests/core/browserSessionLossTelemetry.test.ts create mode 100644 packages/integrations/core/tests/facade-stdio-session-age.test.ts create mode 100644 packages/sdk-ts/src/rpcErrors.ts diff --git a/packages/evals/core/contracts/tool.ts b/packages/evals/core/contracts/tool.ts index e6dbdc0f5..15a8b0c76 100644 --- a/packages/evals/core/contracts/tool.ts +++ b/packages/evals/core/contracts/tool.ts @@ -140,6 +140,11 @@ export interface BrowserSessionLoss { cause: string; tool?: string; at?: string; + provider?: "local" | "browserbase"; + sessionId?: string; + /** Elapsed time since the facade started browser launch, including initialization. */ + sessionAgeMs?: number; + sessionTimeoutMs?: number; } /** MCP content returned unchanged by a runner call into its existing surface. */ diff --git a/packages/evals/core/tools/browserSessionLoss.ts b/packages/evals/core/tools/browserSessionLoss.ts index 4d8504a68..c84e4b42a 100644 --- a/packages/evals/core/tools/browserSessionLoss.ts +++ b/packages/evals/core/tools/browserSessionLoss.ts @@ -33,6 +33,22 @@ export function parseSessionLossTelemetry(line: string): BrowserSessionLoss | un cause: sanitizeErrorMessage(parsed.cause), ...(typeof parsed.tool === "string" && { tool: parsed.tool }), ...(typeof parsed.at === "string" && { at: parsed.at }), + ...((parsed.provider === "local" || parsed.provider === "browserbase") && { + provider: parsed.provider, + }), + ...(typeof parsed.sessionId === "string" && { + sessionId: sanitizeErrorMessage(parsed.sessionId), + }), + ...(typeof parsed.sessionAgeMs === "number" && + Number.isFinite(parsed.sessionAgeMs) && + parsed.sessionAgeMs >= 0 && { + sessionAgeMs: parsed.sessionAgeMs, + }), + ...(typeof parsed.sessionTimeoutMs === "number" && + Number.isFinite(parsed.sessionTimeoutMs) && + parsed.sessionTimeoutMs >= 0 && { + sessionTimeoutMs: parsed.sessionTimeoutMs, + }), }; } catch { return undefined; diff --git a/packages/evals/tests/core/browserSessionLossTelemetry.test.ts b/packages/evals/tests/core/browserSessionLossTelemetry.test.ts new file mode 100644 index 000000000..6f36ec71c --- /dev/null +++ b/packages/evals/tests/core/browserSessionLossTelemetry.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { + parseSessionLossTelemetry, + SESSION_LOST_TELEMETRY_PREFIX, +} from "../../core/tools/browserSessionLoss.js"; + +function line(fields: Record) { + return ( + SESSION_LOST_TELEMETRY_PREFIX + JSON.stringify({ cause: "CDP connection closed", ...fields }) + ); +} + +describe("facade session loss diagnostics", () => { + it("retains browser identity, measured age and configured timeout", () => { + expect( + parseSessionLossTelemetry( + line({ + tool: "snapshot", + at: "2026-09-08T00:00:00.000Z", + provider: "browserbase", + sessionId: "session-123", + sessionAgeMs: 12_500, + sessionTimeoutMs: 3_600_000, + }), + ), + ).toEqual({ + cause: "CDP connection closed", + tool: "snapshot", + at: "2026-09-08T00:00:00.000Z", + provider: "browserbase", + sessionId: "session-123", + sessionAgeMs: 12_500, + sessionTimeoutMs: 3_600_000, + }); + }); + + it.each([-1, "12000", null, {}, 1e309])("drops invalid diagnostic durations: %j", (value) => { + expect( + parseSessionLossTelemetry(line({ sessionAgeMs: value, sessionTimeoutMs: value })), + ).toEqual({ cause: "CDP connection closed" }); + }); + + it("accepts zero age and omits invalid identity metadata", () => { + expect( + parseSessionLossTelemetry(line({ provider: "other", sessionId: 42, sessionAgeMs: 0 })), + ).toEqual({ cause: "CDP connection closed", sessionAgeMs: 0 }); + }); + + it("drops JSON numeric overflow without losing the terminal cause", () => { + expect( + parseSessionLossTelemetry( + SESSION_LOST_TELEMETRY_PREFIX + + '{"cause":"CDP connection closed","sessionAgeMs":1e309,"sessionTimeoutMs":1e309}', + ), + ).toEqual({ cause: "CDP connection closed" }); + }); + + it("sanitizes string diagnostics while retaining valid numeric metadata", () => { + const parsed = parseSessionLossTelemetry( + line({ + provider: "local", + sessionId: "wss://example.test/?apiKey=synthetic-key", + sessionAgeMs: 1, + }), + ); + expect(parsed?.provider).toBe("local"); + expect(parsed?.sessionAgeMs).toBe(1); + expect(JSON.stringify(parsed)).not.toContain("synthetic-key"); + }); +}); diff --git a/packages/integrations/core/src/facade/screenshot-transport.ts b/packages/integrations/core/src/facade/screenshot-transport.ts index 91c1e0e37..75aff9722 100644 --- a/packages/integrations/core/src/facade/screenshot-transport.ts +++ b/packages/integrations/core/src/facade/screenshot-transport.ts @@ -45,9 +45,10 @@ export async function captureScreenshotWithinBase64Budget( const attempts = screenshotAttempts(requested); for (const [index, options] of attempts.entries()) { const image = await capture(options); + if (Buffer.byteLength(image.data, "utf8") > maxBase64Bytes) continue; const size = imageDimensions(image); const tooLarge = size !== undefined && Math.max(size.width, size.height) > maxSidePx; - if (!tooLarge && Buffer.byteLength(image.data, "utf8") <= maxBase64Bytes) { + if (!tooLarge) { return { image, options, adjusted: index > 0 || !sameOptions(options, requested) }; } } diff --git a/packages/integrations/core/src/facade/stdio-server.ts b/packages/integrations/core/src/facade/stdio-server.ts index 8cf2fac58..ca8a20613 100644 --- a/packages/integrations/core/src/facade/stdio-server.ts +++ b/packages/integrations/core/src/facade/stdio-server.ts @@ -133,11 +133,11 @@ async function ensureResources(): Promise { async function createResources(): Promise { const config = stagehandFacadeConfigFromEnv(); + const launchedAt = Date.now(); const browser = config.browser.type === "browserbase" ? await browserbase.launch(config.browser.launchOptions) : await localBrowser.launch(config.browser.launchOptions); - const launchedAt = Date.now(); try { const stagehand = await Stagehand.create({ browser, ...config.stagehand }); const tools = new StagehandFacadeTools(stagehand, { @@ -146,8 +146,8 @@ async function createResources(): Promise { // The browser is not recreated on purpose: a fresh session would silently // change the evidence trail mid-task. Tools keep answering with the // terminal error and the host decides what to do with the run. - // sessionAgeMs against the configured session timeout tells a Browserbase - // TIMED_OUT apart from a remote close. + // Age includes launch and initialization time. Compare it with configured + // timeout and remote session status when diagnosing a disconnect. onSessionLost: (loss) => process.stderr.write( `${SESSION_LOST_TELEMETRY_PREFIX}${JSON.stringify({ diff --git a/packages/integrations/core/src/facade/tools.ts b/packages/integrations/core/src/facade/tools.ts index 2a090b43f..c9bd12d7c 100644 --- a/packages/integrations/core/src/facade/tools.ts +++ b/packages/integrations/core/src/facade/tools.ts @@ -382,13 +382,13 @@ export class StagehandFacadeTools { } const cause = `executor unresponsive: ${this.consecutiveDeadlines} consecutive capture timeouts (last: ${error.message})`; this.loss = { cause, tool, at: new Date().toISOString() }; - this.options.onSessionLost?.(this.loss); + this.notifySessionLost(this.loss); throw new StagehandFacadeSessionLostError(this.loss); } const cause = sessionLossCause(error); if (cause === undefined) throw error; this.loss = { cause, tool, at: new Date().toISOString() }; - this.options.onSessionLost?.(this.loss); + this.notifySessionLost(this.loss); throw new StagehandFacadeSessionLostError(this.loss); } }; @@ -399,6 +399,15 @@ export class StagehandFacadeTools { ); return result; } + + private notifySessionLost(loss: FacadeSessionLoss): void { + // Diagnostic observers cannot replace the terminal error or reopen the queue. + try { + void Promise.resolve(this.options.onSessionLost?.(loss)).catch(() => undefined); + } catch { + // Preserve the first browser failure when an observer throws synchronously. + } + } } class FacadeDeadlineError extends Error { diff --git a/packages/integrations/core/tests/facade-screenshot-transport.test.ts b/packages/integrations/core/tests/facade-screenshot-transport.test.ts index f35a69cbd..9e4d1a2f2 100644 --- a/packages/integrations/core/tests/facade-screenshot-transport.test.ts +++ b/packages/integrations/core/tests/facade-screenshot-transport.test.ts @@ -95,6 +95,25 @@ describe("facade screenshot transport", () => { ); expect(capture).toHaveBeenCalledTimes(3); }); + + it("rejects over-budget candidates before decoding their image data", async () => { + const oversized = "a".repeat(90_000); + const bounded = jpeg(640, 480); + const capture = vi + .fn() + .mockResolvedValueOnce({ data: oversized, mimeType: "image/png" }) + .mockResolvedValueOnce({ data: bounded, mimeType: "image/jpeg" }); + const decode = vi.spyOn(Buffer, "from"); + try { + await expect( + captureScreenshotWithinBase64Budget(capture, { type: "png" }, 60_000), + ).resolves.toMatchObject({ image: { data: bounded }, adjusted: true }); + expect(decode.mock.calls.filter(([data]) => data === oversized)).toHaveLength(0); + expect(decode).toHaveBeenCalledWith(bounded, "base64"); + } finally { + decode.mockRestore(); + } + }); }); function png(width: number, height: number): string { diff --git a/packages/integrations/core/tests/facade-stdio-session-age.test.ts b/packages/integrations/core/tests/facade-stdio-session-age.test.ts new file mode 100644 index 000000000..d4a2a3f8d --- /dev/null +++ b/packages/integrations/core/tests/facade-stdio-session-age.test.ts @@ -0,0 +1,86 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { SESSION_LOST_TELEMETRY_PREFIX, type FacadeSessionLoss } from "../src/facade/contract.js"; + +const mocks = vi.hoisted(() => ({ + launch: vi.fn(), + create: vi.fn(), + setRequestHandler: vi.fn(), + onSessionLost: undefined as ((loss: FacadeSessionLoss) => void) | undefined, +})); + +vi.mock("@browserbasehq/stagehand", () => ({ + browserbase: { launch: mocks.launch }, + localBrowser: { launch: mocks.launch }, + Stagehand: { create: mocks.create }, +})); +vi.mock("@modelcontextprotocol/sdk/server/mcp.js", () => ({ + McpServer: class { + server = { removeRequestHandler: vi.fn(), setRequestHandler: mocks.setRequestHandler }; + registerTool = vi.fn(); + connect = vi.fn(async () => undefined); + }, +})); +vi.mock("@modelcontextprotocol/sdk/server/stdio.js", () => ({ StdioServerTransport: class {} })); +vi.mock("../src/facade/config.js", () => ({ + stagehandFacadeConfigFromEnv: () => ({ + browser: { type: "browserbase", launchOptions: { timeout: 3600 } }, + stagehand: {}, + }), +})); +vi.mock("../src/facade/tools.js", () => ({ + StagehandFacadeTools: class { + constructor( + _stagehand: unknown, + options: { onSessionLost: (loss: FacadeSessionLoss) => void }, + ) { + mocks.onSessionLost = options.onSessionLost; + } + async snapshot() { + vi.setSystemTime(15_000); + mocks.onSessionLost?.({ + cause: "CDP connection closed", + tool: "snapshot", + at: new Date().toISOString(), + }); + return "captured"; + } + }, +})); + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + vi.resetModules(); +}); + +it("includes browser launch and Stagehand initialization in measured session age", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(2_000); + const browser = { provider: "browserbase", sessionId: "session-123", close: vi.fn() }; + mocks.launch.mockImplementation(async () => { + vi.setSystemTime(7_000); + return browser; + }); + mocks.create.mockImplementation(async () => { + vi.setSystemTime(11_000); + return {}; + }); + const output = vi.spyOn(process.stderr, "write").mockReturnValue(true); + vi.spyOn(process, "once").mockReturnValue(process); + vi.spyOn(process.stdin, "once").mockReturnValue(process.stdin); + await import("../src/facade/stdio-server.js"); + const handler = mocks.setRequestHandler.mock.calls.at(-1)?.[1] as (request: { + params: { name: string; arguments: object }; + }) => Promise; + await handler({ params: { name: "snapshot", arguments: {} } }); + const telemetry = output.mock.calls + .map(([chunk]) => String(chunk)) + .find((line) => line.startsWith(SESSION_LOST_TELEMETRY_PREFIX)); + expect(telemetry).toBeDefined(); + expect(JSON.parse(telemetry!.slice(SESSION_LOST_TELEMETRY_PREFIX.length))).toMatchObject({ + provider: "browserbase", + sessionId: "session-123", + sessionAgeMs: 13_000, + sessionTimeoutMs: 3_600_000, + }); +}); diff --git a/packages/integrations/core/tests/facade-tools.test.ts b/packages/integrations/core/tests/facade-tools.test.ts index cd1e0545a..5e823c667 100644 --- a/packages/integrations/core/tests/facade-tools.test.ts +++ b/packages/integrations/core/tests/facade-tools.test.ts @@ -539,6 +539,45 @@ describe("StagehandFacadeTools session loss", () => { expect(page.screenshot).not.toHaveBeenCalled(); }); + it.each([ + ["transport", "throws"], + ["transport", "rejects"], + ["capture deadlines", "throws"], + ["capture deadlines", "rejects"], + ])("preserves terminal %s loss when its observer %s", async (failure, observer) => { + vi.useFakeTimers(); + const page = createFakePage(); + const { stagehand, experimentalBatch, context } = createFakeStagehand(page); + const onSessionLost = vi.fn(() => { + const error = new Error("diagnostic observer failed"); + if (observer === "throws") throw error; + return Promise.reject(error); + }); + const tools = new StagehandFacadeTools(stagehand, { onSessionLost }); + if (failure === "transport") { + experimentalBatch.mockRejectedValueOnce(batchTimeoutError()); + } else { + page.snapshot.mockImplementation(() => new Promise(() => undefined)); + for (let count = 0; count < 2; count++) { + const rejection = expect(tools.snapshot()).rejects.toThrow( + "page.snapshot received no response", + ); + await vi.advanceTimersByTimeAsync(120_000); + await rejection; + } + } + const pending = failure === "transport" ? tools.run("return 1;") : tools.snapshot(); + const outcome = pending.catch((error: unknown) => error); + if (failure !== "transport") await vi.advanceTimersByTimeAsync(120_000); + expect(await outcome).toBeInstanceOf(StagehandFacadeSessionLostError); + expect(tools.sessionLoss).toBeDefined(); + const calls = context.activePage.mock.calls.length; + await expect(tools.screenshot()).rejects.toBeInstanceOf(StagehandFacadeSessionLostError); + expect(context.activePage.mock.calls.length).toBe(calls); + expect(page.screenshot).not.toHaveBeenCalled(); + expect(onSessionLost).toHaveBeenCalledOnce(); + }); + it("does not replay successful actions when a later action fails", async () => { const world = createFakeWorld(); world.snapshot = { formattedTree: "actions", xpathMap: { "0-1": "/first", "0-2": "/second" } }; diff --git a/packages/sdk-ts/src/index.ts b/packages/sdk-ts/src/index.ts index 8c699ee11..b39105832 100644 --- a/packages/sdk-ts/src/index.ts +++ b/packages/sdk-ts/src/index.ts @@ -39,7 +39,7 @@ export { } from "./response.js"; export { WebMCPInvocation, WebMCPTool } from "./webmcp.js"; export { CDPConnectionClosedError } from "./cdpClient.js"; -export { RPCResponseTimeoutError } from "./rpcClient.js"; +export { RPCResponseTimeoutError } from "./rpcErrors.js"; export type { InitScriptSource } from "./pageScripts.js"; export { Stagehand, type ExtractResult } from "./stagehand.js"; export { diff --git a/packages/sdk-ts/src/rpcClient.ts b/packages/sdk-ts/src/rpcClient.ts index 9e08c8a60..df557a153 100644 --- a/packages/sdk-ts/src/rpcClient.ts +++ b/packages/sdk-ts/src/rpcClient.ts @@ -39,6 +39,7 @@ import type { StagehandRpcNotification } from "@browserbasehq/stagehand-protocol import { z } from "zod/v4"; import { CDPClient, type ServiceWorkerInfo } from "./cdpClient.js"; import { abortReason } from "./abort.js"; +import { RPCResponseTimeoutError } from "./rpcErrors.js"; type PendingRequest = { method: RPCMethod; @@ -58,18 +59,6 @@ type RPCSendOptions = { responseTimeoutMs?: number; }; -export class RPCResponseTimeoutError extends Error { - readonly method: string; - readonly timeoutMs: number; - - constructor(method: string, timeoutMs: number) { - super(`RPC response timed out: ${method}`, { cause: { method, timeoutMs } }); - this.name = "RPCResponseTimeoutError"; - this.method = method; - this.timeoutMs = timeoutMs; - } -} - const TRACER = trace.getTracer("@browserbasehq/stagehand"); const W3C_TRACE_CONTEXT_PROPAGATOR = new W3CTraceContextPropagator(); const MAX_PENDING_NOTIFICATIONS = 100; diff --git a/packages/sdk-ts/src/rpcErrors.ts b/packages/sdk-ts/src/rpcErrors.ts new file mode 100644 index 000000000..c427ed606 --- /dev/null +++ b/packages/sdk-ts/src/rpcErrors.ts @@ -0,0 +1,13 @@ +export class RPCResponseTimeoutError extends Error { + readonly method: string; + readonly timeoutMs: number; + + constructor(method: string, timeoutMs: number) { + super(`RPC response timed out: ${method} after ${timeoutMs}ms`, { + cause: { method, timeoutMs }, + }); + this.name = "RPCResponseTimeoutError"; + this.method = method; + this.timeoutMs = timeoutMs; + } +} diff --git a/packages/sdk-ts/src/stagehand.ts b/packages/sdk-ts/src/stagehand.ts index c008f8221..d19f73fa0 100644 --- a/packages/sdk-ts/src/stagehand.ts +++ b/packages/sdk-ts/src/stagehand.ts @@ -1,4 +1,5 @@ -import { RPCClient, RPCResponseTimeoutError } from "./rpcClient.js"; +import { RPCClient } from "./rpcClient.js"; +import { RPCResponseTimeoutError } from "./rpcErrors.js"; import { DefaultExtractDataSchema, MAX_CALLBACK_BATCH_TIMEOUT_MS, diff --git a/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts b/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts index fb3b7ea7d..ded13be10 100644 --- a/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts +++ b/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts @@ -5,10 +5,11 @@ import { MAX_CALLBACK_BATCH_TIMEOUT_MS } from "@browserbasehq/stagehand-protocol import { BrowserContext, CALLBACK_BATCH_CLIENT_GRACE_MS, + RPCResponseTimeoutError, Stagehand, StagehandBatchTimeoutError, } from "../src/index.js"; -import { RPCClient, RPCResponseTimeoutError, type CDPTransport } from "../src/rpcClient.js"; +import { RPCClient, type CDPTransport } from "../src/rpcClient.js"; import { attachStagehandBrowserContext, claimStagehandBrowserHandle, @@ -72,6 +73,11 @@ describe("experimentalBatch client deadline", () => { expect(typed.timeout).toBe(60_000); expect(typed.clientTimeout).toBe(60_000 + CALLBACK_BATCH_CLIENT_GRACE_MS); expect(typed.cause).toBeInstanceOf(RPCResponseTimeoutError); + expect(typed.cause).toMatchObject({ + message: `RPC response timed out: ${StagehandMethods.stagehandCallbackBatch.name} after 75000ms`, + method: StagehandMethods.stagehandCallbackBatch.name, + timeoutMs: 75_000, + }); return true; }); From 90721ea28e9d9613b06aee8428783b704e4fc7e9 Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 18:19:14 -0700 Subject: [PATCH 4/5] fix(evals): defer runtime loading for general CLI help --- packages/evals/tests/cli.test.ts | 48 ++++++++++++-------- packages/evals/tests/tui/helpImports.test.ts | 35 ++++++++++++++ packages/evals/tui/commands/config.ts | 4 +- packages/evals/tui/commands/core.ts | 4 +- packages/evals/tui/commands/help.ts | 12 +++-- 5 files changed, 75 insertions(+), 28 deletions(-) create mode 100644 packages/evals/tests/tui/helpImports.test.ts diff --git a/packages/evals/tests/cli.test.ts b/packages/evals/tests/cli.test.ts index 7aca4500c..fb106d7bb 100644 --- a/packages/evals/tests/cli.test.ts +++ b/packages/evals/tests/cli.test.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll, onTestFinished } from "vitest"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import fs from "node:fs"; @@ -9,6 +9,7 @@ const exec = promisify(execFile); const repoRoot = path.resolve(__dirname, "..", "..", ".."); const CLI_PATH = path.join(repoRoot, "packages", "evals", "cli.ts"); const SOURCE_CONFIG = path.join(repoRoot, "packages", "evals", "evals.config.json"); +const CLI_CHILD_TIMEOUT_MS = 15_000; // File-level snapshot/restore: any `evals run …` invocation through the // real CLI writes `_meta.firstRunCompletedAt` into the source config @@ -24,15 +25,20 @@ afterAll(() => { async function runCli(args: string[]): Promise<{ stdout: string; stderr: string; code: number }> { try { - const { stdout, stderr } = await exec( - process.execPath, - ["--import", "tsx", CLI_PATH, ...args], - { - cwd: repoRoot, - timeout: 15_000, - env: { ...process.env, NODE_NO_WARNINGS: "1" }, - }, - ); + const execution = exec(process.execPath, ["--import", "tsx", CLI_PATH, ...args], { + cwd: repoRoot, + timeout: CLI_CHILD_TIMEOUT_MS, + killSignal: "SIGKILL", + env: { ...process.env, NODE_NO_WARNINGS: "1" }, + }); + // A test timeout must also stop its own CLI child. SIGTERM enters the + // CLI's async cleanup path, which is not a bounded subprocess deadline. + onTestFinished(() => { + if (execution.child.exitCode === null && execution.child.signalCode === null) { + execution.child.kill("SIGKILL"); + } + }); + const { stdout, stderr } = await execution; return { stdout, stderr, code: 0 }; } catch (err: any) { return { @@ -55,15 +61,19 @@ function readSourceWelcomeCompletedAt(): string | undefined { } describe("CLI entrypoint", () => { - it("shows help", async () => { - const { stdout, code } = await runCli(["-h"]); - expect(code).toBe(0); - expect(stdout).toContain("Commands:"); - expect(stdout).toContain("run"); - expect(stdout).toContain("list"); - expect(stdout).toContain("config"); - expect(stdout).toContain("experiments"); - }); + it( + "shows help", + async () => { + const { stdout, code } = await runCli(["-h"]); + expect(code).toBe(0); + expect(stdout).toContain("Commands:"); + expect(stdout).toContain("run"); + expect(stdout).toContain("list"); + expect(stdout).toContain("config"); + expect(stdout).toContain("experiments"); + }, + CLI_CHILD_TIMEOUT_MS + 2_000, + ); it("shows experiments overview help", async () => { const { stdout, code } = await runCli(["experiments"]); diff --git a/packages/evals/tests/tui/helpImports.test.ts b/packages/evals/tests/tui/helpImports.test.ts new file mode 100644 index 000000000..5b0475943 --- /dev/null +++ b/packages/evals/tests/tui/helpImports.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildCommandTree, dispatch } from "../../tui/commandTree.js"; + +vi.mock("../../framework/benchHarness.js", () => { + throw new Error("Help must not initialize the harness runtime"); +}); +vi.mock("../../core/tools/registry.js", () => { + throw new Error("Help must not initialize the tool runtime"); +}); + +afterEach(() => vi.restoreAllMocks()); + +describe("help without runtime imports", () => { + it.each([ + { args: ["--help"], expected: "Commands:" }, + { args: ["list", "--help"], expected: "evals list" }, + { args: ["new", "--help"], expected: "evals new" }, + { args: ["experiments", "--help"], expected: "evals experiments" }, + { args: ["config", "tracing", "--help"], expected: "evals config tracing" }, + ])("prints $args with harness and tool modules unavailable", async ({ args, expected }) => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const getRegistry = vi.fn(async () => { + throw new Error("Help must not discover tasks"); + }); + await dispatch(buildCommandTree(), args, { + entryDir: "/unused", + getRegistry, + setRegistry: vi.fn(), + abortRef: null, + contextPath: null, + }); + expect(log.mock.calls.flat().join("\n")).toContain(expected); + expect(getRegistry).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/evals/tui/commands/config.ts b/packages/evals/tui/commands/config.ts index 8a2420f7a..9fd9eee0d 100644 --- a/packages/evals/tui/commands/config.ts +++ b/packages/evals/tui/commands/config.ts @@ -177,7 +177,7 @@ export async function handleConfig(args: string[], entryDir: string): Promise { + const { listBenchHarnesses, listBenchHarnessesForTaskKind } = + await import("../../framework/benchHarness.js"); const suiteHarness = listBenchHarnessesForTaskKind("suite")[0]; print([ "", @@ -156,7 +156,8 @@ export function printNewHelp(): void { ]); } -export function printConfigHelp(): void { +export async function printConfigHelp(): Promise { + const { listCoreRunnableTools } = await import("../../core/tools/registry.js"); print([ "", ` ${dustyCyanHeader("evals config")} ${dim("[subcommand]")}`, @@ -197,7 +198,8 @@ export function printConfigHelp(): void { ]); } -export function printConfigCoreHelp(): void { +export async function printConfigCoreHelp(): Promise { + const { listCoreRunnableTools } = await import("../../core/tools/registry.js"); print([ "", ` ${dustyCyanHeader("evals config core")} ${dim("[subcommand]")}`, From 5671f3c9d0167b31bf586cc1e5c2a8980e891853 Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 14 Sep 2026 14:45:08 -0700 Subject: [PATCH 5/5] Refresh CI after syncing the reviewed parent