From 5c9e7205a8376f20139898d9b81d8c9c9904affa Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 16:49:19 -0700 Subject: [PATCH 01/35] Recover bounded capture deadlines and preserve terminal session loss --- .changeset/eval-capture-deadlines.md | 5 + .../integrations/core/src/facade/index.ts | 1 + .../core/src/facade/screenshot-transport.ts | 44 ++- .../core/src/facade/stdio-server.ts | 21 ++ .../integrations/core/src/facade/tools.ts | 123 ++++++-- .../tests/facade-screenshot-transport.test.ts | 66 +++++ .../core/tests/facade-tools.test.ts | 263 +++++++++++++++++- packages/sdk-ts/src/batch.ts | 32 ++- packages/sdk-ts/src/cdpClient.ts | 13 +- packages/sdk-ts/src/index.ts | 16 +- packages/sdk-ts/src/rpcClient.ts | 21 +- packages/sdk-ts/src/stagehand.ts | 49 +++- .../tests/experimentalBatchDeadline.test.ts | 149 ++++++++++ 13 files changed, 756 insertions(+), 47 deletions(-) create mode 100644 .changeset/eval-capture-deadlines.md create mode 100644 packages/sdk-ts/tests/experimentalBatchDeadline.test.ts diff --git a/.changeset/eval-capture-deadlines.md b/.changeset/eval-capture-deadlines.md new file mode 100644 index 000000000..94edd0cd9 --- /dev/null +++ b/.changeset/eval-capture-deadlines.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Bound experimental batch and RPC deadlines so callers can stop waiting without replaying actions or accepting late capture state. diff --git a/packages/integrations/core/src/facade/index.ts b/packages/integrations/core/src/facade/index.ts index c78f8df1f..9a9271d8b 100644 --- a/packages/integrations/core/src/facade/index.ts +++ b/packages/integrations/core/src/facade/index.ts @@ -37,6 +37,7 @@ export { } from "./contract.js"; export { StagehandFacadeTools, + StagehandFacadeSessionLostError, type StagehandFacadeRunReport, type StagehandFacadeToolsOptions, } from "./tools.js"; diff --git a/packages/integrations/core/src/facade/screenshot-transport.ts b/packages/integrations/core/src/facade/screenshot-transport.ts index 065bff1db..91c1e0e37 100644 --- a/packages/integrations/core/src/facade/screenshot-transport.ts +++ b/packages/integrations/core/src/facade/screenshot-transport.ts @@ -27,24 +27,64 @@ export function screenshotBase64BudgetFromArgs(args: string[]): number | undefin return budget; } +/** + * Model APIs reject images with a side longer than this. Anthropic allows + * 8000 px for a lone image but only 2000 px once a request carries many + * images, which every multi-step agent conversation does. Full-page captures + * of long pages exceed both and killed whole runs with a 400, so oversized + * captures fall back to the viewport like over-budget ones do. + */ +export const MAX_SCREENSHOT_SIDE_PX = 2000; + export async function captureScreenshotWithinBase64Budget( capture: CaptureScreenshot, requested: ScreenshotOptions, maxBase64Bytes: number, + maxSidePx = MAX_SCREENSHOT_SIDE_PX, ): Promise { const attempts = screenshotAttempts(requested); for (const [index, options] of attempts.entries()) { const image = await capture(options); - if (Buffer.byteLength(image.data, "utf8") <= maxBase64Bytes) { + const size = imageDimensions(image); + const tooLarge = size !== undefined && Math.max(size.width, size.height) > maxSidePx; + if (!tooLarge && Buffer.byteLength(image.data, "utf8") <= maxBase64Bytes) { return { image, options, adjusted: index > 0 || !sameOptions(options, requested) }; } } throw new Error( - `Screenshot exceeds the ${maxBase64Bytes}-byte MCP transport budget after compressed viewport retries.`, + `Screenshot exceeds the ${maxBase64Bytes}-byte MCP transport budget or the ${maxSidePx}px side limit after compressed viewport retries.`, ); } +/** Reads width/height from a PNG or JPEG header; undefined when unparseable. */ +export function imageDimensions( + image: StagehandFacadeScreenshot, +): { width: number; height: number } | undefined { + const bytes = Buffer.from(image.data, "base64"); + if (image.mimeType === "image/png") { + if (bytes.length < 24 || bytes.toString("ascii", 1, 4) !== "PNG") return undefined; + return { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) }; + } + // JPEG: walk the marker segments to the first SOFn (C0–CF except C4, C8, CC). + if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return undefined; + let offset = 2; + while (offset + 9 <= bytes.length) { + if (bytes[offset] !== 0xff) return undefined; + const marker = bytes[offset + 1]!; + if (marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd7) || marker === 0x01) { + offset += 2; + continue; + } + const length = bytes.readUInt16BE(offset + 2); + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { + return { height: bytes.readUInt16BE(offset + 5), width: bytes.readUInt16BE(offset + 7) }; + } + offset += 2 + length; + } + return undefined; +} + function screenshotAttempts(requested: ScreenshotOptions): ScreenshotOptions[] { const initial: ScreenshotOptions = { fullPage: requested.fullPage ?? false, diff --git a/packages/integrations/core/src/facade/stdio-server.ts b/packages/integrations/core/src/facade/stdio-server.ts index 167fa56d2..8cf2fac58 100644 --- a/packages/integrations/core/src/facade/stdio-server.ts +++ b/packages/integrations/core/src/facade/stdio-server.ts @@ -17,6 +17,7 @@ import { facadeSurfaceFromArgs, facadeToolsFor, SESSION_INFO_TOOL_NAME, + SESSION_LOST_TELEMETRY_PREFIX, ScreenshotInputSchema, SnapshotInputSchema, } from "./contract.js"; @@ -136,11 +137,31 @@ async function createResources(): Promise { config.browser.type === "browserbase" ? await browserbase.launch(config.browser.launchOptions) : await localBrowser.launch(config.browser.launchOptions); + const launchedAt = Date.now(); try { const stagehand = await Stagehand.create({ browser, ...config.stagehand }); const tools = new StagehandFacadeTools(stagehand, { onRunReport: (report) => process.stderr.write(`stagehand_playwright_compat ${JSON.stringify(report)}\n`), + // The browser is not recreated on purpose: a fresh session would silently + // change the evidence trail mid-task. Tools keep answering with the + // terminal error and the host decides what to do with the run. + // sessionAgeMs against the configured session timeout tells a Browserbase + // TIMED_OUT apart from a remote close. + onSessionLost: (loss) => + process.stderr.write( + `${SESSION_LOST_TELEMETRY_PREFIX}${JSON.stringify({ + ...loss, + cause: sanitizeErrorMessage(loss.cause), + provider: browser.provider, + ...(browser.sessionId && { sessionId: browser.sessionId }), + sessionAgeMs: Date.now() - launchedAt, + ...(config.browser.type === "browserbase" && + typeof config.browser.launchOptions.timeout === "number" && { + sessionTimeoutMs: config.browser.launchOptions.timeout * 1000, + }), + })}\n`, + ), }); return { browser, stagehand, tools }; } catch (error) { diff --git a/packages/integrations/core/src/facade/tools.ts b/packages/integrations/core/src/facade/tools.ts index 8341fa8d4..2a090b43f 100644 --- a/packages/integrations/core/src/facade/tools.ts +++ b/packages/integrations/core/src/facade/tools.ts @@ -3,10 +3,12 @@ import path from "node:path"; import type { ExperimentalBatchCallback, Page, Stagehand } from "@browserbasehq/stagehand"; import { sanitizeErrorMessage } from "../harness/redact.js"; import { + browserSessionLostError, NAVIGATED_SNAPSHOT_ERROR, NO_HYDRATED_SNAPSHOT_ERROR, RefActionSchema, staleSnapshotIdError, + type FacadeSessionLoss, type RefAction, } from "./contract.js"; import { createPlaywrightCompatRuntime, type PlaywrightCompatTelemetry } from "./runtime.js"; @@ -39,6 +41,8 @@ export type StagehandFacadeToolsOptions = { artifactRoot?: string; /** Observes every completed `run` batch (including ones whose code threw). */ onRunReport?: (report: StagehandFacadeRunReport) => void; + /** Fires once, the first time a call proves the browser session is gone. */ + onSessionLost?: (loss: FacadeSessionLoss) => void; /** * Keep a hidden about:blank tab open for the whole session (default true). * Chrome exits when its last tab closes, so a renderer crash on the agent's @@ -51,7 +55,21 @@ export type StagehandFacadeToolsOptions = { keeperPage?: boolean; }; +/** Every facade tool returns this once the browser session is gone. */ +export class StagehandFacadeSessionLostError extends Error { + override readonly name = "StagehandFacadeSessionLostError"; + constructor(readonly loss: FacadeSessionLoss) { + super(browserSessionLostError(loss.cause)); + } +} + const RUN_BATCH_TIMEOUT_MS = 60_000; +/** + * snapshot/screenshot RPCs have no executor-side deadline. Calls are serialized, + * so one that never answers must release the queue at a bounded deadline. + * Repeated capture deadlines latch terminal loss; no underlying RPC is retried. + */ +const PAGE_CAPTURE_DEADLINE_MS = 120_000; export type StagehandFacadeScreenshot = { data: string; @@ -129,12 +147,23 @@ type RunInput = { hiddenPageIds?: string[] }; export class StagehandFacadeTools { private readonly snapshotsByPage = new Map(); private queue: Promise = Promise.resolve(); + private loss: FacadeSessionLoss | undefined; private keeper: Promise | undefined; + // Consecutive capture-deadline timeouts; reset by any successful tool call. + // Three consecutive failures end this facade; this does not prove transport loss. + private consecutiveDeadlines = 0; + private static readonly MAX_CONSECUTIVE_DEADLINES = 3; + constructor( private readonly stagehand: Stagehand, private readonly options: StagehandFacadeToolsOptions = {}, ) {} + /** Set once a call has proven the browser session is gone; never cleared. */ + get sessionLoss(): FacadeSessionLoss | undefined { + return this.loss; + } + snapshot(options: { includeIframes?: boolean } = {}): Promise { return this.enqueue("snapshot", () => this.snapshotNow(options)); } @@ -155,11 +184,18 @@ export class StagehandFacadeTools { private async snapshotNow(options: { includeIframes?: boolean }): Promise { const page = await this.activePage(); - const snapshot = await page.snapshot({ includeIframes: options.includeIframes ?? true }); - this.snapshotsByPage.set(page.pageId, { - url: await page.url(), - xpathById: { ...snapshot.xpathMap }, - }); + // Failed captures invalidate the preceding snapshot too. A late response + // must never replace the IDs installed by a subsequent successful capture. + this.snapshotsByPage.delete(page.pageId); + const { snapshot, url } = await withDeadline( + (async () => { + const snapshot = await page.snapshot({ includeIframes: options.includeIframes ?? true }); + return { snapshot, url: await page.url() }; + })(), + PAGE_CAPTURE_DEADLINE_MS, + "page.snapshot", + ); + this.snapshotsByPage.set(page.pageId, { url, xpathById: { ...snapshot.xpathMap } }); return snapshot.formattedTree; } @@ -173,11 +209,15 @@ export class StagehandFacadeTools { // CDP only accepts quality for jpeg, and only as an integer. const quality = type === "jpeg" && options.quality !== undefined ? Math.round(options.quality) : undefined; - const bytes = await page.screenshot({ - type, - ...(options.fullPage === undefined ? {} : { fullPage: options.fullPage }), - ...(quality === undefined ? {} : { quality }), - }); + const bytes = await withDeadline( + page.screenshot({ + type, + ...(options.fullPage === undefined ? {} : { fullPage: options.fullPage }), + ...(quality === undefined ? {} : { quality }), + }), + PAGE_CAPTURE_DEADLINE_MS, + "page.screenshot", + ); return { data: Buffer.from(bytes).toString("base64"), mimeType: type === "jpeg" ? "image/jpeg" : "image/png", @@ -200,10 +240,12 @@ export class StagehandFacadeTools { if (!xpath) throw new Error(staleSnapshotIdError(action.id)); return { ...action, selector: `xpath=${xpath}` }; }); + // The callback can already have dispatched earlier actions when one fails. + // Never replay an action batch after a partial failure. const result = await this.stagehand.experimentalBatch( actionRunner, { actions: hydrated }, - { page, timeout: 60_000 }, + { page, timeout: RUN_BATCH_TIMEOUT_MS }, ); return { completed: result?.completed ?? hydrated.length, url: await page.url() }; } @@ -322,12 +364,35 @@ export class StagehandFacadeTools { return keeper.pageId; } - private enqueue(_tool: string, operation: () => Promise): Promise { - const execute = async (): Promise => { - await this.ensureKeeperPage(); - return operation(); + private enqueue(tool: string, operation: () => Promise): Promise { + const guarded = async (): Promise => { + if (this.loss) throw new StagehandFacadeSessionLostError(this.loss); + try { + await this.ensureKeeperPage(); + const value = await operation(); + this.consecutiveDeadlines = 0; // a real response proves the session is alive + return value; + } catch (error) { + // Permit two consecutive capture timeouts. The deadline does not cancel + // the underlying RPC, and recovery never replays that capture or an action. + if (error instanceof FacadeDeadlineError) { + this.consecutiveDeadlines += 1; + if (this.consecutiveDeadlines < StagehandFacadeTools.MAX_CONSECUTIVE_DEADLINES) { + throw error; + } + const cause = `executor unresponsive: ${this.consecutiveDeadlines} consecutive capture timeouts (last: ${error.message})`; + this.loss = { cause, tool, at: new Date().toISOString() }; + this.options.onSessionLost?.(this.loss); + throw new StagehandFacadeSessionLostError(this.loss); + } + const cause = sessionLossCause(error); + if (cause === undefined) throw error; + this.loss = { cause, tool, at: new Date().toISOString() }; + this.options.onSessionLost?.(this.loss); + throw new StagehandFacadeSessionLostError(this.loss); + } }; - const result = this.queue.then(execute, execute); + const result = this.queue.then(guarded, guarded); this.queue = result.then( () => undefined, () => undefined, @@ -336,6 +401,32 @@ export class StagehandFacadeTools { } } +class FacadeDeadlineError extends Error { + override readonly name = "FacadeDeadlineError"; + constructor(operation: string, timeoutMs: number) { + super(`${operation} received no response within ${timeoutMs}ms`); + } +} + +function withDeadline(promise: Promise, timeoutMs: number, operation: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new FacadeDeadlineError(operation, timeoutMs)), + timeoutMs, + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error: unknown) => { + clearTimeout(timer); + reject(error); + }, + ); + }); +} + /** * Maps an error to the reason the browser session is unusable, or undefined * when it is an ordinary tool failure the agent can act on. A batch that hit diff --git a/packages/integrations/core/tests/facade-screenshot-transport.test.ts b/packages/integrations/core/tests/facade-screenshot-transport.test.ts index 109aec417..f35a69cbd 100644 --- a/packages/integrations/core/tests/facade-screenshot-transport.test.ts +++ b/packages/integrations/core/tests/facade-screenshot-transport.test.ts @@ -3,6 +3,7 @@ import { captureScreenshotWithinBase64Budget, screenshotBase64BudgetFromArgs, type ScreenshotOptions, + imageDimensions, } from "../src/facade/screenshot-transport.js"; describe("facade screenshot transport", () => { @@ -95,3 +96,68 @@ describe("facade screenshot transport", () => { expect(capture).toHaveBeenCalledTimes(3); }); }); + +function png(width: number, height: number): string { + const b = Buffer.alloc(24); + b.write("\x89PNG\r\n\x1a\n", 0, "binary"); + b.writeUInt32BE(13, 8); + b.write("IHDR", 12, "ascii"); + b.writeUInt32BE(width, 16); + b.writeUInt32BE(height, 20); + return b.toString("base64"); +} +function jpeg(width: number, height: number): string { + // SOI, APP0 (empty), SOF0 with the given size. + const b = Buffer.from([ + 0xff, + 0xd8, + 0xff, + 0xe0, + 0x00, + 0x02, + 0xff, + 0xc0, + 0x00, + 0x0b, + 0x08, + (height >> 8) & 0xff, + height & 0xff, + (width >> 8) & 0xff, + width & 0xff, + 0x01, + 0x01, + 0x11, + 0x00, + ]); + return b.toString("base64"); +} + +describe("screenshot dimension guard", () => { + it("parses PNG and JPEG headers", () => { + expect(imageDimensions({ data: png(1288, 9400), mimeType: "image/png" })).toStrictEqual({ + width: 1288, + height: 9400, + }); + expect(imageDimensions({ data: jpeg(640, 480), mimeType: "image/jpeg" })).toStrictEqual({ + width: 640, + height: 480, + }); + }); + + it("falls back to the viewport when a full-page capture exceeds the side limit", async () => { + const calls: Array<{ fullPage?: boolean }> = []; + const result = await captureScreenshotWithinBase64Budget( + async (options) => { + calls.push(options); + return options.fullPage + ? { data: png(1288, 2400), mimeType: "image/png" as const } + : { data: jpeg(1288, 711), mimeType: "image/jpeg" as const }; + }, + { fullPage: true, type: "png" }, + 10_000_000, + ); + expect(calls[0]?.fullPage).toBe(true); + expect(result.options.fullPage).toBe(false); + expect(result.adjusted).toBe(true); + }); +}); diff --git a/packages/integrations/core/tests/facade-tools.test.ts b/packages/integrations/core/tests/facade-tools.test.ts index c32a4b491..7b1445853 100644 --- a/packages/integrations/core/tests/facade-tools.test.ts +++ b/packages/integrations/core/tests/facade-tools.test.ts @@ -1,13 +1,13 @@ import fsp from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { type Stagehand } from "@browserbasehq/stagehand"; +import { CDPConnectionClosedError, type Stagehand } from "@browserbasehq/stagehand"; import { afterEach, describe, expect, it, vi } from "vitest"; import { BROWSER_SESSION_LOST_ERROR_PREFIX, type FacadeSessionLoss, } from "../src/facade/contract.js"; -import { StagehandFacadeTools, type StagehandFacadeRunReport } from "../src/facade/tools.js"; +import { StagehandFacadeSessionLostError, StagehandFacadeTools, type StagehandFacadeRunReport } from "../src/facade/tools.js"; type FakePage = ReturnType; @@ -320,3 +320,262 @@ describe("StagehandFacadeTools keeper tab", () => { expect(context.newPage).not.toHaveBeenCalled(); }); }); + +describe("StagehandFacadeTools session loss", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + function batchTimeoutError() { + const error = new Error("stagehand.experimentalBatch() received no response within 75000ms"); + error.name = "StagehandBatchTimeoutError"; + Object.assign(error, { timeout: 60_000, clientTimeout: 75_000 }); + return error; + } + + it("turns a batch client deadline into the terminal error and stays dead", async () => { + const page = createFakePage(); + const { stagehand, experimentalBatch, context } = createFakeStagehand(page); + experimentalBatch.mockRejectedValueOnce(batchTimeoutError()); + const losses: FacadeSessionLoss[] = []; + const tools = new StagehandFacadeTools(stagehand, { + onSessionLost: (loss) => losses.push(loss), + }); + + const first = tools.run("await page.getByRole('button', { name: 'Search now' }).click();"); + await expect(first).rejects.toBeInstanceOf(StagehandFacadeSessionLostError); + await expect(first).rejects.toThrow( + "Browser session lost (batch received no response within 75000ms). The task cannot continue; report your final result now.", + ); + expect(losses).toEqual([ + { cause: "batch received no response within 75000ms", tool: "run", at: expect.any(String) }, + ]); + expect(tools.sessionLoss).toBe(losses[0]); + + // Every later call gets the same terminal answer without touching the browser. + const callsBefore = context.activePage.mock.calls.length; + await expect(tools.snapshot()).rejects.toThrow(BROWSER_SESSION_LOST_ERROR_PREFIX); + await expect(tools.run("return 1;")).rejects.toThrow(BROWSER_SESSION_LOST_ERROR_PREFIX); + await expect(tools.screenshot()).rejects.toThrow(BROWSER_SESSION_LOST_ERROR_PREFIX); + expect(context.activePage.mock.calls.length).toBe(callsBefore); + expect(experimentalBatch).toHaveBeenCalledTimes(1); + expect(losses).toHaveLength(1); + }); + + it("treats a closed RPC/CDP transport as session loss", async () => { + const page = createFakePage(); + const { stagehand, context } = createFakeStagehand(page); + context.activePage.mockRejectedValueOnce(new Error("RPC client is closed")); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.snapshot()).rejects.toThrow("Browser session lost (RPC client closed)."); + expect(tools.sessionLoss?.tool).toBe("snapshot"); + }); + + it("preserves socket diagnostics when a CDP error arrives before close", async () => { + // The SDK error-before-close path wraps the socket failure as cause, + // without a close code in the outer message. + const error = new CDPConnectionClosedError({ + cause: new TypeError("WebSocket failed", { + cause: Object.assign(new Error("other side closed"), { code: "UND_ERR_SOCKET" }), + }), + }); + const { stagehand, context } = createFakeStagehand(createFakePage()); + context.activePage.mockRejectedValueOnce(error); + const onSessionLost = vi.fn(); + const tools = new StagehandFacadeTools(stagehand, { onSessionLost }); + const cause = + "CDP connection closed; caused by TypeError: WebSocket failed; caused by Error [UND_ERR_SOCKET]: other side closed"; + + await expect(tools.snapshot()).rejects.toThrow(cause); + await expect(tools.snapshot()).rejects.toThrow(cause); + expect(tools.sessionLoss?.cause).toBe(cause); + expect(onSessionLost).toHaveBeenCalledOnce(); + expect(context.activePage).toHaveBeenCalledOnce(); + }); + + it("redacts credentials in CDP close reasons and nested socket causes before emitting loss", async () => { + const socketError = new TypeError( + "wss://browser.example/session?signingKey=url-secret&apiKey=api-secret&token=token-secret " + + "sk-abcdef1234567890 bb_live_abcd1234567890 Bearer bearer-secret-value", + ); + const error = new CDPConnectionClosedError({ + code: 1006, + reason: "https://browser.example/?key=reason-secret", + cause: socketError, + }); + const { stagehand, context } = createFakeStagehand(createFakePage()); + context.activePage.mockRejectedValueOnce(error); + const onSessionLost = vi.fn(); + const tools = new StagehandFacadeTools(stagehand, { onSessionLost }); + + const failure = await tools.snapshot().catch((error: unknown) => error); + expect(failure).toBeInstanceOf(StagehandFacadeSessionLostError); + const output = JSON.stringify({ + message: (failure as Error).message, + loss: tools.sessionLoss, + callback: onSessionLost.mock.calls, + }); + for (const secret of [ + "url-secret", + "api-secret", + "token-secret", + "reason-secret", + "sk-abcdef1234567890", + "bb_live_abcd1234567890", + "bearer-secret-value", + ]) { + expect(output).not.toContain(secret); + } + expect(tools.sessionLoss?.cause).toContain("close code 1006"); + expect(tools.sessionLoss?.cause).toContain( + "caused by TypeError: wss://browser.example/session", + ); + expect(tools.sessionLoss?.cause).toContain( + "signingKey=[redacted]&apiKey=[redacted]&token=[redacted]", + ); + }); + + it("handles empty and cyclic CDP causes without losing the error type", async () => { + const socketError = new TypeError(); + const error = new CDPConnectionClosedError({ cause: socketError }); + socketError.cause = error; + const { stagehand, context } = createFakeStagehand(createFakePage()); + context.activePage.mockRejectedValueOnce(error); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.snapshot()).rejects.toThrow("CDP connection closed; caused by TypeError"); + expect(tools.sessionLoss?.cause).toBe("CDP connection closed; caused by TypeError"); + }); + + it("does not treat an executor-side batch timeout or agent code errors as session loss", async () => { + const page = createFakePage(); + const { stagehand, experimentalBatch } = createFakeStagehand(page); + experimentalBatch.mockRejectedValueOnce( + new Error("Stagehand callback batch timed out after 60000ms"), + ); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.run("return 1;")).rejects.toThrow("callback batch timed out"); + expect(tools.sessionLoss).toBeUndefined(); + await expect(tools.run("throw new Error('RPC client is closed');")).rejects.toThrow( + "RPC client is closed", + ); + expect(tools.sessionLoss).toBeUndefined(); + await expect(tools.run("return 2;")).resolves.toBe(2); + }); + + it("resets capture timeout count after a successful operation", async () => { + vi.useFakeTimers(); + const page = createFakePage(); + page.snapshot.mockImplementation(() => new Promise(() => undefined)); + const { stagehand } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + const timeout = async () => { + const rejection = expect(tools.snapshot()).rejects.toThrow("received no response"); + await vi.advanceTimersByTimeAsync(120_000); + await rejection; + }; + await timeout(); + await timeout(); + await expect(tools.run("return 1;")).resolves.toBe(1); + await timeout(); + await timeout(); + expect(tools.sessionLoss).toBeUndefined(); + }); + + it("ignores a late timed-out snapshot after newer IDs have been installed", async () => { + vi.useFakeTimers(); + const world = createFakeWorld(); + const page = createFakePage("https://example.com", world); + let resolveLate!: (value: { formattedTree: string; xpathMap: Record }) => void; + page.snapshot.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveLate = resolve; + }), + ); + const { stagehand } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + const rejection = expect(tools.snapshot()).rejects.toThrow("received no response"); + await vi.advanceTimersByTimeAsync(120_000); + await rejection; + world.snapshot = { formattedTree: "new", xpathMap: { "0-2": "/new/button" } }; + await expect(tools.snapshot()).resolves.toBe("new"); + resolveLate({ formattedTree: "old", xpathMap: { "0-1": "/old/button" } }); + await Promise.resolve(); + await expect(tools.runActions([{ op: "click", id: "0-2" }])).resolves.toMatchObject({ + completed: 1, + }); + expect(page.locator).toHaveBeenLastCalledWith("xpath=/new/button"); + await expect(tools.runActions([{ op: "click", id: "0-1" }])).rejects.toThrow(); + }); + + it("latches terminal loss once and never dispatches work already queued behind it", async () => { + const page = createFakePage(); + const { stagehand, experimentalBatch } = createFakeStagehand(page); + experimentalBatch.mockRejectedValueOnce(batchTimeoutError()); + const onSessionLost = vi.fn(); + const tools = new StagehandFacadeTools(stagehand, { onSessionLost }); + const outcomes = await Promise.allSettled([ + tools.run("return 1;"), + tools.snapshot(), + tools.screenshot(), + tools.run("return 2;"), + ]); + expect( + outcomes.every( + (result) => + result.status === "rejected" && result.reason instanceof StagehandFacadeSessionLostError, + ), + ).toBe(true); + expect(onSessionLost).toHaveBeenCalledOnce(); + expect(experimentalBatch).toHaveBeenCalledOnce(); + expect(page.snapshot).not.toHaveBeenCalled(); + expect(page.screenshot).not.toHaveBeenCalled(); + }); + + it("does not replay successful actions when a later action fails", async () => { + const world = createFakeWorld(); + world.snapshot = { formattedTree: "actions", xpathMap: { "0-1": "/first", "0-2": "/second" } }; + world.clickErrors["xpath=/second"] = new Error("Node does not have a layout object"); + const page = createFakePage("https://example.com", world); + const { stagehand, experimentalBatch } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + await tools.snapshot(); + await expect( + tools.runActions([ + { op: "click", id: "0-1" }, + { op: "click", id: "0-2" }, + ]), + ).rejects.toThrow("layout object"); + expect(experimentalBatch).toHaveBeenCalledOnce(); + expect(world.locators.filter((locator) => locator.selector === "xpath=/first")).toHaveLength(1); + expect(world.locators[0]?.click).toHaveBeenCalledOnce(); + expect(page.waitForTimeout).not.toHaveBeenCalled(); + }); + + it("treats a single capture deadline as recoverable, escalating to session loss only after repeated consecutive timeouts", async () => { + vi.useFakeTimers(); + const page = createFakePage(); + page.snapshot.mockImplementation(() => new Promise(() => undefined)); + const { stagehand } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + + for (let i = 0; i < 2; i++) { + const pending = tools.snapshot(); + const rejection = expect(pending).rejects.toThrow( + "page.snapshot received no response within 120000ms", + ); + await vi.advanceTimersByTimeAsync(120_000); + await rejection; + expect(tools.sessionLoss).toBeUndefined(); + } + + const pending = tools.snapshot(); + const rejection = expect(pending).rejects.toThrow("Browser session lost"); + await vi.advanceTimersByTimeAsync(120_000); + await rejection; + expect(tools.sessionLoss?.cause).toContain("consecutive capture timeouts"); + }); +}); diff --git a/packages/sdk-ts/src/batch.ts b/packages/sdk-ts/src/batch.ts index c63307336..403ddef2c 100644 --- a/packages/sdk-ts/src/batch.ts +++ b/packages/sdk-ts/src/batch.ts @@ -15,10 +15,40 @@ import type { Page } from "./page.js"; export type ExperimentalBatchOptions = { /** Page exposed as `batch.page`. AI operations still default to the active page. */ page?: Page; - /** Overall callback deadline in milliseconds. */ + /** Overall callback deadline in milliseconds, enforced by the browser-side executor. */ timeout?: number; + /** + * Local deadline for the whole round trip in milliseconds. Defaults to + * `timeout + CALLBACK_BATCH_CLIENT_GRACE_MS`. It fires when the executor never + * answers at all (stalled navigation, hung service worker), which the + * browser-side `timeout` cannot cover. + */ + clientTimeoutMs?: number; }; +/** Slack the client grants the executor to report its own `timeout` before giving up locally. */ +export const CALLBACK_BATCH_CLIENT_GRACE_MS = 15_000; + +/** + * The batch round trip exceeded its client-side deadline. The executor may + * still be running the callback; the browser session should be treated as + * unresponsive rather than retried blindly. + */ +export class StagehandBatchTimeoutError extends Error { + readonly timeout: number; + readonly clientTimeout: number; + + constructor(details: { timeout: number; clientTimeout: number }, options?: ErrorOptions) { + super( + `stagehand.experimentalBatch() received no response within ${details.clientTimeout}ms (callback timeout ${details.timeout}ms)`, + options, + ); + this.name = "StagehandBatchTimeoutError"; + this.timeout = details.timeout; + this.clientTimeout = details.clientTimeout; + } +} + export type ExperimentalBatchBrowserContext = Omit< BrowserContext, "close" | "rpcClient" | "clipboardRef" diff --git a/packages/sdk-ts/src/cdpClient.ts b/packages/sdk-ts/src/cdpClient.ts index 487ca813e..2fdddabb5 100644 --- a/packages/sdk-ts/src/cdpClient.ts +++ b/packages/sdk-ts/src/cdpClient.ts @@ -173,8 +173,12 @@ const InstalledExtensionsResultSchema = z.looseObject({ const STAGEHAND_EXTENSION_NAME = "Stagehand Runtime"; export class CDPConnectionClosedError extends Error { - constructor(options?: ErrorOptions) { - super("CDP connection closed", options); + constructor(options?: ErrorOptions & { code?: number; reason?: string }) { + const detail = + options?.code !== undefined + ? ` (close code ${options.code}${options.reason ? `: ${options.reason}` : ""})` + : ""; + super(`CDP connection closed${detail}`, options); this.name = "CDPConnectionClosedError"; } } @@ -203,10 +207,11 @@ export class CDPClient { }); }); - this.socket.addEventListener("close", () => { + this.socket.addEventListener("close", (event) => { if (this.closed) return; this.closed = true; - const reason = new CDPConnectionClosedError(); + const { code, reason: closeReason } = event as Event & { code?: number; reason?: string }; + const reason = new CDPConnectionClosedError({ code, reason: closeReason }); this.rejectPending(reason); this.onclose?.(reason); }); diff --git a/packages/sdk-ts/src/index.ts b/packages/sdk-ts/src/index.ts index 5e3b28382..8c699ee11 100644 --- a/packages/sdk-ts/src/index.ts +++ b/packages/sdk-ts/src/index.ts @@ -38,14 +38,18 @@ export { type ResponseServerAddr, } from "./response.js"; export { WebMCPInvocation, WebMCPTool } from "./webmcp.js"; +export { CDPConnectionClosedError } from "./cdpClient.js"; +export { RPCResponseTimeoutError } from "./rpcClient.js"; export type { InitScriptSource } from "./pageScripts.js"; export { Stagehand, type ExtractResult } from "./stagehand.js"; -export type { - ExperimentalBatchCallback, - ExperimentalBatchBrowserContext, - ExperimentalBatchContext, - ExperimentalBatchExtractOptions, - ExperimentalBatchOptions, +export { + CALLBACK_BATCH_CLIENT_GRACE_MS, + StagehandBatchTimeoutError, + type ExperimentalBatchCallback, + type ExperimentalBatchBrowserContext, + type ExperimentalBatchContext, + type ExperimentalBatchExtractOptions, + type ExperimentalBatchOptions, } from "./batch.js"; export { browserbase, localBrowser } from "./browser/factories.js"; export type { diff --git a/packages/sdk-ts/src/rpcClient.ts b/packages/sdk-ts/src/rpcClient.ts index dee85b525..9e08c8a60 100644 --- a/packages/sdk-ts/src/rpcClient.ts +++ b/packages/sdk-ts/src/rpcClient.ts @@ -54,8 +54,22 @@ type RegisteredRequestHandler = { type RPCSendOptions = { signal?: AbortSignal; + /** Replaces the method's derived response deadline for this one request. */ + responseTimeoutMs?: number; }; +export class RPCResponseTimeoutError extends Error { + readonly method: string; + readonly timeoutMs: number; + + constructor(method: string, timeoutMs: number) { + super(`RPC response timed out: ${method}`, { cause: { method, timeoutMs } }); + this.name = "RPCResponseTimeoutError"; + this.method = method; + this.timeoutMs = timeoutMs; + } +} + const TRACER = trace.getTracer("@browserbasehq/stagehand"); const W3C_TRACE_CONTEXT_PROPAGATOR = new W3CTraceContextPropagator(); const MAX_PENDING_NOTIFICATIONS = 100; @@ -190,7 +204,8 @@ export class RPCClient { ...getTraceContextFields(requestContext), }); span.setAttribute("jsonrpc.request.id", String(request.id)); - const responseTimeoutMs = rpcResponseTimeoutMs(method.name, parsedParams); + const responseTimeoutMs = + options.responseTimeoutMs ?? rpcResponseTimeoutMs(method.name, parsedParams); const timeoutController = responseTimeoutMs === undefined ? undefined : new AbortController(); const signal = @@ -201,9 +216,7 @@ export class RPCClient { timeoutController && responseTimeoutMs !== undefined ? setTimeout(() => { timeoutController.abort( - new Error(`RPC response timed out: ${method.name}`, { - cause: { method: method.name, timeoutMs: responseTimeoutMs }, - }), + new RPCResponseTimeoutError(method.name, responseTimeoutMs), ); }, responseTimeoutMs) : undefined; diff --git a/packages/sdk-ts/src/stagehand.ts b/packages/sdk-ts/src/stagehand.ts index f5a10d4fe..c008f8221 100644 --- a/packages/sdk-ts/src/stagehand.ts +++ b/packages/sdk-ts/src/stagehand.ts @@ -1,4 +1,4 @@ -import { RPCClient } from "./rpcClient.js"; +import { RPCClient, RPCResponseTimeoutError } from "./rpcClient.js"; import { DefaultExtractDataSchema, MAX_CALLBACK_BATCH_TIMEOUT_MS, @@ -45,7 +45,12 @@ import { } from "./browser/factories.js"; import { attachStagehandBrowserContext, detachStagehandBrowserContext } from "./browser/index.js"; import { withStagehandInitDeadline } from "./timeouts.js"; -import type { ExperimentalBatchCallback, ExperimentalBatchOptions } from "./batch.js"; +import { + CALLBACK_BATCH_CLIENT_GRACE_MS, + StagehandBatchTimeoutError, + type ExperimentalBatchCallback, + type ExperimentalBatchOptions, +} from "./batch.js"; type ProtocolExtractResult = import("@browserbasehq/stagehand-protocol/types").ExtractResult; @@ -61,6 +66,9 @@ const isZodSchema = (value: unknown): value is z.ZodType => "safeParse" in value && typeof value.safeParse === "function"; +// setTimeout treats larger delays as 1ms; MAX_CALLBACK_BATCH_TIMEOUT_MS leaves 10s below this. +const MAX_TIMER_DELAY_MS = 2_147_483_647; + const nativeFunctionSourcePattern = /^\s*function(?:\s+[^()]*)?\([^)]*\)\s*\{\s*\[native code\]\s*\}\s*$/; @@ -154,18 +162,35 @@ export class Stagehand { if (nativeFunctionSourcePattern.test(callbackSource)) { throw new TypeError("stagehand.experimentalBatch() callback must be serializable JavaScript"); } + const clientTimeout = + options.clientTimeoutMs ?? + Math.min(timeout + CALLBACK_BATCH_CLIENT_GRACE_MS, MAX_TIMER_DELAY_MS); + if (!Number.isInteger(clientTimeout) || clientTimeout <= 0 || clientTimeout > 2_147_483_647) { + throw new RangeError( + "stagehand.experimentalBatch() clientTimeoutMs must be an integer between 1 and 2147483647", + ); + } - const result: CallbackBatchResult = await this.connectedRpcClient.send( - StagehandMethods.stagehandCallbackBatch, - { - callbackSource, - ...(parsedInput === undefined ? {} : { input: parsedInput }), - options: { - ...(options.page ? { pageId: options.page.pageId } : {}), - timeout, + let result: CallbackBatchResult; + try { + result = await this.connectedRpcClient.send( + StagehandMethods.stagehandCallbackBatch, + { + callbackSource, + ...(parsedInput === undefined ? {} : { input: parsedInput }), + options: { + ...(options.page ? { pageId: options.page.pageId } : {}), + timeout, + }, }, - }, - ); + { responseTimeoutMs: clientTimeout }, + ); + } catch (error) { + if (error instanceof RPCResponseTimeoutError) { + throw new StagehandBatchTimeoutError({ timeout, clientTimeout }, { cause: error }); + } + throw error; + } return result.value as Awaited; } diff --git a/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts b/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts new file mode 100644 index 000000000..fb3b7ea7d --- /dev/null +++ b/packages/sdk-ts/tests/experimentalBatchDeadline.test.ts @@ -0,0 +1,149 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { JSONRPCMessage } from "@browserbasehq/stagehand-protocol/json-rpc/types"; +import { StagehandMethods } from "@browserbasehq/stagehand-protocol/schema-registry"; +import { MAX_CALLBACK_BATCH_TIMEOUT_MS } from "@browserbasehq/stagehand-protocol/schemas"; +import { + BrowserContext, + CALLBACK_BATCH_CLIENT_GRACE_MS, + Stagehand, + StagehandBatchTimeoutError, +} from "../src/index.js"; +import { RPCClient, RPCResponseTimeoutError, type CDPTransport } from "../src/rpcClient.js"; +import { + attachStagehandBrowserContext, + claimStagehandBrowserHandle, + createStagehandBrowserHandle, +} from "../src/browser/index.js"; + +/** A transport that accepts requests and never answers them. */ +class SilentCDPTransport implements CDPTransport { + readonly serviceWorker = { + targetId: "worker-target", + url: "chrome-extension://stagehand/service-worker.js", + title: "Stagehand", + extensionId: "stagehand", + }; + onmessage?: (message: unknown) => void | Promise; + onclose?: (reason?: Error) => void; + onerror?: (error: Error) => void; + readonly sent: JSONRPCMessage[] = []; + + async send(message: JSONRPCMessage): Promise { + this.sent.push(message); + } + + close(): void {} +} + +function createStagehand(client: RPCClient): Stagehand { + const browser = createStagehandBrowserHandle({ + provider: "local", + origin: "connected", + attachment: {}, + close: () => {}, + }); + claimStagehandBrowserHandle(browser); + attachStagehandBrowserContext(browser, new BrowserContext(client, () => browser.close())); + const stagehand = Object.create(Stagehand.prototype) as Stagehand; + Object.assign(stagehand, { browserHandle: browser }); + stagehand.rpcClient = client; + stagehand.isInitialized = true; + return stagehand; +} + +describe("experimentalBatch client deadline", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("rejects with a typed error when the executor never answers", async () => { + vi.useFakeTimers(); + const transport = new SilentCDPTransport(); + const client = new RPCClient(transport); + const stagehand = createStagehand(client); + + try { + const pending = stagehand.experimentalBatch(async () => "never", undefined, { + timeout: 60_000, + }); + const rejection = expect(pending).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(StagehandBatchTimeoutError); + const typed = error as StagehandBatchTimeoutError; + expect(typed.timeout).toBe(60_000); + expect(typed.clientTimeout).toBe(60_000 + CALLBACK_BATCH_CLIENT_GRACE_MS); + expect(typed.cause).toBeInstanceOf(RPCResponseTimeoutError); + return true; + }); + + await vi.advanceTimersByTimeAsync(60_000 + CALLBACK_BATCH_CLIENT_GRACE_MS - 1); + expect(client.pending.size).toBe(1); + await vi.advanceTimersByTimeAsync(1); + await rejection; + expect(client.pending.size).toBe(0); + expect(transport.sent).toHaveLength(1); + expect((transport.sent[0] as { method: string }).method).toBe( + StagehandMethods.stagehandCallbackBatch.name, + ); + } finally { + client.close(); + } + }); + + it("lets callers shorten the round-trip deadline below the executor timeout", async () => { + vi.useFakeTimers(); + const client = new RPCClient(new SilentCDPTransport()); + const stagehand = createStagehand(client); + + try { + const pending = stagehand.experimentalBatch(async () => "never", undefined, { + timeout: 60_000, + clientTimeoutMs: 5_000, + }); + const rejection = expect(pending).rejects.toMatchObject({ + name: "StagehandBatchTimeoutError", + timeout: 60_000, + clientTimeout: 5_000, + }); + await vi.advanceTimersByTimeAsync(5_000); + await rejection; + } finally { + client.close(); + } + }); + + it("keeps the maximum executor timeout within the timer limit", async () => { + vi.useFakeTimers(); + const client = new RPCClient(new SilentCDPTransport()); + const stagehand = createStagehand(client); + + try { + const pending = stagehand.experimentalBatch(async () => "never", undefined, { + timeout: MAX_CALLBACK_BATCH_TIMEOUT_MS, + }); + const rejection = expect(pending).rejects.toMatchObject({ clientTimeout: 2_147_483_647 }); + // A delay above the limit would have fired immediately; the request must still be pending. + await vi.advanceTimersByTimeAsync(1_000); + expect(client.pending.size).toBe(1); + await vi.advanceTimersByTimeAsync(2_147_483_647); + await rejection; + } finally { + client.close(); + } + }); + + it.each([0, -1, 0.5, 2_147_483_648, NaN, Infinity])( + "validates clientTimeoutMs %s before sending", + async (clientTimeoutMs) => { + const client = new RPCClient(new SilentCDPTransport()); + const stagehand = createStagehand(client); + try { + await expect( + stagehand.experimentalBatch(async () => undefined, undefined, { clientTimeoutMs }), + ).rejects.toThrow(RangeError); + expect(client.pending.size).toBe(0); + } finally { + client.close(); + } + }, + ); +}); From 2c791a2e6d08fa201bda1aee3d347b31fd19d6a2 Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 16:49:19 -0700 Subject: [PATCH 02/35] Add bounded CDP heartbeat and sanitized disconnect diagnostics --- .changeset/eval-cdp-heartbeat.md | 5 + packages/sdk-ts/src/cdpClient.ts | 124 ++++++++++++-- packages/sdk-ts/tests/cdpHeartbeat.test.ts | 186 +++++++++++++++++++++ 3 files changed, 301 insertions(+), 14 deletions(-) create mode 100644 .changeset/eval-cdp-heartbeat.md create mode 100644 packages/sdk-ts/tests/cdpHeartbeat.test.ts diff --git a/.changeset/eval-cdp-heartbeat.md b/.changeset/eval-cdp-heartbeat.md new file mode 100644 index 000000000..73125c9a6 --- /dev/null +++ b/.changeset/eval-cdp-heartbeat.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Add configurable bounded CDP heartbeats and sanitized disconnect diagnostics with cleanup on shutdown. diff --git a/packages/sdk-ts/src/cdpClient.ts b/packages/sdk-ts/src/cdpClient.ts index 2fdddabb5..825c85cf9 100644 --- a/packages/sdk-ts/src/cdpClient.ts +++ b/packages/sdk-ts/src/cdpClient.ts @@ -1,3 +1,4 @@ +import { appendFileSync } from "node:fs"; import type { JSONRPCMessage } from "@browserbasehq/stagehand-protocol/json-rpc/types"; import { STAGEHAND_SEND_TO_HOST_BINDING, @@ -193,6 +194,13 @@ export class CDPClient { sessionId: string | undefined; attachedServiceWorker: ServiceWorkerInfo | undefined; closed = false; + private readonly heartbeatMs = cdpHeartbeatMs(process.env.STAGEHAND_CDP_HEARTBEAT_MS); + private heartbeatTimer?: ReturnType; + private heartbeatAbort?: AbortController; + private readonly connectedAt = Date.now(); + private lastMsgAt = this.connectedAt; + private lastSentAt = this.connectedAt; + private lastMethod = ""; constructor( readonly socket: WebSocket, @@ -200,35 +208,101 @@ export class CDPClient { ) { this.webSocketDebuggerUrl = webSocketDebuggerUrl; this.socket.addEventListener("message", (event) => { + if (this.closed) return; + this.lastMsgAt = Date.now(); this.handleMessage(event.data).catch((error: unknown) => { - const normalized = asError(error); - this.rejectPending(normalized); - this.onerror?.(normalized); + this.finishDrop(asError(error), "error"); }); }); this.socket.addEventListener("close", (event) => { - if (this.closed) return; - this.closed = true; const { code, reason: closeReason } = event as Event & { code?: number; reason?: string }; - const reason = new CDPConnectionClosedError({ code, reason: closeReason }); - this.rejectPending(reason); - this.onclose?.(reason); + this.finishDrop(new CDPConnectionClosedError({ code, reason: closeReason }), "close", code); }); this.socket.addEventListener("error", (event) => { - if (this.closed) return; - this.closed = true; const socketError = asError((event as Event & { error?: unknown }).error ?? event); - const reason = new CDPConnectionClosedError({ cause: socketError }); - this.rejectPending(reason); + this.finishDrop(new CDPConnectionClosedError({ cause: socketError }), "error"); + }); + this.startHeartbeat(); + } + + /** Browser-level traffic only; no replay, transport replacement or reconnection. */ + private startHeartbeat(): void { + this.heartbeatTimer = setInterval(() => { + if (this.closed || this.socket.readyState !== WebSocket.OPEN || this.heartbeatAbort) return; + const controller = new AbortController(); + this.heartbeatAbort = controller; + const timeout = setTimeout( + () => controller.abort(new Error("CDP heartbeat timed out")), + Math.min(this.heartbeatMs, 10_000), + ); + timeout.unref?.(); + void this.sendCommand("Browser.getVersion", {}, undefined, controller.signal) + .catch(() => undefined) + .finally(() => { + clearTimeout(timeout); + if (this.heartbeatAbort === controller) this.heartbeatAbort = undefined; + }); + }, this.heartbeatMs); + this.heartbeatTimer.unref?.(); + } + + private stopHeartbeat(): void { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = undefined; + this.heartbeatAbort?.abort(new Error("CDP heartbeat stopped")); + this.heartbeatAbort = undefined; + } + + private finishDrop(reason: Error, kind: "close" | "error", code?: number): void { + if (this.closed) return; + // Mark terminal before closing the socket: synchronous error->close must + // notify once and retain the original error rather than the cleanup close. + this.closed = true; + this.logDrop(reason, kind, code); + this.stopHeartbeat(); + this.rejectPending(reason); + if (kind === "error") { try { this.socket.close(); } catch { - // The transport is already terminal; preserve the original socket failure. + /* preserve the terminal error */ } this.onerror?.(reason); - }); + } else { + this.onclose?.(reason); + } + } + + private logDrop(reason: Error, kind: "close" | "error", code?: number): void { + if (process.env.STAGEHAND_CDP_LOG !== "1") return; + const now = Date.now(); + const line = `CDP_DROP ${JSON.stringify({ + ts: new Date(now).toISOString(), + kind, + code: code ?? null, + idle_ms: now - this.lastMsgAt, + since_send_ms: now - this.lastSentAt, + age_ms: now - this.connectedAt, + pending: this.pending.size, + last_method: /^[A-Za-z][A-Za-z0-9_.]*$/u.test(this.lastMethod) ? this.lastMethod : "", + reason: sanitizeCdpDiagnostic(reason.message).slice(0, 160), + })}\n`; + // Diagnostics never change the connection's terminal outcome. + try { + process.stderr.write(line); + } catch { + /* best effort */ + } + const file = process.env.STAGEHAND_CDP_LOG_FILE; + if (file) { + try { + appendFileSync(file, line); + } catch { + /* best effort */ + } + } } static async connect(options: CDPClientOptions): Promise { @@ -336,6 +410,7 @@ export class CDPClient { signal?: AbortSignal, ): Promise { throwIfAborted(signal); + if (this.closed) throw new CDPConnectionClosedError(); if (this.socket.readyState !== WebSocket.OPEN) { throw new Error("CDP connection is not open"); } @@ -365,6 +440,8 @@ export class CDPClient { } try { this.socket.send(JSON.stringify(message)); + this.lastSentAt = Date.now(); + if (method !== "Browser.getVersion") this.lastMethod = method; } catch (error) { this.pending.delete(id); signal?.removeEventListener("abort", onAbort); @@ -376,6 +453,7 @@ export class CDPClient { close(): void { if (this.closed) return; this.closed = true; + this.stopHeartbeat(); this.onmessage = undefined; this.onclose = undefined; this.onerror = undefined; @@ -735,3 +813,21 @@ function isExtensionsLoadUnpackedUnavailable(error: unknown): boolean { function asError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); } + +function cdpHeartbeatMs(raw: string | undefined): number { + const value = Number(raw); + return raw?.trim() && Number.isInteger(value) && value >= 1_000 && value <= 2_147_483_647 + ? value + : 20_000; +} + +/** The diagnostic contains no payloads or connection URLs, including close text URLs. */ +function sanitizeCdpDiagnostic(message: string): string { + return message + .replace(/(?:https?|wss?):\/\/[^\s"']+/giu, "[url]") + .replace( + /\b(?:sk-[A-Za-z0-9_-]+|bb_(?:live|test)_[A-Za-z0-9_-]+|AIza[A-Za-z0-9_-]{20,})/gu, + "[redacted]", + ) + .replace(/\bBearer\s+[^\s]+/giu, "Bearer [redacted]"); +} diff --git a/packages/sdk-ts/tests/cdpHeartbeat.test.ts b/packages/sdk-ts/tests/cdpHeartbeat.test.ts new file mode 100644 index 000000000..eb0f0a811 --- /dev/null +++ b/packages/sdk-ts/tests/cdpHeartbeat.test.ts @@ -0,0 +1,186 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CDPClient, CDPConnectionClosedError } from "../src/cdpClient.js"; + +class Socket extends EventTarget { + readyState: number = WebSocket.OPEN; + send = vi.fn<(raw: string) => void>(); + close = vi.fn(() => { + this.readyState = WebSocket.CLOSED; + }); + remoteClose(reason = "", code = 1006): void { + this.readyState = WebSocket.CLOSED; + const event = new Event("close"); + Object.assign(event, { code, reason }); + this.dispatchEvent(event); + } + fail(error: Error): void { + const event = new Event("error"); + Object.assign(event, { error }); + this.dispatchEvent(event); + } +} + +const clients: CDPClient[] = []; +function connect() { + const socket = new Socket(); + const client = new CDPClient( + socket as unknown as WebSocket, + "wss://browser.test?apiKey=connection-secret", + ); + clients.push(client); + return { socket, client }; +} + +beforeEach(() => { + vi.useFakeTimers(); + vi.stubEnv("STAGEHAND_CDP_HEARTBEAT_MS", "20000"); + vi.stubEnv("STAGEHAND_CDP_LOG", ""); + vi.stubEnv("STAGEHAND_CDP_LOG_FILE", ""); +}); +afterEach(async () => { + for (const client of clients.splice(0)) client.close(); + await Promise.resolve(); + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +describe("CDP heartbeat lifecycle", () => { + it("has at most one heartbeat pending and removes an unanswered heartbeat by its own deadline", async () => { + const { client, socket } = connect(); + await vi.advanceTimersByTimeAsync(20_000); + expect(client.pending.size).toBe(1); + expect(socket.send).toHaveBeenCalledOnce(); + expect(JSON.parse(socket.send.mock.calls[0]![0])).toMatchObject({ + method: "Browser.getVersion", + }); + await vi.advanceTimersByTimeAsync(9_999); + expect(client.pending.size).toBe(1); + await vi.advanceTimersByTimeAsync(1); + expect(client.pending.size).toBe(0); + expect(client.closed).toBe(false); + await vi.advanceTimersByTimeAsync(10_000); + expect(client.pending.size).toBe(1); + expect(socket.send).toHaveBeenCalledTimes(2); + }); + + it("uses the configured interval and does not retain the process", async () => { + vi.stubEnv("STAGEHAND_CDP_HEARTBEAT_MS", "1000"); + const intervals = vi.spyOn(globalThis, "setInterval"); + const { client, socket } = connect(); + expect(intervals.mock.results[0]?.value.hasRef()).toBe(false); + await vi.advanceTimersByTimeAsync(999); + expect(socket.send).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(socket.send).toHaveBeenCalledOnce(); + client.close(); + await vi.advanceTimersByTimeAsync(20_000); + expect(socket.send).toHaveBeenCalledOnce(); + expect(client.pending.size).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it.each(["0", "999", "1000junk", "2147483648", "NaN"])( + "falls back safely for invalid interval %s", + async (raw) => { + vi.stubEnv("STAGEHAND_CDP_HEARTBEAT_MS", raw); + const { socket } = connect(); + await vi.advanceTimersByTimeAsync(19_999); + expect(socket.send).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(socket.send).toHaveBeenCalledOnce(); + }, + ); + + it("stops on a close and rejects subsequent commands even if a socket still reports OPEN", async () => { + const { client, socket } = connect(); + const onclose = vi.fn(); + client.onclose = onclose; + await vi.advanceTimersByTimeAsync(20_000); + socket.remoteClose("remote ended", 1006); + socket.readyState = WebSocket.OPEN; + await expect( + client.sendCommand("Page.navigate", { url: "https://example.test" }), + ).rejects.toBeInstanceOf(CDPConnectionClosedError); + await vi.advanceTimersByTimeAsync(60_000); + expect(onclose).toHaveBeenCalledOnce(); + expect(client.pending.size).toBe(0); + expect(socket.send).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("preserves the first socket error when closing emits a synchronous close event", async () => { + const { client, socket } = connect(); + socket.close.mockImplementation(() => socket.remoteClose("cleanup")); + const onerror = vi.fn(); + const onclose = vi.fn(); + client.onerror = onerror; + client.onclose = onclose; + const request = client.sendCommand("Runtime.evaluate"); + const rejected = expect(request).rejects.toMatchObject({ cause: new Error("socket failed") }); + socket.fail(new Error("socket failed")); + await rejected; + await vi.advanceTimersByTimeAsync(60_000); + expect(onerror).toHaveBeenCalledOnce(); + expect(onclose).not.toHaveBeenCalled(); + expect(socket.close).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("stops heartbeat after an invalid transport message", async () => { + const { client, socket } = connect(); + const onerror = vi.fn(); + client.onerror = onerror; + socket.dispatchEvent(new MessageEvent("message", { data: "invalid json" })); + await vi.advanceTimersByTimeAsync(60_000); + expect(client.closed).toBe(true); + expect(onerror).toHaveBeenCalledOnce(); + expect(socket.send).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("writes sanitized metadata to stderr and the ESM file sink exactly once", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "cdp-diagnostic-")); + const file = path.join(directory, "drop.jsonl"); + vi.stubEnv("STAGEHAND_CDP_LOG", "1"); + vi.stubEnv("STAGEHAND_CDP_LOG_FILE", file); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + try { + const { client, socket } = connect(); + const request = client.sendCommand("Runtime.evaluate", { expression: "private payload" }); + const rejected = expect(request).rejects.toBeInstanceOf(CDPConnectionClosedError); + await vi.advanceTimersByTimeAsync(50); + socket.remoteClose( + "https://example.test?token=reason-secret sk-secret123456 bb_live_secret123456 Bearer bearer-secret-value", + ); + socket.remoteClose(); + await rejected; + const output = await readFile(file, "utf8"); + expect(stderr).toHaveBeenCalledExactlyOnceWith(output); + const metadata = JSON.parse(output.slice("CDP_DROP ".length)); + expect(metadata).toMatchObject({ + kind: "close", + code: 1006, + pending: 1, + last_method: "Runtime.evaluate", + age_ms: 50, + idle_ms: 50, + }); + for (const secret of [ + "connection-secret", + "reason-secret", + "private payload", + "sk-secret123456", + "bb_live_secret123456", + "bearer-secret-value", + ]) + expect(output).not.toContain(secret); + expect(output).toContain("[redacted]"); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); +}); From 260314927891922a0b3c6f984bd03e5bb8a251fd Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 16:49:19 -0700 Subject: [PATCH 03/35] Align label, strict locator, and URL wait semantics --- .../core/integration/facade-dom.test.ts | 233 ++++++ packages/integrations/core/package.json | 4 +- .../integrations/core/src/facade/runtime.ts | 731 ++++++++++++++++-- .../core/tests/facade-wait-for-url.test.ts | 189 +++++ packages/integrations/core/tsconfig.json | 2 +- .../core/vitest.browser.config.ts | 9 + pnpm-lock.yaml | 3 + 7 files changed, 1104 insertions(+), 67 deletions(-) create mode 100644 packages/integrations/core/integration/facade-dom.test.ts create mode 100644 packages/integrations/core/tests/facade-wait-for-url.test.ts create mode 100644 packages/integrations/core/vitest.browser.config.ts diff --git a/packages/integrations/core/integration/facade-dom.test.ts b/packages/integrations/core/integration/facade-dom.test.ts new file mode 100644 index 000000000..a5f502266 --- /dev/null +++ b/packages/integrations/core/integration/facade-dom.test.ts @@ -0,0 +1,233 @@ +import assert from "node:assert/strict"; +import { afterAll, beforeAll, describe, it } from "vitest"; +import { chromium, type Browser, type Locator, type Page } from "playwright"; +import { createPlaywrightCompatRuntime } from "../src/facade/runtime.js"; + +// Local, static DOM fixtures only. No extension, remote site, credentials or model. +// Run separately with pnpm --filter @browserbasehq/stagehand-integrations test:browser. +let browser: Browser; +beforeAll(async () => { + browser = await chromium.launch({ + channel: process.env.PLAYWRIGHT_CHROMIUM_CHANNEL ?? "chrome", + headless: true, + }); +}); +afterAll(async () => { + await browser?.close(); +}); + +describe("facade DOM compatibility against native Playwright", () => { + it("matches native labels, priority, whitespace and shadow roots", async () => { + const page = await browser.newPage(); + try { + await page.setContent(` +
+ + + + + First referenceSecond reference + + + + + + + + + + Wrong outer label
+ `); + await page.locator("#shadow-host").evaluate((host) => { + host.attachShadow({ mode: "open" }).innerHTML = + 'Shadow label' + + ''; + }); + const runtime = await createPlaywrightCompatRuntime({ + page, + context: { pages: async () => [page], activePage: async () => page }, + } as unknown as Parameters[0]); + const facade = runtime.page as Pick; + const cases: Array<{ + name: string; + label: string | RegExp; + exact?: boolean; + scope?: string; + }> = [ + { name: "Booking child age aria-label", label: "Child 1 age", exact: true }, + { name: "scoped aria-label", label: "Child 1 age", exact: true, scope: "#booking" }, + { name: "case-insensitive substring", label: "CHILD 1" }, + { name: "exact case sensitivity", label: "child 1 age", exact: true }, + { name: "native label", label: "Native label", exact: true }, + { name: "wrapped label", label: "Wrapped label", exact: true }, + { name: "first associated label", label: "First label", exact: true }, + { name: "second associated label", label: "Second label", exact: true }, + { name: "labels are not concatenated", label: "First label Second label", exact: true }, + { name: "first ARIA reference", label: "First reference", exact: true }, + { name: "second ARIA reference", label: "Second reference", exact: true }, + { + name: "ARIA references are not concatenated", + label: "First reference Second reference", + exact: true, + }, + { name: "labelledby takes priority", label: "Overridden ARIA", exact: true }, + { name: "empty referenced label takes priority", label: "Ignored fallback", exact: true }, + { name: "ARIA takes priority over native label", label: "Ignored native", exact: true }, + { name: "ARIA priority match", label: "ARIA wins", exact: true }, + { name: "broken labelledby falls back", label: "Fallback label", exact: true }, + { name: "normalized string whitespace", label: "Child 2 age", exact: true }, + { name: "empty ARIA falls back", label: "Empty ARIA fallback", exact: true }, + { name: "label text excludes script/style", label: "Clean label", exact: true }, + { name: "regular expression", label: /^child 1 age$/i }, + { name: "regex keeps original whitespace", label: /^Line\nBreak$/ }, + { name: "empty query excludes unlabeled elements", label: "", exact: true }, + { name: "shadow-root reference", label: "Shadow label", exact: true }, + { name: "shadow-root ARIA", label: "Shadow ARIA", exact: true }, + { name: "reference cannot cross shadow boundary", label: "Wrong outer label", exact: true }, + ]; + const results = []; + for (const test of cases) { + const nativeScope = test.scope ? page.locator(test.scope) : page; + const facadeScope = test.scope ? facade.locator(test.scope) : facade; + const options = { exact: test.exact }; + const native = await nativeScope + .getByLabel(test.label, options) + .evaluateAll((els) => els.map((el) => el.id)); + const actual = await facadeScope + .getByLabel(test.label, options) + .evaluateAll((els) => els.map((el) => el.id)); + results.push({ + name: test.name, + native, + actual, + pass: JSON.stringify(native) === JSON.stringify(actual), + }); + } + assert.ok( + results.every((r) => r.pass), + "Facade getByLabel differs from Playwright", + ); + await facade.getByLabel("Child 1 age", { exact: true }).selectOption("8"); + assert.equal(await page.locator("#age").inputValue(), "8"); + } finally { + await page.close(); + } + }); + it("enforces strict reads, bounded waits and non-dispatched trial actions", async () => { + const page = await browser.newPage(); + try { + await page.setContent( + '', + ); + const runtime = await createPlaywrightCompatRuntime({ + page, + context: { pages: async () => [page], activePage: async () => page }, + } as unknown as Parameters[0]); + const facade = runtime.page as Page; + const results: Array<{ name: string; pass: boolean; detail?: string }> = []; + const check = async (name: string, action: () => Promise): Promise => { + try { + await action(); + results.push({ name, pass: true }); + } catch (error) { + results.push({ name, pass: false, detail: String(error) }); + } + }; + const methods: Array<[string, (locator: Locator) => Promise]> = [ + ["textContent", (locator) => locator.textContent({ timeout: 150 })], + ["innerText", (locator) => locator.innerText({ timeout: 150 })], + ["innerHTML", (locator) => locator.innerHTML({ timeout: 150 })], + ["inputValue", (locator) => locator.inputValue({ timeout: 150 })], + ["getAttribute", (locator) => locator.getAttribute("value", { timeout: 150 })], + ["isChecked", (locator) => locator.isChecked({ timeout: 150 })], + ["isDisabled", (locator) => locator.isDisabled({ timeout: 150 })], + ["isEnabled", (locator) => locator.isEnabled({ timeout: 150 })], + ["isVisible", (locator) => locator.isVisible()], + ["boundingBox", (locator) => locator.boundingBox({ timeout: 150 })], + ["focus", (locator) => locator.focus({ timeout: 150 })], + ["evaluate", (locator) => locator.evaluate((el) => el.setAttribute("data-mutated", "yes"))], + [ + "evaluateHandle", + (locator) => locator.evaluateHandle((el) => el.setAttribute("data-mutated", "yes")), + ], + ]; + for (const [name, run] of methods) { + await check(`strict ${name}`, async () => { + await assert.rejects(run(page.locator(".duplicate")), /strict mode violation/); + await assert.rejects(run(facade.locator(".duplicate")), /strict mode violation/); + }); + } + await check("ambiguous callbacks never execute", async () => { + assert.equal(await page.locator("[data-mutated]").count(), 0); + }); + for (const [name, scope] of [ + ["native", page], + ["facade", facade], + ] as const) { + await check(`${name} waits for delayed element`, async () => { + const id = `delayed-${name}`; + await page.evaluate((id) => { + setTimeout(() => { + const el = document.createElement("input"); + el.id = id; + el.value = "arrived"; + document.body.append(el); + }, 120); + }, id); + assert.equal(await scope.locator(`#${id}`).inputValue({ timeout: 1000 }), "arrived"); + }); + await check(`${name} honors explicit timeout`, async () => { + const start = Date.now(); + await assert.rejects( + scope.locator("#missing").textContent({ timeout: 150 }), + /timed out|Timeout/, + ); + assert.ok(Date.now() - start >= 100 && Date.now() - start < 1500); + }); + await check(`${name} timeout zero waits`, async () => { + const id = `unlimited-${name}`; + await page.evaluate((id) => { + setTimeout(() => { + const el = document.createElement("div"); + el.id = id; + el.textContent = "arrived"; + document.body.append(el); + }, 120); + }, id); + assert.equal(await scope.locator(`#${id}`).textContent({ timeout: 0 }), "arrived"); + }); + } + await check("collection reads and visibility stay immediate", async () => { + assert.deepEqual(await facade.locator(".duplicate").allTextContents(), ["", ""]); + assert.equal(await facade.locator("#missing").count(), 0); + assert.equal(await facade.locator("#missing").isVisible(), false); + }); + await check("nth disambiguates reads", async () => { + assert.equal(await facade.locator(".duplicate").nth(1).inputValue(), "two"); + }); + await check("unsupported trial never clicks, including force", async () => { + await page.evaluate(() => { + document.body.insertAdjacentHTML( + "beforeend", + '', + ); + }); + for (const force of [false, true]) { + await assert.rejects( + facade.locator("#trial").click({ trial: true, force }), + /trial clicks are not supported/, + ); + } + assert.equal(await page.locator("#trial").getAttribute("data-clicked"), null); + }); + assert.ok( + results.every((result) => result.pass), + "Facade locator behavior differs from Playwright", + ); + } finally { + await page.close(); + } + }); +}); diff --git a/packages/integrations/core/package.json b/packages/integrations/core/package.json index 88d771559..d81a79eef 100644 --- a/packages/integrations/core/package.json +++ b/packages/integrations/core/package.json @@ -29,7 +29,8 @@ "build": "tsdown", "test": "pnpm run build && vitest run --root ../../.. packages/integrations/core/tests", "test:unit": "vitest run --root ../../.. packages/integrations/core/tests", - "typecheck": "tsc --noEmit -p tsconfig.json" + "typecheck": "tsc --noEmit -p tsconfig.json", + "test:browser": "vitest run --root ../../.. --config packages/integrations/core/vitest.browser.config.ts" }, "dependencies": { "@browserbasehq/stagehand": "workspace:*", @@ -38,6 +39,7 @@ }, "devDependencies": { "@types/node": "catalog:", + "playwright": ">=1.55.1 <1.57.0", "tsdown": "catalog:", "typescript": "catalog:", "vitest": "catalog:" diff --git a/packages/integrations/core/src/facade/runtime.ts b/packages/integrations/core/src/facade/runtime.ts index e610e543f..cba136e30 100644 --- a/packages/integrations/core/src/facade/runtime.ts +++ b/packages/integrations/core/src/facade/runtime.ts @@ -27,7 +27,24 @@ type QueryStep = hasNot?: QueryStep[]; visible?: boolean; } - | { kind: "nth"; index: number }; + | { kind: "nth"; index: number } + /** + * A `role` step that was resolved against the browser's accessibility tree. + * `values` are document-relative XPaths for the nodes whose role and name + * matched; the state filters are carried over from the original role step. + */ + | { + kind: "xpaths"; + values: string[]; + checked?: boolean; + disabled?: boolean; + selected?: boolean; + expanded?: boolean; + pressed?: boolean; + level?: number; + }; + +type RoleStep = Extract; type RawLocator = { click(options?: { button?: "left" | "right" | "middle"; clickCount?: number }): Promise; @@ -36,6 +53,15 @@ type RawLocator = { type(text: string, options?: { delay?: number }): Promise; selectOption(values: string | string[]): Promise; setInputFiles(files: unknown): Promise; + count(): Promise; + nth(index: number): RawLocator; + isVisible(): Promise; + isChecked(): Promise; + inputValue(): Promise; + innerText(): Promise; + innerHtml(): Promise; + textContent(): Promise; + scrollTo(percent: number): Promise; }; type CompatSelectOption = @@ -119,7 +145,7 @@ export type PlaywrightCompatRuntime = { * extension service worker with Function#toString. */ export type PlaywrightCompatRuntimeOptions = { - /** Host-owned pages excluded from the agent context and page events. */ + /** Pages the host keeps for itself (the facade's keeper tab); never surfaced to agent code. */ hiddenPageIds?: string[]; }; @@ -148,6 +174,137 @@ export async function createPlaywrightCompatRuntime( ? { kind: "regexp", source: value.source, flags: value.flags } : { kind: "string", value: String(value), exact }; + // --------------------------------------------------------------------------- + // getByRole fallback through the accessibility tree. + // + // The in-page role matcher reimplements accessible-name computation and + // disagrees with Chrome's on real sites (descendant aria-label / alt / svg + // titles, labelledby across shadow roots, custom elements). When a plan that + // contains a role step matches nothing in the DOM, resolve the role step + // against `page.snapshot()` — the same accessibility tree the `snapshot` + // tool shows the agent — and re-run the plan with those nodes' XPaths. + // --------------------------------------------------------------------------- + + /** Playwright role → roles as they appear in Stagehand's formatted tree. */ + const ACCESSIBILITY_ROLE_ALIASES: Record = { + img: ["image", "img"], + image: ["image", "img"], + textbox: ["textbox", "searchbox"], + cell: ["cell", "gridcell"], + gridcell: ["gridcell", "cell"], + }; + + const ACCESSIBILITY_FALLBACK_CACHE_TTL_MS = 750; + + type AccessibilityTreeNode = { id: string; role: string; name: string }; + + const parseAccessibilityTree = (formattedTree: string): AccessibilityTreeNode[] => { + const nodes: AccessibilityTreeNode[] = []; + for (const rawLine of formattedTree.split("\n")) { + const line = rawLine.match(/^\s*\[([^\]]+)\]\s+(.*)$/u); + if (!line) continue; + let rest = line[2] ?? ""; + // Trailing state flags rendered by formatStateFlags. + rest = rest.replace(/(?:\s\[(?:selected|checked)\])+$/u, ""); + const separator = rest.indexOf(": "); + const roleToken = separator === -1 ? rest : rest.slice(0, separator); + const name = separator === -1 ? "" : rest.slice(separator + 2); + // "scrollable, html" style lines carry the role before the comma. + const role = roleToken.split(",")[0]?.trim() ?? ""; + if (!role) continue; + nodes.push({ id: line[1] ?? "", role, name }); + } + return nodes; + }; + + const matchesAccessibleName = (value: string, expected: JsonMatcher): boolean => { + const normalized = value.replace(/\s+/gu, " ").trim(); + if (expected.kind === "regexp") { + return new RegExp(expected.source, expected.flags).test(normalized); + } + const target = expected.value.replace(/\s+/gu, " ").trim(); + return expected.exact + ? normalized === target + : normalized.toLocaleLowerCase().includes(target.toLocaleLowerCase()); + }; + + const planHasRoleStep = (plan: QueryStep[]): boolean => + plan.some( + (step) => + step.kind === "role" || + (step.kind === "filter" && + ((step.has && planHasRoleStep(step.has)) || + (step.hasNot && planHasRoleStep(step.hasNot)))), + ); + + const resolveRoleStepWithTree = ( + step: RoleStep, + nodes: AccessibilityTreeNode[], + xpathMap: Record, + ): QueryStep | null => { + const roles = new Set(ACCESSIBILITY_ROLE_ALIASES[step.role] ?? [step.role]); + const values = nodes + .filter( + (node) => + roles.has(node.role) && (!step.name || matchesAccessibleName(node.name, step.name)), + ) + .map((node) => xpathMap[node.id]) + .filter((xpath): xpath is string => typeof xpath === "string" && xpath.length > 0); + if (values.length === 0) return null; + const { kind: _kind, role: _role, name: _name, includeHidden: _hidden, ...state } = step; + return { kind: "xpaths", values, ...state }; + }; + + /** + * Returns a copy of `plan` whose top-level role steps are replaced by + * accessibility-tree resolved XPath steps, or null when the tree has no + * candidate for at least one of them (or no tree is available). + */ + const resolvePlanWithAccessibilityTree = async ( + page: RawPage, + plan: QueryStep[], + ): Promise => { + if (typeof page.snapshot !== "function") { + record("misses", "getByRole.accessibilityTree:snapshotUnavailable"); + return null; + } + let snapshot: unknown; + try { + snapshot = await page.snapshot({ includeIframes: false }); + } catch { + record("misses", "getByRole.accessibilityTree:snapshotError"); + return null; + } + const tree = snapshot as { formattedTree?: unknown; xpathMap?: unknown } | null; + if ( + !tree || + typeof tree.formattedTree !== "string" || + !tree.xpathMap || + typeof tree.xpathMap !== "object" + ) { + record("misses", "getByRole.accessibilityTree:noTree"); + return null; + } + const nodes = parseAccessibilityTree(tree.formattedTree); + const xpathMap = tree.xpathMap as Record; + let replaced = false; + const resolved: QueryStep[] = []; + for (const step of plan) { + if (step.kind !== "role") { + resolved.push(step); + continue; + } + const fallback = resolveRoleStepWithTree(step, nodes, xpathMap); + if (!fallback) { + record("misses", "getByRole.accessibilityTree:noCandidates"); + return null; + } + resolved.push(fallback); + replaced = true; + } + return replaced ? resolved : null; + }; + const unsupported = (surface: string, method: PropertyKey): never => { const name = `${surface}.${String(method)}`; record("misses", name); @@ -170,6 +327,7 @@ export async function createPlaywrightCompatRuntime( plan?: QueryStep[]; operation: | "inspect" + | "describe" | "tag" | "tagAll" | "untag" @@ -198,6 +356,7 @@ export async function createPlaywrightCompatRuntime( attribute?: string; functionSource?: string; argument?: unknown; + strict?: boolean; }): Promise { type QueryRoot = Document | Element | ShadowRoot; @@ -257,6 +416,72 @@ export async function createPlaywrightCompatRuntime( } return elements; }; + /** + * Resolve an XPath produced by Stagehand's accessibility snapshot. Those + * paths are positional (`/html[1]/body[1]/x-host[1]//div[2]/button[1]`) + * and encode a shadow-root boundary as `//`, which native + * `document.evaluate` cannot follow. Mirrors the extension's + * resolveStagehandShadowHopMatches: child steps walk light-DOM children, + * a `//` step after the first walks into the host's (open) shadow root. + */ + const resolveStagehandXPath = (expression: string): Element[] => { + const path = expression.trim().replace(/^xpath=/iu, ""); + if (!path) return []; + type Step = { hop: boolean; tag: string; index?: number }; + const steps: Step[] = []; + let cursor = 0; + while (cursor < path.length) { + let hop = false; + if (path.startsWith("//", cursor)) { + hop = true; + cursor += 2; + } else if (path[cursor] === "/") { + cursor += 1; + } + const start = cursor; + while (cursor < path.length && path[cursor] !== "/") cursor += 1; + const raw = path.slice(start, cursor).trim(); + if (!raw) continue; + const parsed = raw.match(/^([^[]+)(?:\[(\d+)\])?$/u); + if (!parsed) return queryXPath(document, path); + steps.push({ + hop, + tag: (parsed[1] ?? "*").toLowerCase(), + ...(parsed[2] ? { index: Number(parsed[2]) } : {}), + }); + } + const hasShadowHop = steps.some((step, position) => step.hop && position > 0); + if (!hasShadowHop) { + try { + return queryXPath(document, path); + } catch { + return []; + } + } + let current: Array = [document]; + for (const [position, step] of steps.entries()) { + const next: Element[] = []; + for (const root of current) { + let pool: Element[]; + if (root instanceof Document) { + pool = root.documentElement ? [root.documentElement] : []; + } else if (step.hop && position > 0) { + pool = root instanceof Element ? [...(root.shadowRoot?.children ?? [])] : []; + } else { + pool = [...root.children]; + } + const tagged = pool.filter( + (element) => step.tag === "*" || element.localName.toLowerCase() === step.tag, + ); + const picked = + step.index === undefined ? tagged : [tagged[step.index - 1]!].filter(Boolean); + for (const element of picked) if (!next.includes(element)) next.push(element); + } + if (!next.length) return []; + current = next; + } + return current as Element[]; + }; const splitSelectorList = (selector: string): string[] => { const parts: string[] = []; let start = 0; @@ -383,6 +608,52 @@ export async function createPlaywrightCompatRuntime( } return ""; }; + const labelNodeText = (node: Node): string => { + if ( + ["SCRIPT", "STYLE", "NOSCRIPT"].includes(node.nodeName) || + node.ownerDocument?.head?.contains(node) + ) { + return ""; + } + if (node instanceof HTMLInputElement && (node.type === "submit" || node.type === "button")) { + return node.value; + } + let text = ""; + for (const child of node.childNodes) { + if (child.nodeType === Node.TEXT_NODE) text += child.nodeValue ?? ""; + else if (child.nodeType === Node.ELEMENT_NODE) text += labelNodeText(child); + } + if (node instanceof Element && node.shadowRoot) text += labelNodeText(node.shadowRoot); + return text; + }; + // Label locators match each label separately, with labelledby > aria-label >