From 5c2670d10c6ee0b4af185e0be4ed959163bca0fb Mon Sep 17 00:00:00 2001 From: miguel Date: Thu, 17 Sep 2026 00:05:30 -0700 Subject: [PATCH 1/2] Reorder #2895: preserve its changes after session ownership --- packages/extension/callbackBatch.ts | 7 + packages/extension/errors.ts | 14 + packages/extension/runtime.ts | 9 + .../tests/shadow-root-evaluation.test.ts | 73 ++ packages/extension/understudy/page.ts | 10 + .../understudy/shadowRootEvaluation.ts | 53 ++ .../core/integration/facade-dom.test.ts | 470 ++++++++++ packages/integrations/core/package.json | 4 +- .../integrations/core/src/facade/contract.ts | 2 +- .../integrations/core/src/facade/runtime.ts | 879 +++++++++++++++--- .../core/tests/facade-locator-actions.test.ts | 98 ++ .../core/tests/facade-wait-for-url.test.ts | 189 ++++ packages/integrations/core/tsconfig.json | 2 +- .../core/vitest.browser.config.ts | 9 + .../extensionassets/stagehand-extension.zip | Bin 440798 -> 441540 bytes pnpm-lock.yaml | 3 + turbo.json | 6 + 17 files changed, 1718 insertions(+), 110 deletions(-) create mode 100644 packages/extension/tests/shadow-root-evaluation.test.ts create mode 100644 packages/extension/understudy/shadowRootEvaluation.ts create mode 100644 packages/integrations/core/integration/facade-dom.test.ts create mode 100644 packages/integrations/core/tests/facade-locator-actions.test.ts create mode 100644 packages/integrations/core/tests/facade-wait-for-url.test.ts create mode 100644 packages/integrations/core/vitest.browser.config.ts diff --git a/packages/extension/callbackBatch.ts b/packages/extension/callbackBatch.ts index d30bee1736..38a646edde 100644 --- a/packages/extension/callbackBatch.ts +++ b/packages/extension/callbackBatch.ts @@ -74,6 +74,7 @@ class InProcessCommandClient implements StagehandCommandClient { export type CallbackStagehand = { page: Page; + evaluateWithShadowRoots(pageId: string, functionSource: string): Promise; context: ExperimentalBatchBrowserContext; act(instruction: string | Action, options?: StagehandClientActOptions): Promise; observe(instruction?: string, options?: StagehandClientObserveOptions): Promise; @@ -127,6 +128,12 @@ export function createCallbackBatchController(router: RPCRouter) { const stagehand: CallbackStagehand = { page, context: createCallbackContextFacade(context), + evaluateWithShadowRoots: async (pageId, functionSource) => { + if (controller.signal.aborted) throw controller.signal.reason; + const result = await router.runtime.evaluateWithShadowRoots(pageId, functionSource); + if (controller.signal.aborted) throw controller.signal.reason; + return result; + }, act: async (instruction, operationOptions) => { const { page: operationPage, ...clientOptions } = StagehandClientActOptionsSchema.parse( operationOptions ?? {}, diff --git a/packages/extension/errors.ts b/packages/extension/errors.ts index 853c5786b7..da709d73e6 100644 --- a/packages/extension/errors.ts +++ b/packages/extension/errors.ts @@ -20,3 +20,17 @@ export class DuplicatePageEventSubscriptionError extends Error { this.name = "DuplicatePageEventSubscriptionError"; } } + +export class ShadowRootEvaluationError extends Error { + constructor() { + super("Shadow-root evaluation failed"); + this.name = "ShadowRootEvaluationError"; + } +} + +export class ShadowRootEvaluationUnavailableError extends Error { + constructor() { + super("Shadow-root evaluation is unavailable"); + this.name = "ShadowRootEvaluationUnavailableError"; + } +} diff --git a/packages/extension/runtime.ts b/packages/extension/runtime.ts index d3765a8ac0..a908c8ed0f 100644 --- a/packages/extension/runtime.ts +++ b/packages/extension/runtime.ts @@ -1,3 +1,4 @@ +import { ShadowRootEvaluationUnavailableError } from "./errors.js"; import type { ClearCookieOptions, ContextActivePageResult, @@ -148,6 +149,7 @@ export type UnderstudyRuntimePage = { type(text: string, options?: PageTypeParams["options"]): Promise; keyPress(key: string, options?: PageKeyPressParams["options"]): Promise; evaluate(expression: string): Promise; + evaluateWithShadowRoots?(functionSource: string): Promise; addInitScript(source: string): Promise; setExtraHTTPHeaders(headers: PageSetExtraHTTPHeadersParams["headers"]): Promise; setViewportSize( @@ -624,6 +626,13 @@ export class StagehandRuntime { return { ok: true }; } + async evaluateWithShadowRoots(pageId: string, functionSource: string): Promise { + const page = this.resolvePage(pageId); + this.logger.debug("page.evaluateWithShadowRoots", { pageId }); + if (!page.evaluateWithShadowRoots) throw new ShadowRootEvaluationUnavailableError(); + return page.evaluateWithShadowRoots(functionSource); + } + async pageEvaluate(params: PageEvaluateParams): Promise { const value = await this.resolvePage(params.pageId).evaluate(params.expression); return { diff --git a/packages/extension/tests/shadow-root-evaluation.test.ts b/packages/extension/tests/shadow-root-evaluation.test.ts new file mode 100644 index 0000000000..e42a00b642 --- /dev/null +++ b/packages/extension/tests/shadow-root-evaluation.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it, vi } from "vitest"; +import { evaluateWithShadowRoots } from "../understudy/shadowRootEvaluation.js"; + +describe("main-world shadow-root evaluation", () => { + it("uses ordinary main-world evaluation when there are no author closed roots", async () => { + const send = vi.fn().mockResolvedValue({ + root: { backendNodeId: 1, shadowRoots: [{ backendNodeId: 2, shadowRootType: "user-agent" }] }, + }); + const evaluate = vi.fn().mockResolvedValue(42); + await expect( + evaluateWithShadowRoots({ send } as never, evaluate, "roots => roots.length"), + ).resolves.toBe(42); + expect(evaluate).toHaveBeenCalledWith("(roots => roots.length)([])"); + expect(send).toHaveBeenCalledOnce(); + }); + + it.each(["DOM.resolveNode", "Runtime.callFunctionOn"])( + "releases references after %s fails", + async (failureMethod) => { + let resolved = 0; + const send = vi.fn(async (method: string) => { + if (method === "DOM.getDocument") + return { + root: { + backendNodeId: 1, + shadowRoots: [ + { backendNodeId: 2, shadowRootType: "closed" }, + { backendNodeId: 3, shadowRootType: "closed" }, + ], + }, + }; + if (method === "DOM.resolveNode") { + resolved += 1; + if (method === failureMethod && resolved === 2) throw new Error("detached"); + return { object: { objectId: `root-${resolved}` } }; + } + if (method === failureMethod) throw new Error("detached"); + return {}; + }); + await expect( + evaluateWithShadowRoots({ send } as never, vi.fn(), "roots => roots.length"), + ).rejects.toThrow("detached"); + expect(send.mock.calls.at(-1)?.[0]).toBe("Runtime.releaseObjectGroup"); + }, + ); + it.each([false, true])( + "preserves query outcome when cleanup fails (callback throws: %s)", + async (throws) => { + const send = vi.fn(async (method: string) => { + if (method === "DOM.getDocument") + return { + root: { + backendNodeId: 1, + shadowRoots: [{ backendNodeId: 2, shadowRootType: "closed" }], + }, + }; + if (method === "DOM.resolveNode") return { object: { objectId: "root" } }; + if (method === "Runtime.releaseObjectGroup") throw new Error("context destroyed"); + return throws + ? { exceptionDetails: { text: "sensitive page data" } } + : { result: { value: 42 } }; + }); + const result = evaluateWithShadowRoots({ send } as never, vi.fn(), "roots => roots.length"); + if (throws) + await expect(result).rejects.toMatchObject({ + name: "ShadowRootEvaluationError", + message: "Shadow-root evaluation failed", + }); + else await expect(result).resolves.toBe(42); + expect(send.mock.calls.at(-1)?.[0]).toBe("Runtime.releaseObjectGroup"); + }, + ); +}); diff --git a/packages/extension/understudy/page.ts b/packages/extension/understudy/page.ts index 6ec5cb30a0..1fd5e650c9 100644 --- a/packages/extension/understudy/page.ts +++ b/packages/extension/understudy/page.ts @@ -2,6 +2,7 @@ import { Protocol } from "devtools-protocol"; import type { StagehandLogger } from "../logger.js"; import type { CDPSessionLike } from "./cdp.js"; import { CdpConnection } from "./cdp.js"; +import { evaluateWithShadowRoots } from "./shadowRootEvaluation.js"; import { Frame } from "./frame.js"; import { FrameLocator } from "./frameLocator.js"; import { deepLocatorFromPage, resolveLocatorTarget } from "./deepLocator.js"; @@ -1435,6 +1436,15 @@ export class Page { return targetFrame.evaluateInLocatorWorld(expression); } + /** Internal batch evaluation; page.evaluate continues to use the main world unchanged. */ + async evaluateWithShadowRoots(functionSource: string): Promise { + return evaluateWithShadowRoots( + this.mainSession, + (expression) => this.evaluate(expression), + functionSource, + ); + } + /** * Evaluate a function or expression in the current main frame's main world. * - If a string is provided, it is treated as a JS expression. diff --git a/packages/extension/understudy/shadowRootEvaluation.ts b/packages/extension/understudy/shadowRootEvaluation.ts new file mode 100644 index 0000000000..5cf2748b78 --- /dev/null +++ b/packages/extension/understudy/shadowRootEvaluation.ts @@ -0,0 +1,53 @@ +import { ShadowRootEvaluationError } from "../errors.js"; +import type { Protocol } from "devtools-protocol"; +import type { CDPSessionLike } from "./cdp.js"; + +/** Evaluate in the main world with temporary references to author closed roots. */ +export async function evaluateWithShadowRoots( + session: CDPSessionLike, + evaluate: (expression: string) => Promise, + functionSource: string, +): Promise { + const { root } = await session.send("DOM.getDocument", { + depth: -1, + pierce: true, + }); + const closedIds: number[] = []; + const visit = (node: Protocol.DOM.Node): void => { + if (node.shadowRootType === "closed") closedIds.push(node.backendNodeId); + for (const child of node.children ?? []) visit(child); + for (const shadow of node.shadowRoots ?? []) { + if (shadow.shadowRootType !== "user-agent") visit(shadow); + } + // Frame documents have separate main worlds and are resolved by frame locators. + }; + visit(root); + if (!closedIds.length) return (await evaluate(`(${functionSource})([])`)) as Result; + const objectGroup = `stagehand-shadow-query-${crypto.randomUUID()}`; + try { + const objects: Array<{ objectId: string }> = []; + for (const backendNodeId of closedIds) { + const { object } = await session.send("DOM.resolveNode", { + backendNodeId, + objectGroup, + }); + if (!object.objectId) throw new ShadowRootEvaluationError(); + objects.push({ objectId: object.objectId }); + } + const response = await session.send( + "Runtime.callFunctionOn", + { + objectId: objects[0]!.objectId, + functionDeclaration: `function(...roots) { return (${functionSource})(roots); }`, + arguments: objects, + awaitPromise: true, + returnByValue: true, + }, + ); + if (response.exceptionDetails) throw new ShadowRootEvaluationError(); + return response.result.value as Result; + } finally { + // Navigation can destroy the group before cleanup; preserve the query outcome. + await session.send("Runtime.releaseObjectGroup", { objectGroup }).catch(() => {}); + } +} diff --git a/packages/integrations/core/integration/facade-dom.test.ts b/packages/integrations/core/integration/facade-dom.test.ts new file mode 100644 index 0000000000..d594746675 --- /dev/null +++ b/packages/integrations/core/integration/facade-dom.test.ts @@ -0,0 +1,470 @@ +import assert from "node:assert/strict"; +import { afterAll, beforeAll, describe, it } from "vitest"; +import { chromium, type Browser, type Locator, type Page } from "playwright"; +import { createPlaywrightCompatRuntime } from "../src/facade/runtime.js"; + +// Local, static DOM fixtures only. No extension, remote site, credentials or model. +// Run separately with pnpm --filter @browserbasehq/stagehand-integrations test:browser. +let browser: Browser; +beforeAll(async () => { + browser = await chromium.launch({ + ...(process.env.CHROME_PATH + ? { executablePath: process.env.CHROME_PATH } + : { channel: process.env.PLAYWRIGHT_CHROMIUM_CHANNEL ?? "chrome" }), + headless: true, + }); +}); +afterAll(async () => { + await browser?.close(); +}); + +describe("facade DOM compatibility against native Playwright", () => { + it("matches native labels, priority, whitespace and shadow roots", async () => { + const page = await browser.newPage(); + try { + await page.setContent(` +
+ + + + + First referenceSecond reference + + + + + + + + + + Wrong outer label
+ `); + await page.locator("#shadow-host").evaluate((host) => { + host.attachShadow({ mode: "open" }).innerHTML = + 'Shadow label' + + ''; + }); + const runtime = await createPlaywrightCompatRuntime({ + page, + context: { pages: async () => [page], activePage: async () => page }, + } as unknown as Parameters[0]); + const facade = runtime.page as Pick; + const cases: Array<{ + name: string; + label: string | RegExp; + exact?: boolean; + scope?: string; + expected: string[]; + }> = [ + { + name: "Booking child age aria-label", + expected: ["age"], + label: "Child 1 age", + exact: true, + }, + { + name: "scoped aria-label", + expected: ["age"], + label: "Child 1 age", + exact: true, + scope: "#booking", + }, + { name: "case-insensitive substring", expected: ["age"], label: "CHILD 1" }, + { name: "exact case sensitivity", expected: [], label: "child 1 age", exact: true }, + { name: "native label", expected: ["native"], label: "Native label", exact: true }, + { name: "wrapped label", expected: ["wrapped"], label: "Wrapped label", exact: true }, + { name: "first associated label", expected: ["multi"], label: "First label", exact: true }, + { + name: "second associated label", + expected: ["multi"], + label: "Second label", + exact: true, + }, + { + name: "labels are not concatenated", + expected: [], + label: "First label Second label", + exact: true, + }, + { name: "first ARIA reference", expected: ["refs"], label: "First reference", exact: true }, + { + name: "second ARIA reference", + expected: ["refs"], + label: "Second reference", + exact: true, + }, + { + name: "ARIA references are not concatenated", + expected: [], + label: "First reference Second reference", + exact: true, + }, + { name: "labelledby takes priority", expected: [], label: "Overridden ARIA", exact: true }, + { + name: "empty referenced label takes priority", + expected: [], + label: "Ignored fallback", + exact: true, + }, + { + name: "ARIA takes priority over native label", + expected: [], + label: "Ignored native", + exact: true, + }, + { name: "ARIA priority match", expected: ["aria-first"], label: "ARIA wins", exact: true }, + { + name: "broken labelledby falls back", + expected: ["broken-ref"], + label: "Fallback label", + exact: true, + }, + { + name: "normalized string whitespace", + expected: ["spaces"], + label: "Child 2 age", + exact: true, + }, + { + name: "empty ARIA falls back", + expected: ["empty-aria"], + label: "Empty ARIA fallback", + exact: true, + }, + { + name: "label text excludes script/style", + expected: ["mixed"], + label: "Clean label", + exact: true, + }, + { name: "regular expression", expected: ["age"], label: /^child 1 age$/i }, + { name: "regex keeps original whitespace", expected: ["regex"], label: /^Line\nBreak$/ }, + { + name: "empty query excludes unlabeled elements", + expected: ["blank-ref"], + label: "", + exact: true, + }, + { + name: "shadow-root reference", + expected: ["shadow-input"], + label: "Shadow label", + exact: true, + }, + { name: "shadow-root ARIA", expected: ["shadow-aria"], label: "Shadow ARIA", exact: true }, + { + name: "reference cannot cross shadow boundary", + expected: [], + label: "Wrong outer label", + exact: true, + }, + ]; + const results = []; + for (const test of cases) { + const nativeScope = test.scope ? page.locator(test.scope) : page; + const facadeScope = test.scope ? facade.locator(test.scope) : facade; + const options = { exact: test.exact }; + const native = await nativeScope + .getByLabel(test.label, options) + .evaluateAll((els) => els.map((el) => el.id)); + const actual = await facadeScope + .getByLabel(test.label, options) + .evaluateAll((els) => els.map((el) => el.id)); + results.push({ + name: test.name, + native, + actual, + expected: test.expected, + pass: + JSON.stringify(native) === JSON.stringify(actual) && + JSON.stringify(actual) === JSON.stringify(test.expected), + }); + } + assert.ok( + results.every((r) => r.pass), + `Facade label mismatch: ${JSON.stringify(results.filter((result) => !result.pass))}`, + ); + await facade.getByLabel("Child 1 age", { exact: true }).selectOption("8"); + assert.equal(await page.locator("#age").inputValue(), "8"); + } finally { + await page.close(); + } + }); + it("resolves nested role filters before applying hasNot or invoking callbacks", async () => { + const page = await browser.newPage(); + try { + await page.setContent(` +
+
+ `); + // This snapshot is the browser-computed name that the DOM approximation + // misses for image-only buttons. Native Playwright independently checks it. + const rawPage = { + url: () => page.url(), + evaluate: page.evaluate.bind(page), + snapshot: async () => ({ + formattedTree: "[1] button: Pay now\n[2] button: Cancel order", + xpathMap: { + "1": "/html/body/section[1]/div/button", + "2": "/html/body/section[2]/div/button", + }, + }), + }; + const runtime = await createPlaywrightCompatRuntime({ + page: rawPage, + context: { pages: async () => [rawPage] }, + } as unknown as Parameters[0]); + const facade = runtime.page as Page; + for (const scope of [page, facade]) { + const pay = scope.getByRole("button", { name: "Pay now", exact: true }); + assert.deepEqual( + await scope + .locator("section") + .filter({ has: pay }) + .evaluateAll((els) => els.map((el) => el.id)), + ["pay"], + ); + assert.deepEqual( + await scope + .locator("section") + .filter({ hasNot: pay }) + .evaluateAll((els) => els.map((el) => el.id)), + ["cancel"], + ); + assert.deepEqual( + await scope + .locator("section") + .filter({ + has: scope.locator("div").filter({ has: pay }), + }) + .evaluateAll((els) => els.map((el) => el.id)), + ["pay"], + ); + assert.deepEqual( + await scope + .locator("section") + .filter({ + hasNot: scope.getByRole("button", { name: "Missing", exact: true }), + }) + .evaluateAll((els) => els.map((el) => el.id)), + ["pay", "cancel"], + ); + } + await facade + .locator("section") + .filter({ + hasNot: facade.getByRole("button", { + name: "Pay now", + exact: true, + }), + }) + .evaluate((el) => el.setAttribute("data-mutated", "yes")); + assert.deepEqual( + await page.locator("[data-mutated]").evaluateAll((els) => els.map((el) => el.id)), + ["cancel"], + ); + } finally { + await page.close(); + } + }); + it.each(["open", "closed"] as const)( + "keeps scoped role matches inside nested %s shadow roots", + async (mode) => { + const page = await browser.newPage(); + const cdp = await page.context().newCDPSession(page); + const { evaluateWithShadowRoots } = await import( + new URL("../../../extension/understudy/shadowRootEvaluation.ts", import.meta.url).href + ); + try { + await page.setContent( + '
', + ); + await page.evaluate((mode) => { + (window as unknown as { pageOwnedValue: number }).pageOwnedValue = 42; + const outer = document.querySelector("#host")!.attachShadow({ mode }); + outer.innerHTML = '
'; + const inner = outer.querySelector("#nested")!.attachShadow({ mode }); + inner.innerHTML = + ''; + }, mode); + const rawPage = { + pageId: "fixture", + url: () => page.url(), + evaluate: page.evaluate.bind(page), + snapshot: async () => ({ + formattedTree: "[1] button: Pay now", + xpathMap: { "1": "/html/body/section[1]/div[1]//div[1]//button[1]" }, + }), + }; + const runtime = await createPlaywrightCompatRuntime({ + page: rawPage, + context: { pages: async () => [rawPage] }, + evaluateWithShadowRoots: (_pageId: string, source: string) => + evaluateWithShadowRoots(cdp, (expression: string) => page.evaluate(expression), source), + } as unknown as Parameters[0]); + const facade = runtime.page as Pick; + const pay = facade.getByRole("button", { name: "Pay now", exact: true }); + assert.equal(await pay.count(), 1); + assert.equal( + await facade.locator("#inside").getByRole("button", { name: "Pay now" }).count(), + 1, + ); + assert.equal( + await facade.locator("#outside").getByRole("button", { name: "Pay now" }).count(), + 0, + ); + assert.deepEqual( + await facade + .locator("section") + .filter({ has: pay }) + .evaluateAll((els) => els.map((el) => el.id)), + ["inside"], + ); + assert.deepEqual( + await facade + .locator("section") + .filter({ hasNot: pay }) + .evaluateAll((els) => els.map((el) => el.id)), + ["outside"], + ); + assert.equal(await facade.getByLabel("Amount").inputValue(), "10"); + assert.equal(await facade.locator("#host").locator("#pay").count(), 1); + assert.equal( + await pay.evaluate((el) => { + el.setAttribute("data-checked", "yes"); + return (window as unknown as { pageOwnedValue: number }).pageOwnedValue; + }), + 42, + ); + assert.equal(await facade.locator('[data-checked="yes"]').count(), 1); + // Fresh roots after navigation must not reuse remote references. + await page.goto("about:blank"); + assert.equal(await facade.locator("#pay").count(), 0); + } finally { + await cdp.detach(); + await page.close(); + } + }, + ); + + it("enforces strict reads, bounded waits and non-dispatched trial actions", async () => { + const page = await browser.newPage(); + try { + await page.setContent( + '', + ); + const runtime = await createPlaywrightCompatRuntime({ + page, + context: { pages: async () => [page], activePage: async () => page }, + } as unknown as Parameters[0]); + const facade = runtime.page as Page; + const results: Array<{ name: string; pass: boolean; detail?: string }> = []; + const check = async (name: string, action: () => Promise): Promise => { + try { + await action(); + results.push({ name, pass: true }); + } catch (error) { + results.push({ name, pass: false, detail: String(error) }); + } + }; + // Strictness should be reported before a busy CI browser exhausts its command budget. + const strictTimeout = 5000; + const methods: Array<[string, (locator: Locator) => Promise]> = [ + ["textContent", (locator) => locator.textContent({ timeout: strictTimeout })], + ["innerText", (locator) => locator.innerText({ timeout: strictTimeout })], + ["innerHTML", (locator) => locator.innerHTML({ timeout: strictTimeout })], + ["inputValue", (locator) => locator.inputValue({ timeout: strictTimeout })], + ["getAttribute", (locator) => locator.getAttribute("value", { timeout: strictTimeout })], + ["isChecked", (locator) => locator.isChecked({ timeout: strictTimeout })], + ["isDisabled", (locator) => locator.isDisabled({ timeout: strictTimeout })], + ["isEnabled", (locator) => locator.isEnabled({ timeout: strictTimeout })], + ["isVisible", (locator) => locator.isVisible()], + ["boundingBox", (locator) => locator.boundingBox({ timeout: strictTimeout })], + ["focus", (locator) => locator.focus({ timeout: strictTimeout })], + ["evaluate", (locator) => locator.evaluate((el) => el.setAttribute("data-mutated", "yes"))], + [ + "evaluateHandle", + (locator) => locator.evaluateHandle((el) => el.setAttribute("data-mutated", "yes")), + ], + ]; + for (const [name, run] of methods) { + await check(`strict ${name}`, async () => { + await assert.rejects(run(page.locator(".duplicate")), /strict mode violation/); + await assert.rejects(run(facade.locator(".duplicate")), /strict mode violation/); + }); + } + await check("ambiguous callbacks never execute", async () => { + assert.equal(await page.locator("[data-mutated]").count(), 0); + }); + for (const [name, scope] of [ + ["native", page], + ["facade", facade], + ] as const) { + await check(`${name} waits for delayed element`, async () => { + const id = `delayed-${name}`; + await page.evaluate((id) => { + setTimeout(() => { + const el = document.createElement("input"); + el.id = id; + el.value = "arrived"; + document.body.append(el); + }, 120); + }, id); + assert.equal(await scope.locator(`#${id}`).inputValue({ timeout: 1000 }), "arrived"); + }); + await check(`${name} honors explicit timeout`, async () => { + const start = Date.now(); + await assert.rejects( + scope.locator("#missing").textContent({ timeout: 150 }), + /timed out|Timeout/, + ); + assert.ok(Date.now() - start >= 100 && Date.now() - start < 1500); + }); + await check(`${name} timeout zero waits`, async () => { + const id = `unlimited-${name}`; + await page.evaluate((id) => { + setTimeout(() => { + const el = document.createElement("div"); + el.id = id; + el.textContent = "arrived"; + document.body.append(el); + }, 120); + }, id); + assert.equal(await scope.locator(`#${id}`).textContent({ timeout: 0 }), "arrived"); + }); + } + await check("collection reads and visibility stay immediate", async () => { + assert.deepEqual(await facade.locator(".duplicate").allTextContents(), ["", ""]); + assert.equal(await facade.locator("#missing").count(), 0); + assert.equal(await facade.locator("#missing").isVisible(), false); + }); + await check("nth disambiguates reads", async () => { + assert.equal(await facade.locator(".duplicate").nth(1).inputValue(), "two"); + }); + await check("unsupported trial never clicks, including force", async () => { + await page.evaluate(() => { + document.body.insertAdjacentHTML( + "beforeend", + '', + ); + }); + for (const force of [false, true]) { + await assert.rejects( + facade.locator("#trial").click({ trial: true, force }), + /trial clicks are not supported/, + ); + } + assert.equal(await page.locator("#trial").getAttribute("data-clicked"), null); + }); + assert.ok( + results.every((result) => result.pass), + `Facade locator mismatch: ${JSON.stringify(results.filter((result) => !result.pass))}`, + ); + } finally { + await page.close(); + } + }); +}); diff --git a/packages/integrations/core/package.json b/packages/integrations/core/package.json index f12516bb8e..d17505aad9 100644 --- a/packages/integrations/core/package.json +++ b/packages/integrations/core/package.json @@ -29,7 +29,8 @@ "build": "tsdown", "test": "pnpm run build && vitest run --root ../../.. packages/integrations/core/tests", "test:unit": "vitest run --root ../../.. packages/integrations/core/tests", - "typecheck": "tsc --noEmit -p tsconfig.json" + "typecheck": "tsc --noEmit -p tsconfig.json", + "test:browser": "vitest run --root ../../.. --config packages/integrations/core/vitest.browser.config.ts" }, "dependencies": { "@browserbasehq/stagehand": "workspace:*", @@ -38,6 +39,7 @@ }, "devDependencies": { "@types/node": "catalog:", + "playwright": ">=1.55.1 <1.57.0", "tsdown": "catalog:", "typescript": "catalog:", "vitest": "catalog:" diff --git a/packages/integrations/core/src/facade/contract.ts b/packages/integrations/core/src/facade/contract.ts index fcf27ed221..d11bc74fdb 100644 --- a/packages/integrations/core/src/facade/contract.ts +++ b/packages/integrations/core/src/facade/contract.ts @@ -202,7 +202,7 @@ export function facadeAgentInstructions(env: NodeJS.ProcessEnv = process.env): s */ export const FACADE_AGENT_INSTRUCTIONS = `Browser tool surface: Stagehand Playwright facade. You control one persistent browser through exactly three tools: -- run: execute JavaScript against an initialized Playwright page, context, and browser (page.goto, page.locator(selector).click()/fill(), page.getByRole(...), page.evaluate(...), and the supported Playwright-shaped API). Use await directly and return JSON-serializable values so you can inspect progress. Alternatively, pass snapshot actions. +- run: execute JavaScript against an initialized Playwright page, context, and browser (page.goto, page.locator(selector).click()/fill(), page.getByRole(...), page.evaluate(...), page.waitForURL(...), and the supported Playwright-shaped API). Use await directly and return JSON-serializable values so you can inspect progress. Alternatively, pass snapshot actions. - snapshot: inspect the active page's accessibility tree and hydrate bracketed element IDs for run actions. - screenshot: inspect the rendered page visually. diff --git a/packages/integrations/core/src/facade/runtime.ts b/packages/integrations/core/src/facade/runtime.ts index ef6d171051..2004b2e4b3 100644 --- a/packages/integrations/core/src/facade/runtime.ts +++ b/packages/integrations/core/src/facade/runtime.ts @@ -27,7 +27,24 @@ type QueryStep = hasNot?: QueryStep[]; visible?: boolean; } - | { kind: "nth"; index: number }; + | { kind: "nth"; index: number } + /** + * A `role` step that was resolved against the browser's accessibility tree. + * `values` are document-relative XPaths for the nodes whose role and name + * matched; the state filters are carried over from the original role step. + */ + | { + kind: "xpaths"; + values: string[]; + checked?: boolean; + disabled?: boolean; + selected?: boolean; + expanded?: boolean; + pressed?: boolean; + level?: number; + }; + +type RoleStep = Extract; type RawLocator = { click(options?: { button?: "left" | "right" | "middle"; clickCount?: number }): Promise; @@ -36,6 +53,15 @@ type RawLocator = { type(text: string, options?: { delay?: number }): Promise; selectOption(values: string | string[]): Promise; setInputFiles(files: unknown): Promise; + count(): Promise; + nth(index: number): RawLocator; + isVisible(): Promise; + isChecked(): Promise; + inputValue(): Promise; + innerText(): Promise; + innerHtml(): Promise; + textContent(): Promise; + scrollTo(percent: number): Promise; }; type CompatSelectOption = @@ -97,7 +123,11 @@ type RawContext = { }; }; -type BatchStagehandRuntime = { page: RawPage; context: RawContext }; +type BatchStagehandRuntime = { + page: RawPage; + context: RawContext; + evaluateWithShadowRoots?(pageId: string, functionSource: string): Promise; +}; export type PlaywrightCompatTelemetry = { calls: Record; @@ -119,7 +149,7 @@ export type PlaywrightCompatRuntime = { * extension service worker with Function#toString. */ export type PlaywrightCompatRuntimeOptions = { - /** Host-owned pages excluded from the agent context and page events. */ + /** Pages the host keeps for itself (the facade's keeper tab); never surfaced to agent code. */ hiddenPageIds?: string[]; }; @@ -148,6 +178,139 @@ export async function createPlaywrightCompatRuntime( ? { kind: "regexp", source: value.source, flags: value.flags } : { kind: "string", value: String(value), exact }; + // --------------------------------------------------------------------------- + // getByRole fallback through the accessibility tree. + // + // The in-page role matcher reimplements accessible-name computation and + // disagrees with Chrome's on real sites (descendant aria-label / alt / svg + // titles, labelledby across shadow roots, custom elements). When a plan that + // contains a role step matches nothing in the DOM, resolve the role step + // against `page.snapshot()` — the same accessibility tree the `snapshot` + // tool shows the agent — and re-run the plan with those nodes' XPaths. + // --------------------------------------------------------------------------- + + /** Playwright role → roles as they appear in Stagehand's formatted tree. */ + const ACCESSIBILITY_ROLE_ALIASES: Record = { + img: ["image", "img"], + image: ["image", "img"], + textbox: ["textbox", "searchbox"], + cell: ["cell", "gridcell"], + gridcell: ["gridcell", "cell"], + }; + + const ACCESSIBILITY_FALLBACK_CACHE_TTL_MS = 750; + + type AccessibilityTreeNode = { id: string; role: string; name: string }; + + const parseAccessibilityTree = (formattedTree: string): AccessibilityTreeNode[] => { + const nodes: AccessibilityTreeNode[] = []; + for (const rawLine of formattedTree.split("\n")) { + const line = rawLine.match(/^\s*\[([^\]]+)\]\s+(.*)$/u); + if (!line) continue; + let rest = line[2] ?? ""; + // Trailing state flags rendered by formatStateFlags. + rest = rest.replace(/(?:\s\[(?:selected|checked)\])+$/u, ""); + const separator = rest.indexOf(": "); + const roleToken = separator === -1 ? rest : rest.slice(0, separator); + const name = separator === -1 ? "" : rest.slice(separator + 2); + // "scrollable, html" style lines carry the role before the comma. + const role = roleToken.split(",")[0]?.trim() ?? ""; + if (!role) continue; + nodes.push({ id: line[1] ?? "", role, name }); + } + return nodes; + }; + + const matchesAccessibleName = (value: string, expected: JsonMatcher): boolean => { + const normalized = value.replace(/\s+/gu, " ").trim(); + if (expected.kind === "regexp") { + return new RegExp(expected.source, expected.flags).test(normalized); + } + const target = expected.value.replace(/\s+/gu, " ").trim(); + return expected.exact + ? normalized === target + : normalized.toLocaleLowerCase().includes(target.toLocaleLowerCase()); + }; + + const planHasRoleStep = (plan: QueryStep[]): boolean => + plan.some( + (step) => + step.kind === "role" || + (step.kind === "filter" && + ((step.has && planHasRoleStep(step.has)) || + (step.hasNot && planHasRoleStep(step.hasNot)))), + ); + + const resolveRoleStepWithTree = ( + step: RoleStep, + nodes: AccessibilityTreeNode[], + xpathMap: Record, + ): QueryStep => { + const roles = new Set(ACCESSIBILITY_ROLE_ALIASES[step.role] ?? [step.role]); + const values = nodes + .filter( + (node) => + roles.has(node.role) && (!step.name || matchesAccessibleName(node.name, step.name)), + ) + .map((node) => xpathMap[node.id]) + .filter((xpath): xpath is string => typeof xpath === "string" && xpath.length > 0); + if (values.length === 0) record("misses", "getByRole.accessibilityTree:noCandidates"); + const { kind: _kind, role: _role, name: _name, includeHidden: _hidden, ...state } = step; + return { kind: "xpaths", values, ...state }; + }; + + /** + * Resolve role steps recursively, including has/hasNot filters. Empty XPath + * sets preserve negative-filter semantics when no role candidate exists. + * Return null only when no tree is available or the plan contains no roles. + */ + const resolvePlanWithAccessibilityTree = async ( + page: RawPage, + plan: QueryStep[], + ): Promise => { + if (typeof page.snapshot !== "function") { + record("misses", "getByRole.accessibilityTree:snapshotUnavailable"); + return null; + } + let snapshot: unknown; + try { + snapshot = await page.snapshot({ includeIframes: false }); + } catch { + record("misses", "getByRole.accessibilityTree:snapshotError"); + return null; + } + const tree = snapshot as { formattedTree?: unknown; xpathMap?: unknown } | null; + if ( + !tree || + typeof tree.formattedTree !== "string" || + !tree.xpathMap || + typeof tree.xpathMap !== "object" + ) { + record("misses", "getByRole.accessibilityTree:noTree"); + return null; + } + const nodes = parseAccessibilityTree(tree.formattedTree); + const xpathMap = tree.xpathMap as Record; + let replaced = false; + const resolve = (steps: QueryStep[]): QueryStep[] => + steps.map((step) => { + if (step.kind === "role") { + replaced = true; + return resolveRoleStepWithTree(step, nodes, xpathMap); + } + if (step.kind === "filter") { + return { + ...step, + ...(step.has ? { has: resolve(step.has) } : {}), + ...(step.hasNot ? { hasNot: resolve(step.hasNot) } : {}), + }; + } + return step; + }); + const resolved = resolve(plan); + return replaced ? resolved : null; + }; + const unsupported = (surface: string, method: PropertyKey): never => { const name = `${surface}.${String(method)}`; record("misses", name); @@ -166,39 +329,44 @@ export async function createPlaywrightCompatRuntime( // This function is serialized independently by Stagehand page.evaluate, so // every query helper must remain nested inside it rather than closing over // the callback-batch scope. - async function executeQueryInPage(input: { - plan?: QueryStep[]; - operation: - | "inspect" - | "tag" - | "tagAll" - | "untag" - | "textContent" - | "innerText" - | "innerHTML" - | "inputValue" - | "isChecked" - | "isDisabled" - | "isEnabled" - | "getAttribute" - | "boundingBox" - | "focus" - | "blur" - | "selectText" - | "domClick" - | "scrollIntoView" - | "allTextContents" - | "allInnerTexts" - | "evaluate" - | "evaluateAll" - | "pageContent" - | "pageEvaluateHandle" - | "elementEvaluateHandle"; - token?: string; - attribute?: string; - functionSource?: string; - argument?: unknown; - }): Promise { + async function executeQueryInPage( + input: { + plan?: QueryStep[]; + operation: + | "inspect" + | "describe" + | "tag" + | "tagAll" + | "untag" + | "textContent" + | "innerText" + | "innerHTML" + | "inputValue" + | "isChecked" + | "isDisabled" + | "isEnabled" + | "getAttribute" + | "boundingBox" + | "focus" + | "blur" + | "selectText" + | "domClick" + | "scrollIntoView" + | "allTextContents" + | "allInnerTexts" + | "evaluate" + | "evaluateAll" + | "pageContent" + | "pageEvaluateHandle" + | "elementEvaluateHandle"; + token?: string; + attribute?: string; + functionSource?: string; + argument?: unknown; + strict?: boolean; + }, + closedRoots: ShadowRoot[] = [], + ): Promise { type QueryRoot = Document | Element | ShadowRoot; const normalize = (value: string): string => value.replace(/\s+/gu, " ").trim(); @@ -226,13 +394,25 @@ export async function createPlaywrightCompatRuntime( return true; }); }; + const closedRootsByHost = new Map(closedRoots.map((root) => [root.host, root])); + const shadowRootFor = (element: Element): ShadowRoot | null => + element.shadowRoot ?? closedRootsByHost.get(element) ?? null; + const containsAcrossShadowRoots = (root: QueryRoot, element: Element): boolean => { + let node: Node | null = element; + while (node) { + if (node === root) return true; + node = node instanceof ShadowRoot ? node.host : node.parentNode; + } + return false; + }; const queryCssDeep = (root: QueryRoot, selector: string): Element[] => { const direct = [...root.querySelectorAll(selector)]; - const ownShadow = - root instanceof Element && root.shadowRoot ? queryCssDeep(root.shadowRoot, selector) : []; - const nested = [...root.querySelectorAll("*")].flatMap((element) => - element.shadowRoot ? queryCssDeep(element.shadowRoot, selector) : [], - ); + const ownRoot = root instanceof Element ? shadowRootFor(root) : null; + const ownShadow = ownRoot ? queryCssDeep(ownRoot, selector) : []; + const nested = [...root.querySelectorAll("*")].flatMap((element) => { + const shadow = shadowRootFor(element); + return shadow ? queryCssDeep(shadow, selector) : []; + }); return dedupe([...direct, ...ownShadow, ...nested]); }; const smallestTextMatches = (elements: Element[], expected: JsonMatcher): Element[] => @@ -257,6 +437,72 @@ export async function createPlaywrightCompatRuntime( } return elements; }; + /** + * Resolve an XPath produced by Stagehand's accessibility snapshot. Those + * paths are positional (`/html[1]/body[1]/x-host[1]//div[2]/button[1]`) + * and encode a shadow-root boundary as `//`, which native + * `document.evaluate` cannot follow. Mirrors the extension's + * resolveStagehandShadowHopMatches: child steps walk light-DOM children, + * a `//` step after the first walks into the host's shadow root. + */ + const resolveStagehandXPath = (expression: string): Element[] => { + const path = expression.trim().replace(/^xpath=/iu, ""); + if (!path) return []; + type Step = { hop: boolean; tag: string; index?: number }; + const steps: Step[] = []; + let cursor = 0; + while (cursor < path.length) { + let hop = false; + if (path.startsWith("//", cursor)) { + hop = true; + cursor += 2; + } else if (path[cursor] === "/") { + cursor += 1; + } + const start = cursor; + while (cursor < path.length && path[cursor] !== "/") cursor += 1; + const raw = path.slice(start, cursor).trim(); + if (!raw) continue; + const parsed = raw.match(/^([^[]+)(?:\[(\d+)\])?$/u); + if (!parsed) return queryXPath(document, path); + steps.push({ + hop, + tag: (parsed[1] ?? "*").toLowerCase(), + ...(parsed[2] ? { index: Number(parsed[2]) } : {}), + }); + } + const hasShadowHop = steps.some((step, position) => step.hop && position > 0); + if (!hasShadowHop) { + try { + return queryXPath(document, path); + } catch { + return []; + } + } + let current: Array = [document]; + for (const [position, step] of steps.entries()) { + const next: Element[] = []; + for (const root of current) { + let pool: Element[]; + if (root instanceof Document) { + pool = root.documentElement ? [root.documentElement] : []; + } else if (step.hop && position > 0) { + pool = root instanceof Element ? [...(shadowRootFor(root)?.children ?? [])] : []; + } else { + pool = [...root.children]; + } + const tagged = pool.filter( + (element) => step.tag === "*" || element.localName.toLowerCase() === step.tag, + ); + const picked = + step.index === undefined ? tagged : [tagged[step.index - 1]!].filter(Boolean); + for (const element of picked) if (!next.includes(element)) next.push(element); + } + if (!next.length) return []; + current = next; + } + return current as Element[]; + }; const splitSelectorList = (selector: string): string[] => { const parts: string[] = []; let start = 0; @@ -383,6 +629,55 @@ export async function createPlaywrightCompatRuntime( } return ""; }; + const labelNodeText = (node: Node): string => { + if ( + ["SCRIPT", "STYLE", "NOSCRIPT"].includes(node.nodeName) || + node.ownerDocument?.head?.contains(node) + ) { + return ""; + } + if (node instanceof HTMLInputElement && (node.type === "submit" || node.type === "button")) { + return node.value; + } + let text = ""; + for (const child of node.childNodes) { + if (child.nodeType === Node.TEXT_NODE) text += child.nodeValue ?? ""; + else if (child.nodeType === Node.ELEMENT_NODE) text += labelNodeText(child); + } + if (node instanceof Element) { + const shadow = shadowRootFor(node); + if (shadow) text += labelNodeText(shadow); + } + return text; + }; + // Label locators match each label separately, with labelledby > aria-label >