diff --git a/.changeset/tidy-actions-run.md b/.changeset/tidy-actions-run.md new file mode 100644 index 0000000000..aa521b6270 --- /dev/null +++ b/.changeset/tidy-actions-run.md @@ -0,0 +1,8 @@ +--- +"@browserbasehq/stagehand": patch +"@browserbasehq/stagehand-extension": patch +"@browserbasehq/stagehand-go": patch +"browse": patch +--- + +Allow deterministic Action objects to run without configuring a model, and use that path for CLI click and fill commands. diff --git a/packages/cli/src/lib/driver/commands/elements.ts b/packages/cli/src/lib/driver/commands/elements.ts index 1e88e97551..4706a88207 100644 --- a/packages/cli/src/lib/driver/commands/elements.ts +++ b/packages/cli/src/lib/driver/commands/elements.ts @@ -1,12 +1,18 @@ +import type { Action } from "@browserbasehq/stagehand"; import { z } from "zod"; +import type { DriverSessionManager } from "../session-manager.js"; import type { DriverCommandHandlers } from "./types.js"; export const elementsHandlers: DriverCommandHandlers = { async click(manager, params) { const { selector } = z.object({ selector: z.string().min(1) }).parse(params); - const page = await manager.activePage(); - await page.locator(manager.resolveSelector(selector)).click(); + await performAction(manager, { + arguments: [], + description: "click element", + method: "click", + selector: manager.resolveSelector(selector), + }); return { clicked: true }; }, @@ -18,9 +24,14 @@ export const elementsHandlers: DriverCommandHandlers = { value: z.string(), }) .parse(params); - const page = await manager.activePage(); - await page.locator(manager.resolveSelector(selector)).fill(value); + await performAction(manager, { + arguments: [value], + description: "fill element", + method: "fill", + selector: manager.resolveSelector(selector), + }); if (pressEnter) { + const page = await manager.activePage(); await page.keyPress("Enter"); } return { filled: true, pressedEnter: pressEnter ?? false }; @@ -64,3 +75,12 @@ export const elementsHandlers: DriverCommandHandlers = { return { highlighted: true }; }, }; + +async function performAction(manager: DriverSessionManager, action: Action): Promise { + const stagehand = await manager.stagehandInstance(); + const page = await manager.activePage(); + const result = await stagehand.act(action, { page }); + if (!result.data.success) { + throw new Error(result.data.message); + } +} diff --git a/packages/cli/tests/driver-commands.test.ts b/packages/cli/tests/driver-commands.test.ts index 18b3fb227a..6d47aa15ef 100644 --- a/packages/cli/tests/driver-commands.test.ts +++ b/packages/cli/tests/driver-commands.test.ts @@ -170,20 +170,23 @@ describe("driver commands", () => { }); }); - it("routes selector click and fill through V4 locators", async () => { - const locator = { - click: vi.fn(), - fill: vi.fn(), - }; - const page = { - keyPress: vi.fn(), - locator: vi.fn(() => locator), - }; + it("routes selector click and fill through deterministic V4 actions", async () => { + const page = { keyPress: vi.fn() }; + const act = vi.fn().mockResolvedValue({ + data: { + success: true, + message: "Action completed", + actionDescription: "action", + actions: [], + }, + metadata: {}, + }); const manager = { activePage: vi.fn(async () => page), resolveSelector: vi.fn((selector: string) => selector === "@0-1" ? "/html/body/button" : selector, ), + stagehandInstance: vi.fn(async () => ({ act })), } as unknown as Parameters>[0]; await expect(elementsHandlers.click!(manager, { selector: "@0-1" })).resolves.toEqual({ @@ -197,13 +200,52 @@ describe("driver commands", () => { }), ).resolves.toEqual({ filled: true, pressedEnter: true }); - expect(page.locator).toHaveBeenNthCalledWith(1, "/html/body/button"); - expect(page.locator).toHaveBeenNthCalledWith(2, "#email"); - expect(locator.click).toHaveBeenCalledOnce(); - expect(locator.fill).toHaveBeenCalledWith("user@example.com"); + expect(act).toHaveBeenNthCalledWith( + 1, + { + arguments: [], + description: "click element", + method: "click", + selector: "/html/body/button", + }, + { page }, + ); + expect(act).toHaveBeenNthCalledWith( + 2, + { + arguments: ["user@example.com"], + description: "fill element", + method: "fill", + selector: "#email", + }, + { page }, + ); expect(page.keyPress).toHaveBeenCalledWith("Enter"); }); + it("surfaces a deterministic V4 action failure instead of reporting success", async () => { + const page = {}; + const manager = { + activePage: vi.fn(async () => page), + resolveSelector: vi.fn((selector: string) => selector), + stagehandInstance: vi.fn(async () => ({ + act: vi.fn().mockResolvedValue({ + data: { + success: false, + message: "Failed to perform act: Element detached", + actionDescription: "click element", + actions: [], + }, + metadata: {}, + }), + })), + } as unknown as Parameters>[0]; + + await expect(elementsHandlers.click!(manager, { selector: "#submit" })).rejects.toThrow( + "Failed to perform act: Element detached", + ); + }); + it("keeps select and highlight on V4 locators", async () => { const locator = { highlight: vi.fn(), diff --git a/packages/extension/controllers/stagehandController.ts b/packages/extension/controllers/stagehandController.ts index 79bef41bcf..7e990bc8dd 100644 --- a/packages/extension/controllers/stagehandController.ts +++ b/packages/extension/controllers/stagehandController.ts @@ -71,7 +71,7 @@ export function createStagehandController( const model = params.options?.model ?? state.initParams.model; const gateway = buildGatewayContext(state.initParams); - if (!model && !gateway) { + if (typeof params.instruction === "string" && !model && !gateway) { throw new Error("An LLM was not configured during Stagehand initialization"); } diff --git a/packages/extension/services/actService.ts b/packages/extension/services/actService.ts index 7613dd590f..347b7f5f90 100644 --- a/packages/extension/services/actService.ts +++ b/packages/extension/services/actService.ts @@ -59,7 +59,7 @@ export async function act({ }: { params: StagehandActParams; page: Page; - model: ModelConfig | ClientModelReference | undefined; + model?: ModelConfig | ClientModelReference; clientLLMGenerate: ClientLlmRequest; logger: StagehandLogger; systemPrompt?: string; @@ -69,6 +69,10 @@ export async function act({ gateway?: GatewayContext; }): Promise { const { instruction: actInstruction, options } = params; + if (typeof actInstruction === "string" && !model) { + throw new Error("An LLM was not configured during Stagehand initialization"); + } + const variables = options?.variables; const timeout = options?.timeout; const ensureTimeRemaining = createTimeoutGuard(timeout, (ms) => new TimeoutError("act()", ms)); @@ -82,7 +86,7 @@ export async function act({ clientLLMGenerate, logger, systemPrompt, - selfHeal, + selfHeal: selfHeal && model !== undefined, domSettleTimeoutMs, ensureTimeRemaining, gateway, @@ -277,11 +281,16 @@ async function getActionFromLLM({ xpathMap: Record; context: ActContext; }): Promise<{ action?: Action; response: ActInferenceResponse }> { + const model = context.model; + if (!model) { + throw new Error("An LLM was not configured during Stagehand initialization"); + } + const response = await inference.act({ instruction, domElements, generate: (input) => - llmService.generate(context.model, input, context.clientLLMGenerate, context.gateway), + llmService.generate(model, input, context.clientLLMGenerate, context.gateway), userProvidedInstructions: context.systemPrompt, }); context.recordUsage(response); diff --git a/packages/extension/tests/act.test.ts b/packages/extension/tests/act.test.ts index 2399924d6c..0538199b65 100644 --- a/packages/extension/tests/act.test.ts +++ b/packages/extension/tests/act.test.ts @@ -7,8 +7,10 @@ import { performUnderstudyMethod, waitForDomNetworkQuiet, } from "../handlers/handlerUtils/actHandlerUtils.js"; +import { createStagehandController } from "../controllers/stagehandController.js"; import * as inference from "../inference.js"; import { StagehandLogger } from "../logger.js"; +import type { StagehandRuntime } from "../runtime.js"; import * as actService from "../services/actService.js"; import type { Page } from "../understudy/page.js"; @@ -257,11 +259,10 @@ describe("act service", () => { it("zeroes usage when a supplied Action succeeds without inference", async () => { const frame = {}; - const page = actPage( - frame, - vi.fn(async () => snapshot("0-12", "/html/body/button")), - ); - const clientLLMGenerate = vi.fn(async (): Promise => actGeneration(null)); + const captureSnapshot = vi.fn(); + const page = actPage(frame, captureSnapshot); + const clientLLMGenerate = vi.fn(); + const logger = testLogger(); const result = await actService.act({ params: { @@ -274,12 +275,26 @@ describe("act service", () => { }, }, page, - model: { source: "client" }, clientLLMGenerate, - logger: testLogger(), + logger, }); - expect(result.data.success).toBe(true); + expect(waitForQuiet).not.toHaveBeenCalled(); + expect(captureSnapshot).not.toHaveBeenCalled(); + expect(clientLLMGenerate).not.toHaveBeenCalled(); + expect(performAction).toHaveBeenCalledWith( + page, + frame, + "click", + "xpath=/html/body/button", + [], + logger, + undefined, + ); + expect(result.data).toMatchObject({ + success: true, + actions: [{ selector: "xpath=/html/body/button" }], + }); expect(result.metadata.usage).toStrictEqual({ inputTokens: 0, outputTokens: 0, @@ -287,6 +302,35 @@ describe("act service", () => { cachedInputTokens: 0, inferenceTimeMs: 0, }); + }); + + it("does not attempt model-backed self-healing without a model", async () => { + const captureSnapshot = vi.fn(); + const page = actPage({}, captureSnapshot); + const clientLLMGenerate = vi.fn(); + performAction.mockRejectedValueOnce(new Error("Element detached")); + + const result = await actService.act({ + params: { + pageId: "page-1", + instruction: { + selector: "xpath=/html/body/button", + description: "Submit button", + method: "click", + arguments: [], + }, + }, + page, + clientLLMGenerate, + logger: testLogger(), + selfHeal: true, + }); + + expect(result.data).toMatchObject({ + success: false, + message: "Failed to perform act: Element detached", + }); + expect(captureSnapshot).not.toHaveBeenCalled(); expect(clientLLMGenerate).not.toHaveBeenCalled(); }); @@ -661,6 +705,81 @@ describe("act service", () => { }); }); +describe("act controller", () => { + beforeEach(() => { + performAction.mockReset().mockResolvedValue(); + waitForQuiet.mockReset().mockResolvedValue(); + }); + + it("allows a supplied Action through an initialization without a model", async () => { + const page = actPage({}, vi.fn()); + const resolveUnderstudyPage = vi.fn(() => page); + const runtime = { + adapters: { clientLLMGenerate: vi.fn() }, + metrics: { record: vi.fn() }, + resolveUnderstudyPage, + runWithTelemetryContext: vi.fn((_scope, _logger, run: () => unknown) => run()), + state: { + getState: () => ({ + status: "initialized", + initParams: { + domSettleTimeoutMs: 2_000, + selfHeal: true, + systemPrompt: "", + }, + }), + }, + } as unknown as StagehandRuntime; + const controller = createStagehandController(runtime); + + await expect( + controller.act( + { + pageId: "page-1", + instruction: { + selector: "xpath=/html/body/button", + description: "Submit button", + method: "click", + arguments: [], + }, + }, + { logger: testLogger(), telemetryScope: Symbol("act-controller-test") }, + ), + ).resolves.toMatchObject({ data: { success: true } }); + + expect(resolveUnderstudyPage).toHaveBeenCalledWith("page-1"); + expect(runtime.adapters.clientLLMGenerate).not.toHaveBeenCalled(); + }); + + it("still rejects a natural-language instruction without a model", async () => { + const resolveUnderstudyPage = vi.fn(); + const runtime = { + adapters: { clientLLMGenerate: vi.fn() }, + metrics: { record: vi.fn() }, + resolveUnderstudyPage, + runWithTelemetryContext: vi.fn((_scope, _logger, run: () => unknown) => run()), + state: { + getState: () => ({ + status: "initialized", + initParams: { selfHeal: true, systemPrompt: "" }, + }), + }, + } as unknown as StagehandRuntime; + const controller = createStagehandController(runtime); + + await expect( + controller.act( + { + pageId: "page-1", + instruction: "Click the submit button", + }, + { logger: testLogger(), telemetryScope: Symbol("act-controller-test") }, + ), + ).rejects.toThrow("An LLM was not configured during Stagehand initialization"); + expect(resolveUnderstudyPage).not.toHaveBeenCalled(); + }); +}); + function actGeneration( action: Record | null, twoStep = false, diff --git a/packages/sdk-go/internal/extensionassets/stagehand-extension.zip b/packages/sdk-go/internal/extensionassets/stagehand-extension.zip index 62d2cba257..8f3a053595 100644 Binary files a/packages/sdk-go/internal/extensionassets/stagehand-extension.zip and b/packages/sdk-go/internal/extensionassets/stagehand-extension.zip differ