feat: support deterministic actions without a model

.changeset/tidy-actions-run.md

@@ -0,0 +1,8 @@
+---
+"@browserbasehq/stagehand": patch
+"@browserbasehq/stagehand-extension": patch
+"@browserbasehq/stagehand-go": patch
+"browse": patch
+---
+
+Allow deterministic Action objects to run without configuring a model, and use that path for CLI click and fill commands.

packages/cli/src/lib/driver/commands/elements.ts

@@ -1,12 +1,18 @@
+import type { Action } from "@browserbasehq/stagehand";
 import { z } from "zod";
 
+import type { DriverSessionManager } from "../session-manager.js";
 import type { DriverCommandHandlers } from "./types.js";
 
 export const elementsHandlers: DriverCommandHandlers = {
   async click(manager, params) {
     const { selector } = z.object({ selector: z.string().min(1) }).parse(params);
-    const page = await manager.activePage();
-    await page.locator(manager.resolveSelector(selector)).click();
+    await performAction(manager, {
+      arguments: [],
+      description: "click element",
+      method: "click",
+      selector: manager.resolveSelector(selector),
+    });
     return { clicked: true };
   },
 
@@ -18,9 +24,14 @@ export const elementsHandlers: DriverCommandHandlers = {
         value: z.string(),
       })
       .parse(params);
-    const page = await manager.activePage();
-    await page.locator(manager.resolveSelector(selector)).fill(value);
+    await performAction(manager, {
+      arguments: [value],
+      description: "fill element",
+      method: "fill",
+      selector: manager.resolveSelector(selector),
+    });
     if (pressEnter) {
+      const page = await manager.activePage();
       await page.keyPress("Enter");
     }
     return { filled: true, pressedEnter: pressEnter ?? false };
@@ -64,3 +75,12 @@ export const elementsHandlers: DriverCommandHandlers = {
     return { highlighted: true };
   },
 };
+
+async function performAction(manager: DriverSessionManager, action: Action): Promise<void> {
+  const stagehand = await manager.stagehandInstance();
+  const page = await manager.activePage();
+  const result = await stagehand.act(action, { page });
+  if (!result.data.success) {
+    throw new Error(result.data.message);
+  }
+}

packages/cli/tests/driver-commands.test.ts

@@ -170,20 +170,23 @@ describe("driver commands", () => {
     });
   });
 
-  it("routes selector click and fill through V4 locators", async () => {
-    const locator = {
-      click: vi.fn(),
-      fill: vi.fn(),
-    };
-    const page = {
-      keyPress: vi.fn(),
-      locator: vi.fn(() => locator),
-    };
+  it("routes selector click and fill through deterministic V4 actions", async () => {
+    const page = { keyPress: vi.fn() };
+    const act = vi.fn().mockResolvedValue({
+      data: {
+        success: true,
+        message: "Action completed",
+        actionDescription: "action",
+        actions: [],
+      },
+      metadata: {},
+    });
     const manager = {
       activePage: vi.fn(async () => page),
       resolveSelector: vi.fn((selector: string) =>
         selector === "@0-1" ? "/html/body/button" : selector,
       ),
+      stagehandInstance: vi.fn(async () => ({ act })),
     } as unknown as Parameters<NonNullable<(typeof elementsHandlers)["click"]>>[0];
 
     await expect(elementsHandlers.click!(manager, { selector: "@0-1" })).resolves.toEqual({
@@ -197,13 +200,52 @@ describe("driver commands", () => {
       }),
     ).resolves.toEqual({ filled: true, pressedEnter: true });
 
-    expect(page.locator).toHaveBeenNthCalledWith(1, "/html/body/button");
-    expect(page.locator).toHaveBeenNthCalledWith(2, "#email");
-    expect(locator.click).toHaveBeenCalledOnce();
-    expect(locator.fill).toHaveBeenCalledWith("user@example.com");
+    expect(act).toHaveBeenNthCalledWith(
+      1,
+      {
+        arguments: [],
+        description: "click element",
+        method: "click",
+        selector: "/html/body/button",
+      },
+      { page },
+    );
+    expect(act).toHaveBeenNthCalledWith(
+      2,
+      {
+        arguments: ["user@example.com"],
+        description: "fill element",
+        method: "fill",
+        selector: "#email",
+      },
+      { page },
+    );
     expect(page.keyPress).toHaveBeenCalledWith("Enter");
   });
 
+  it("surfaces a deterministic V4 action failure instead of reporting success", async () => {
+    const page = {};
+    const manager = {
+      activePage: vi.fn(async () => page),
+      resolveSelector: vi.fn((selector: string) => selector),
+      stagehandInstance: vi.fn(async () => ({
+        act: vi.fn().mockResolvedValue({
+          data: {
+            success: false,
+            message: "Failed to perform act: Element detached",
+            actionDescription: "click element",
+            actions: [],
+          },
+          metadata: {},
+        }),
+      })),
+    } as unknown as Parameters<NonNullable<(typeof elementsHandlers)["click"]>>[0];
+
+    await expect(elementsHandlers.click!(manager, { selector: "#submit" })).rejects.toThrow(
+      "Failed to perform act: Element detached",
+    );
+  });
+
   it("keeps select and highlight on V4 locators", async () => {
     const locator = {
       highlight: vi.fn(),

packages/extension/controllers/stagehandController.ts

@@ -71,7 +71,7 @@ export function createStagehandController(
 
       const model = params.options?.model ?? state.initParams.model;
       const gateway = buildGatewayContext(state.initParams);
-      if (!model && !gateway) {
+      if (typeof params.instruction === "string" && !model && !gateway) {
         throw new Error("An LLM was not configured during Stagehand initialization");
       }
 

packages/extension/services/actService.ts

@@ -59,7 +59,7 @@ export async function act({
 }: {
   params: StagehandActParams;
   page: Page;
-  model: ModelConfig | ClientModelReference | undefined;
+  model?: ModelConfig | ClientModelReference;
   clientLLMGenerate: ClientLlmRequest;
   logger: StagehandLogger;
   systemPrompt?: string;
@@ -69,6 +69,10 @@ export async function act({
   gateway?: GatewayContext;
 }): Promise<ActResult> {
   const { instruction: actInstruction, options } = params;
+  if (typeof actInstruction === "string" && !model) {
+    throw new Error("An LLM was not configured during Stagehand initialization");
+  }
+
   const variables = options?.variables;
   const timeout = options?.timeout;
   const ensureTimeRemaining = createTimeoutGuard(timeout, (ms) => new TimeoutError("act()", ms));
@@ -82,7 +86,7 @@ export async function act({
     clientLLMGenerate,
     logger,
     systemPrompt,
-    selfHeal,
+    selfHeal: selfHeal && model !== undefined,
     domSettleTimeoutMs,
     ensureTimeRemaining,
     gateway,
@@ -277,11 +281,16 @@ async function getActionFromLLM({
   xpathMap: Record<string, string>;
   context: ActContext;
 }): Promise<{ action?: Action; response: ActInferenceResponse }> {
+  const model = context.model;
+  if (!model) {
+    throw new Error("An LLM was not configured during Stagehand initialization");
+  }
+
   const response = await inference.act({
     instruction,
     domElements,
     generate: (input) =>
-      llmService.generate(context.model, input, context.clientLLMGenerate, context.gateway),
+      llmService.generate(model, input, context.clientLLMGenerate, context.gateway),
     userProvidedInstructions: context.systemPrompt,
   });
   context.recordUsage(response);

packages/extension/tests/act.test.ts

@@ -7,8 +7,10 @@ import {
   performUnderstudyMethod,
   waitForDomNetworkQuiet,
 } from "../handlers/handlerUtils/actHandlerUtils.js";
+import { createStagehandController } from "../controllers/stagehandController.js";
 import * as inference from "../inference.js";
 import { StagehandLogger } from "../logger.js";
+import type { StagehandRuntime } from "../runtime.js";
 import * as actService from "../services/actService.js";
 import type { Page } from "../understudy/page.js";
 
@@ -257,11 +259,10 @@ describe("act service", () => {
 
   it("zeroes usage when a supplied Action succeeds without inference", async () => {
     const frame = {};
-    const page = actPage(
-      frame,
-      vi.fn(async () => snapshot("0-12", "/html/body/button")),
-    );
-    const clientLLMGenerate = vi.fn(async (): Promise<LLMGenerateResult> => actGeneration(null));
+    const captureSnapshot = vi.fn();
+    const page = actPage(frame, captureSnapshot);
+    const clientLLMGenerate = vi.fn();
+    const logger = testLogger();
 
     const result = await actService.act({
       params: {
@@ -274,19 +275,62 @@ describe("act service", () => {
         },
       },
       page,
-      model: { source: "client" },
       clientLLMGenerate,
-      logger: testLogger(),
+      logger,
     });
 
-    expect(result.data.success).toBe(true);
+    expect(waitForQuiet).not.toHaveBeenCalled();
+    expect(captureSnapshot).not.toHaveBeenCalled();
+    expect(clientLLMGenerate).not.toHaveBeenCalled();
+    expect(performAction).toHaveBeenCalledWith(
+      page,
+      frame,
+      "click",
+      "xpath=/html/body/button",
+      [],
+      logger,
+      undefined,
+    );
+    expect(result.data).toMatchObject({
+      success: true,
+      actions: [{ selector: "xpath=/html/body/button" }],
+    });
     expect(result.metadata.usage).toStrictEqual({
       inputTokens: 0,
       outputTokens: 0,
       reasoningTokens: 0,
       cachedInputTokens: 0,
       inferenceTimeMs: 0,
     });
+  });
+
+  it("does not attempt model-backed self-healing without a model", async () => {
+    const captureSnapshot = vi.fn();
+    const page = actPage({}, captureSnapshot);
+    const clientLLMGenerate = vi.fn();
+    performAction.mockRejectedValueOnce(new Error("Element detached"));
+
+    const result = await actService.act({
+      params: {
+        pageId: "page-1",
+        instruction: {
+          selector: "xpath=/html/body/button",
+          description: "Submit button",
+          method: "click",
+          arguments: [],
+        },
+      },
+      page,
+      clientLLMGenerate,
+      logger: testLogger(),
+      selfHeal: true,
+    });
+
+    expect(result.data).toMatchObject({
+      success: false,
+      message: "Failed to perform act: Element detached",
+    });
+    expect(captureSnapshot).not.toHaveBeenCalled();
     expect(clientLLMGenerate).not.toHaveBeenCalled();
   });
 
@@ -661,6 +705,81 @@ describe("act service", () => {
   });
 });
 
+describe("act controller", () => {
+  beforeEach(() => {
+    performAction.mockReset().mockResolvedValue();
+    waitForQuiet.mockReset().mockResolvedValue();
+  });
+
+  it("allows a supplied Action through an initialization without a model", async () => {
+    const page = actPage({}, vi.fn());
+    const resolveUnderstudyPage = vi.fn(() => page);
+    const runtime = {
+      adapters: { clientLLMGenerate: vi.fn() },
+      metrics: { record: vi.fn() },
+      resolveUnderstudyPage,
+      runWithTelemetryContext: vi.fn((_scope, _logger, run: () => unknown) => run()),
+      state: {
+        getState: () => ({
+          status: "initialized",
+          initParams: {
+            domSettleTimeoutMs: 2_000,
+            selfHeal: true,
+            systemPrompt: "",
+          },
+        }),
+      },
+    } as unknown as StagehandRuntime;
+    const controller = createStagehandController(runtime);
+
+    await expect(
+      controller.act(
+        {
+          pageId: "page-1",
+          instruction: {
+            selector: "xpath=/html/body/button",
+            description: "Submit button",
+            method: "click",
+            arguments: [],
+          },
+        },
+        { logger: testLogger(), telemetryScope: Symbol("act-controller-test") },
+      ),
+    ).resolves.toMatchObject({ data: { success: true } });
+
+    expect(resolveUnderstudyPage).toHaveBeenCalledWith("page-1");
+    expect(runtime.adapters.clientLLMGenerate).not.toHaveBeenCalled();
+  });
+
+  it("still rejects a natural-language instruction without a model", async () => {
+    const resolveUnderstudyPage = vi.fn();
+    const runtime = {
+      adapters: { clientLLMGenerate: vi.fn() },
+      metrics: { record: vi.fn() },
+      resolveUnderstudyPage,
+      runWithTelemetryContext: vi.fn((_scope, _logger, run: () => unknown) => run()),
+      state: {
+        getState: () => ({
+          status: "initialized",
+          initParams: { selfHeal: true, systemPrompt: "" },
+        }),
+      },
+    } as unknown as StagehandRuntime;
+    const controller = createStagehandController(runtime);
+
+    await expect(
+      controller.act(
+        {
+          pageId: "page-1",
+          instruction: "Click the submit button",
+        },
+        { logger: testLogger(), telemetryScope: Symbol("act-controller-test") },
+      ),
+    ).rejects.toThrow("An LLM was not configured during Stagehand initialization");
+    expect(resolveUnderstudyPage).not.toHaveBeenCalled();
+  });
+});
+
 function actGeneration(
   action: Record<string, string | string[]> | null,
   twoStep = false,