diff --git a/packages/evals/framework/codexRunner.ts b/packages/evals/framework/codexRunner.ts index 85e46ca23..fad2dfa52 100644 --- a/packages/evals/framework/codexRunner.ts +++ b/packages/evals/framework/codexRunner.ts @@ -132,6 +132,11 @@ export async function runCodexAgent({ }, outputSchema: EVAL_RESULT_SCHEMA, maxToolSteps, + diagnosticDirectory: process.env.EVAL_CODEX_DIAGNOSTICS_DIR, + allowedMcpServers: + toolAdapter && "allowedMcpServers" in toolAdapter + ? toolAdapter.allowedMcpServers + : undefined, ...(toolAdapter?.env?.CODEX_HOME && { codexHome: toolAdapter.env.CODEX_HOME }), onToolStep: toolAdapter && "recordObservation" in toolAdapter diff --git a/packages/evals/framework/codexToolAdapter.ts b/packages/evals/framework/codexToolAdapter.ts index 79c7d4756..98e8f44c5 100644 --- a/packages/evals/framework/codexToolAdapter.ts +++ b/packages/evals/framework/codexToolAdapter.ts @@ -1,6 +1,7 @@ import fsp from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { isolatedCodexEnv } from "@browserbasehq/stagehand-integrations-codex-sdk"; import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; import { @@ -51,7 +52,8 @@ export interface PreparedCodexCodeAdapter { * it to record per-step observations (their tool calls never pass through * the workspace bridge). */ - recordObservation?: () => void; + recordObservation?: (item: Record) => Promise; + allowedMcpServers?: string[]; /** Which normalized tool-call names consume observation indexes. */ observedToolMatcher?: (name: string) => boolean; cleanup: () => Promise; @@ -85,7 +87,7 @@ const STAGEHAND_FACADE_MCP_TIMEOUTS = { export const CODEX_MCP_TOOLS_APPROVAL_MODE = "approve"; /** Name of the per-run Codex home directory created inside the adapter cwd. */ -export const CODEX_HOME_DIRNAME = ".codex-home"; +export const CODEX_HOME_DIRNAME = path.join("home", ".codex"); export function buildCodexMcpServers( toolSurface: ToolSurface, @@ -116,15 +118,16 @@ export function buildIsolatedCodexEnv( ): Record { const env: Record = {}; for (const [key, value] of Object.entries(baseEnv)) { - if (value !== undefined) env[key] = value; + if (value !== undefined && (!key.startsWith("CODEX_") || key === "CODEX_API_KEY")) + env[key] = value; } + env.HOME = path.dirname(codexHome); env.CODEX_HOME = codexHome; return env; } async function createIsolatedCodexHome(cwd: string): Promise { - const codexHome = path.join(cwd, CODEX_HOME_DIRNAME); - await fsp.mkdir(codexHome, { recursive: true }); + const { CODEX_HOME: codexHome } = await isolatedCodexEnv(cwd); // Every setting the run needs arrives as `--config` overrides from the SDK; // the file exists only so nothing in this home is inherited from elsewhere. await fsp.writeFile( @@ -248,6 +251,7 @@ export async function prepareCodexToolAdapter( promptInstructions: mount.promptInstructions, browserSession: runtime.browserSession, codexConfig: { mcp_servers: codexMcpServers }, + allowedMcpServers: serverNames, ...(runtime.running.browserSessionLoss && { browserSessionLoss: runtime.running.browserSessionLoss, }), @@ -259,7 +263,11 @@ export async function prepareCodexToolAdapter( await recorder.settle(); return recorder.drain(); }, - recordObservation: () => void recorder.record(), + recordObservation: async (item: Record) => { + if (serverNames.includes(String(item.server))) { + await recorder.record(typeof item.id === "string" ? item.id : undefined); + } + }, }), observedToolMatcher: (name: string) => serverNames.some((server) => name.startsWith(`${server}.`)), diff --git a/packages/evals/framework/harnesses/codexAdapter.ts b/packages/evals/framework/harnesses/codexAdapter.ts index 30ee52493..9c570a5c9 100644 --- a/packages/evals/framework/harnesses/codexAdapter.ts +++ b/packages/evals/framework/harnesses/codexAdapter.ts @@ -85,6 +85,7 @@ export class CodexTrajectoryAdapter implements TrajectoryAdapter const call = normalizeItem(itemType, item, pendingReasoning); if (call) { + if (typeof item.id === "string") call.id = item.id; toolCalls.push(call); pendingReasoning = ""; } @@ -96,7 +97,17 @@ export class CodexTrajectoryAdapter implements TrajectoryAdapter // the bridge some other way), ordinals would shift and attach evidence // to the wrong steps — misattribution is worse than a gap, so attach // nothing and let the verifier take its evidence_insufficient path. - const observations = result.stepObservations ?? []; + const keyed = new Map( + (result.stepObservations ?? []) + .filter((observation) => observation.toolCallId) + .map((observation) => [observation.toolCallId, observation.evidence]), + ); + for (const call of toolCalls) { + if (call.id && keyed.has(call.id)) call.probeEvidence = keyed.get(call.id); + } + const observations = (result.stepObservations ?? []).filter( + (observation) => !observation.toolCallId, + ); if (observations.length > 0) { const observedCalls = toolCalls.filter((call) => result.observedToolName diff --git a/packages/evals/framework/harnesses/trajectoryAdapter.ts b/packages/evals/framework/harnesses/trajectoryAdapter.ts index 4dd86f2ba..92b6d6466 100644 --- a/packages/evals/framework/harnesses/trajectoryAdapter.ts +++ b/packages/evals/framework/harnesses/trajectoryAdapter.ts @@ -43,6 +43,7 @@ export interface TrajectoryAdapter { * this shape before mapping to a TrajectoryStep. */ export interface NormalizedToolCall { + id?: string; /** Tool name (e.g., "Bash", "mcp__stagehand_browser__run", "container.exec"). */ name: string; /** Tool arguments. Empty object if the harness doesn't surface them. */ diff --git a/packages/evals/framework/observationRecorder.ts b/packages/evals/framework/observationRecorder.ts index 246a3d7d8..7496f13c1 100644 --- a/packages/evals/framework/observationRecorder.ts +++ b/packages/evals/framework/observationRecorder.ts @@ -3,6 +3,7 @@ import type { ProbeEvidence } from "stagehand-v3"; /** A probe observation captured after the Nth run-tool execution (0-based). */ export interface StepObservation { runIndex: number; + toolCallId?: string; evidence: ProbeEvidence; } @@ -32,13 +33,13 @@ export class ObservationRecorder { constructor(private readonly capture: () => Promise) {} - async record(): Promise { + async record(toolCallId?: string): Promise { const runIndex = this.runIndex++; const attempt = (async () => { try { const evidence = await withTimeout(this.capture(), observationTimeoutMs()); if (evidence.screenshot || evidence.url || evidence.ariaTree) { - this.observations.push({ runIndex, evidence }); + this.observations.push({ runIndex, evidence, ...(toolCallId && { toolCallId }) }); } } catch { // best-effort only — a failed probe must never fail the run tool diff --git a/packages/evals/tests/framework/codexToolAdapter.test.ts b/packages/evals/tests/framework/codexToolAdapter.test.ts index 6884034ed..5d9121d20 100644 --- a/packages/evals/tests/framework/codexToolAdapter.test.ts +++ b/packages/evals/tests/framework/codexToolAdapter.test.ts @@ -64,9 +64,19 @@ describe("codex tool adapter", () => { it("points CODEX_HOME at the per-run directory and drops the inherited one", () => { const env = buildIsolatedCodexEnv( - { PATH: "/usr/bin", CODEX_HOME: "/Users/someone/.codex", UNSET: undefined }, - "/tmp/run/.codex-home", + { + PATH: "/usr/bin", + CODEX_HOME: "/Users/someone/.codex", + CODEX_THREAD_ID: "host", + HOME: "/Users/someone", + UNSET: undefined, + }, + "/tmp/run/home/.codex", ); - expect(env).toEqual({ PATH: "/usr/bin", CODEX_HOME: "/tmp/run/.codex-home" }); + expect(env).toEqual({ + PATH: "/usr/bin", + HOME: "/tmp/run/home", + CODEX_HOME: "/tmp/run/home/.codex", + }); }); }); diff --git a/packages/evals/tests/framework/harnessObservations.test.ts b/packages/evals/tests/framework/harnessObservations.test.ts index 39a3fb804..e6f890c16 100644 --- a/packages/evals/tests/framework/harnessObservations.test.ts +++ b/packages/evals/tests/framework/harnessObservations.test.ts @@ -17,6 +17,55 @@ import { const TASK_SPEC: TaskSpec = { id: "t", instruction: "do the thing" }; +describe("Codex keyed observations", () => { + it("retains early evidence despite unrelated calls and a missing final capture", () => { + const trajectory = codexAdapter.fromHarnessResult( + { + events: [ + { + type: "item.completed", + item: { + id: "other", + type: "mcp_tool_call", + server: "other", + tool: "run", + status: "completed", + }, + }, + { + type: "item.completed", + item: { + id: "first", + type: "mcp_tool_call", + server: "stagehand", + tool: "run", + status: "completed", + }, + }, + { + type: "item.completed", + item: { + id: "last", + type: "mcp_tool_call", + server: "stagehand", + tool: "run", + status: "completed", + }, + }, + ], + stepObservations: [ + { runIndex: 0, toolCallId: "first", evidence: { url: "https://example.com/first" } }, + ], + observedToolName: (name) => name.startsWith("stagehand."), + }, + TASK_SPEC, + ); + expect(trajectory.steps[0].probeEvidence?.url).toBeUndefined(); + expect(trajectory.steps[1].probeEvidence?.url).toBe("https://example.com/first"); + expect(trajectory.steps[2].probeEvidence?.url).toBeUndefined(); + }); +}); + describe("observation recorder", () => { afterEach(() => { delete process.env.EVAL_HARNESS_OBSERVATIONS; diff --git a/packages/integrations/codex-sdk/src/diagnostics.ts b/packages/integrations/codex-sdk/src/diagnostics.ts new file mode 100644 index 000000000..175155f69 --- /dev/null +++ b/packages/integrations/codex-sdk/src/diagnostics.ts @@ -0,0 +1,40 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; + +export async function saveCodexDiagnostic( + directory: string, + error: unknown, + eventCount: number, +): Promise { + await fs.mkdir(directory, { recursive: true, mode: 0o700 }); + const message = error instanceof Error ? error.message : String(error); + const cause = error instanceof Error ? error.cause : undefined; + const filename = path.join(directory, `${Date.now()}-${randomUUID()}.json`); + const sanitized = sanitizeErrorMessage(message); + const maxChars = 2_000_000; + const marker = "Failed to parse item: "; + const rejectedEvent = sanitized.startsWith(marker) ? sanitized.slice(marker.length) : undefined; + await fs.writeFile( + filename, + JSON.stringify( + { + timestamp: new Date().toISOString(), + eventCount, + message: sanitized.slice(0, maxChars), + truncated: sanitized.length > maxChars, + cause: cause ? sanitizeErrorMessage(String(cause)).slice(0, 16_000) : undefined, + rejectedEvent: rejectedEvent?.slice(0, maxChars), + stack: + error instanceof Error + ? sanitizeErrorMessage(error.stack ?? "").slice(0, 16_000) + : undefined, + }, + null, + 2, + ), + { mode: 0o600, flag: "wx" }, + ); + return filename; +} diff --git a/packages/integrations/codex-sdk/src/index.ts b/packages/integrations/codex-sdk/src/index.ts index 9df91886b..12411934b 100644 --- a/packages/integrations/codex-sdk/src/index.ts +++ b/packages/integrations/codex-sdk/src/index.ts @@ -1 +1,2 @@ export * from "./session.js"; +export { isolatedCodexEnv } from "./isolation.js"; diff --git a/packages/integrations/codex-sdk/src/isolation.ts b/packages/integrations/codex-sdk/src/isolation.ts new file mode 100644 index 000000000..c5847cceb --- /dev/null +++ b/packages/integrations/codex-sdk/src/isolation.ts @@ -0,0 +1,28 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +export async function isolatedCodexEnv( + cwd: string, + source: NodeJS.ProcessEnv = process.env, +): Promise> { + const home = path.join(cwd, "home"); + const codexHome = path.join(home, ".codex"); + await fs.mkdir(codexHome, { recursive: true, mode: 0o700 }); + const originalHome = source.CODEX_HOME ?? path.join(source.HOME ?? os.homedir(), ".codex"); + if (!source.OPENAI_API_KEY && !source.CODEX_API_KEY) { + try { + const auth = await fs.readFile(path.join(originalHome, "auth.json")); + await fs.writeFile(path.join(codexHome, "auth.json"), auth, { mode: 0o600 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + const env = Object.fromEntries( + Object.entries(source).filter( + ([key, value]) => + value !== undefined && (!key.startsWith("CODEX_") || key === "CODEX_API_KEY"), + ), + ) as Record; + return { ...env, HOME: home, CODEX_HOME: codexHome }; +} diff --git a/packages/integrations/codex-sdk/src/session.ts b/packages/integrations/codex-sdk/src/session.ts index 5949756ee..9cb5e7042 100644 --- a/packages/integrations/codex-sdk/src/session.ts +++ b/packages/integrations/codex-sdk/src/session.ts @@ -2,6 +2,7 @@ import type { Dirent } from "node:fs"; import fsp from "node:fs/promises"; import path from "node:path"; import type { ModelReasoningEffort } from "@openai/codex-sdk"; +import { saveCodexDiagnostic } from "./diagnostics.js"; import { HarnessAdapterError, harnessEventLogLevel, @@ -58,6 +59,7 @@ export type CodexSessionResult = { usageSource: CodexUsageSource; threadId?: string; iterationError?: unknown; + diagnosticPath?: string; }; export const CODEX_SDK_PACKAGE = "@openai/codex-sdk"; @@ -145,7 +147,8 @@ export async function runCodexSession(input: { thread: CodexThreadConfig; outputSchema?: Record; maxToolSteps?: number; - onToolStep?: () => void | Promise; + onToolStep?: (item: Record) => void | Promise; + allowedMcpServers?: string[]; /** * CODEX_HOME the binary runs with. `codex exec` only reports usage on * `turn.completed`, which never arrives when the turn is aborted (step @@ -153,12 +156,14 @@ export async function runCodexSession(input: { * thread's rollout file under this directory. */ codexHome?: string; + diagnosticDirectory?: string; }): Promise { const sdk = input.sdk ?? (await loadCodexSdk()); const events: CodexEvent[] = []; let finalMessage = ""; let stopReason: string | undefined; let iterationError: unknown; + let diagnosticPath: string | undefined; let tokenUsage = emptyTokenUsage(); let usageSource: CodexUsageSource = "none"; let threadId: string | undefined; @@ -207,6 +212,15 @@ export async function runCodexSession(input: { } const item = isRecord(event.item) ? event.item : undefined; + if ( + item?.type === "mcp_tool_call" && + input.allowedMcpServers && + !input.allowedMcpServers.includes(String(item.server)) + ) { + const policyError = new Error(`Unexpected MCP server: ${String(item.server)}`); + budgetController.abort(policyError); + throw policyError; + } if ( event.type === "item.completed" && item?.type === "agent_message" && @@ -224,11 +238,27 @@ export async function runCodexSession(input: { budgetExhausted = true; budgetController.abort(new Error(stopReason)); } - if (item.type === "mcp_tool_call") await input.onToolStep?.(); + if (item.type === "mcp_tool_call") await input.onToolStep?.(item); } } } catch (error) { iterationError = error; + if (input.diagnosticDirectory) { + try { + diagnosticPath = await saveCodexDiagnostic(input.diagnosticDirectory, error, events.length); + input.logger.warn({ + category: "codex", + level: 0, + message: `Codex diagnostic saved: ${diagnosticPath}`, + }); + } catch (diagnosticError) { + input.logger.warn({ + category: "codex", + level: 0, + message: `Could not save Codex diagnostic: ${sanitizeErrorMessage(stringifyError(diagnosticError))}`, + }); + } + } input.logger.warn({ category: "codex", message: `Codex stopped before a normal result: ${sanitizeErrorMessage(stringifyError(error))}`, @@ -267,6 +297,7 @@ export async function runCodexSession(input: { usageSource, ...(threadId && { threadId }), ...(iterationError !== undefined && { iterationError }), + ...(diagnosticPath && { diagnosticPath }), }; } @@ -469,7 +500,12 @@ export function isRecord(value: unknown): value is Record { export function safeJson(value: unknown): string | undefined { try { - return JSON.stringify(value); + return JSON.stringify(value, (key, item: unknown) => { + if (key === "data" && typeof item === "string" && item.length > 256) + return `[binary omitted: ${item.length} characters; see trajectory artifact]`; + if (typeof item === "string") return clip(sanitizeErrorMessage(item), 32_000); + return item; + }); } catch { return undefined; } diff --git a/packages/integrations/codex-sdk/tests/diagnostics.test.ts b/packages/integrations/codex-sdk/tests/diagnostics.test.ts new file mode 100644 index 000000000..6d2cb2b7a --- /dev/null +++ b/packages/integrations/codex-sdk/tests/diagnostics.test.ts @@ -0,0 +1,36 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { saveCodexDiagnostic } from "../src/diagnostics.js"; +import { safeJson } from "../src/session.js"; + +describe("Codex stream diagnostics", () => { + it("preserves the rejected event and parser cause beyond clipped logs", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codex-diagnostic-test-")); + try { + const raw = '{"type":"item.completed","text":"' + "🏡".repeat(2000); + const file = await saveCodexDiagnostic( + directory, + new Error(`Failed to parse item: ${raw}`, { + cause: new SyntaxError("Unterminated string"), + }), + 26, + ); + const result = JSON.parse(await fs.readFile(file, "utf8")); + expect(result.rejectedEvent).toBe(raw); + expect(result.cause).toContain("Unterminated string"); + expect(result.truncated).toBe(false); + expect(result.eventCount).toBe(26); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("keeps images out of telemetry without mutating original events", () => { + const data = "a".repeat(10000); + const event = { type: "image", data }; + expect(safeJson(event)).toContain("binary omitted"); + expect(event.data).toBe(data); + }); +}); diff --git a/packages/integrations/codex-sdk/tests/isolation.test.ts b/packages/integrations/codex-sdk/tests/isolation.test.ts new file mode 100644 index 000000000..74dd7a048 --- /dev/null +++ b/packages/integrations/codex-sdk/tests/isolation.test.ts @@ -0,0 +1,29 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { isolatedCodexEnv } from "../src/isolation.js"; + +describe("Codex eval isolation", () => { + it("copies only file auth, not plugins, config or inherited thread context", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codex-isolation-test-")); + try { + const original = path.join(directory, "original"); + await fs.mkdir(original); + await fs.writeFile(path.join(original, "auth.json"), '{"tokens":{}}'); + await fs.writeFile(path.join(original, "config.toml"), "[mcp_servers.unrelated]"); + const env = await isolatedCodexEnv(path.join(directory, "eval"), { + CODEX_HOME: original, + CODEX_THREAD_ID: "host-thread", + PATH: "/bin", + }); + expect(env.CODEX_THREAD_ID).toBeUndefined(); + expect(env.PATH).toBe("/bin"); + expect(await fs.readdir(env.CODEX_HOME)).toEqual(["auth.json"]); + expect((await fs.stat(path.join(env.CODEX_HOME, "auth.json"))).mode & 0o777).toBe(0o600); + expect(env.HOME).not.toBe(os.homedir()); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/integrations/codex-sdk/tests/session.test.ts b/packages/integrations/codex-sdk/tests/session.test.ts index 3b84d7c6b..a13799875 100644 --- a/packages/integrations/codex-sdk/tests/session.test.ts +++ b/packages/integrations/codex-sdk/tests/session.test.ts @@ -14,6 +14,100 @@ import { const logger = { log: () => {}, warn: () => {}, error: () => {} }; describe("Codex SDK session", () => { + it("does not mask an SDK failure when diagnostics cannot be written", async () => { + const directory = await fsp.mkdtemp(path.join(os.tmpdir(), "codex-diagnostic-failure-")); + const blocked = path.join(directory, "file"); + const original = new Error("original SDK error"); + await fsp.writeFile(blocked, "not a directory"); + try { + const result = await runCodexSession({ + prompt: "task", + model: "", + logger, + thread: {}, + diagnosticDirectory: blocked, + sdk: { + startThread: () => { + throw original; + }, + }, + }); + expect(result.iterationError).toBe(original); + expect(result.diagnosticPath).toBeUndefined(); + expect(result.status).toBe("sdk_error"); + } finally { + await fsp.rm(directory, { recursive: true, force: true }); + } + }); + it("aborts unexpected MCP servers and saves a diagnostic", async () => { + const directory = await fsp.mkdtemp(path.join(os.tmpdir(), "codex-policy-")); + let signal: AbortSignal | undefined; + const sdk: CodexSdk = { + startThread: () => ({ + runStreamed: async (_prompt, options) => { + signal = options?.signal as AbortSignal; + return { + events: (async function* () { + yield { + type: "item.started", + item: { id: "bad", type: "mcp_tool_call", server: "node_repl" }, + }; + yield { type: "turn.completed" }; + })(), + }; + }, + }), + }; + try { + const result = await runCodexSession({ + prompt: "task", + model: "", + sdk, + logger, + thread: {}, + allowedMcpServers: ["stagehand"], + diagnosticDirectory: directory, + }); + expect(result.status).toBe("sdk_error"); + expect(signal?.aborted).toBe(true); + expect(result.events).toHaveLength(1); + expect(JSON.parse(await fsp.readFile(result.diagnosticPath!, "utf8"))).toMatchObject({ + message: "Unexpected MCP server: node_repl", + eventCount: 1, + }); + } finally { + await fsp.rm(directory, { recursive: true, force: true }); + } + }); + + it("awaits observation callbacks with the completed tool item", async () => { + const item = { id: "step-1", type: "mcp_tool_call", server: "stagehand" }; + let observed = false; + const sdk: CodexSdk = { + startThread: () => ({ + runStreamed: async () => ({ + events: (async function* () { + yield { type: "item.completed", item }; + expect(observed).toBe(true); + yield { type: "turn.completed" }; + })(), + }), + }), + }; + const result = await runCodexSession({ + prompt: "task", + model: "", + sdk, + logger, + thread: {}, + onToolStep: async (event) => { + await Promise.resolve(); + expect(event).toEqual(item); + observed = true; + }, + }); + expect(result.status).toBe("completed"); + }); it("normalizes provider-prefixed and default models", () => { expect(normalizeCodexModel("openai/gpt-5.4-mini")).toBe("gpt-5.4-mini"); expect(normalizeCodexModel("gpt-5.4")).toBe("gpt-5.4"); diff --git a/packages/integrations/codex/README.md b/packages/integrations/codex/README.md index 8b1dfc472..74de1ac71 100644 --- a/packages/integrations/codex/README.md +++ b/packages/integrations/codex/README.md @@ -65,3 +65,48 @@ service worker — browser-side, never on your machine. Browserbase is the recom boundary: the privileged execution environment is a disposable cloud browser. The SDK example spawns the facade server with an explicit `STAGEHAND_*`/`BROWSERBASE_*` allowlist; Codex's own model credentials never reach the browser session. +## Codex isolation and tool-call diagnostics + +The SDK example runs in a temporary working directory with separate `HOME` and +`CODEX_HOME`, so operator browser plugins, skills and MCP configuration are not +inherited. It copies only file-based `auth.json` when API credentials are absent; +keychain-only login is not copied. Use API credentials or file-based login for +this example. Unexpected MCP servers abort the run rather than contaminate its +evidence; this detects unexpected calls, not a security boundary preventing them. + +Enable detailed Stagehand logging before starting the example: + +```bash +STAGEHAND_BROWSER=local \ +STAGEHAND_FACADE_LOG_LEVEL=debug \ +STAGEHAND_FACADE_LOG_FILE=/tmp/stagehand-facade-tools.jsonl \ +STAGEHAND_CODEX_DIAGNOSTICS_DIR=/tmp/stagehand-codex-diagnostics \ +pnpm --filter @browserbasehq/stagehand-integrations-example-codex-facade start \ + "Go to Paint and draw the NFL logo" +``` + +Watch the server's actual tool arguments/code and result previews separately: + +```bash +tail -F /tmp/stagehand-facade-tools.jsonl | jq . +``` + +Logging defaults to off. `STAGEHAND_FACADE_LOG_LEVEL=calls` records arguments, +request IDs, timestamps, duration, output sizes and errors; `debug` also records +bounded result previews. Setting only `STAGEHAND_FACADE_LOG_FILE` enables `calls`. +Without a file, enabled logging uses stderr, never MCP stdout. Configure preview +length with `STAGEHAND_FACADE_LOG_MAX_CHARS` (default 16000; 256–1000000). +Files append across runs and carry a server session UUID and PID. New files use +owner-only permissions. Known credentials are redacted and image base64 omitted, +but code, page text and typed values may remain sensitive. Review before sharing +and rotate files yourself; preview limits do not bound total file size. + +Raw `codex exec --json | tee /tmp/codex-events.jsonl` only saves Codex events. +It does not enable facade logging or the SDK example's isolated profile. Pass +the `STAGEHAND_FACADE_LOG_*` variables in `mcp_servers.stagehand.env` when using +the raw CLI. Tool logging is diagnostic, not an eval scoring input. + +`STAGEHAND_CODEX_DIAGNOSTICS_DIR` enables private, bounded SDK error artifacts +for this example. Evals expose the same option as `EVAL_CODEX_DIAGNOSTICS_DIR`. +Artifacts may contain model/page content despite best-effort credential redaction. +Diagnostic write failures do not mask the original SDK failure. diff --git a/packages/integrations/codex/src/agent.ts b/packages/integrations/codex/src/agent.ts index 68f1fa936..90376761e 100644 --- a/packages/integrations/codex/src/agent.ts +++ b/packages/integrations/codex/src/agent.ts @@ -1,4 +1,11 @@ -import { loadCodexSdk, runCodexSession } from "@browserbasehq/stagehand-integrations-codex-sdk"; +import { + isolatedCodexEnv, + loadCodexSdk, + runCodexSession, +} from "@browserbasehq/stagehand-integrations-codex-sdk"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { fileURLToPath } from "node:url"; @@ -23,6 +30,7 @@ export function buildCodexConfig(): Record { command: process.execPath, args: [serverPath], env: buildAllowlistedEnv(), + default_tools_approval_mode: "approve", // Browser launches exceed the default MCP timeouts. startup_timeout_sec: 60, tool_timeout_sec: 300, @@ -42,46 +50,59 @@ async function main(): Promise { const instruction = (args[0] === "--" ? args.slice(1) : args).join(" ").trim(); if (!instruction) throw new Error('Usage: pnpm start "your instruction"'); - const sdk = await loadCodexSdk({ - ...(process.env.OPENAI_API_KEY ? { apiKey: process.env.OPENAI_API_KEY } : {}), - // pnpm can skip the SDK's vendored-binary postinstall; point at a locally - // installed codex when that happens (same escape hatch the evals harness - // uses). - ...(process.env.CODEX_PATH_OVERRIDE - ? { codexPathOverride: process.env.CODEX_PATH_OVERRIDE } - : {}), - extraConfig: buildCodexConfig(), - }); - const result = await runCodexSession({ - prompt: instruction, - // Codex picks its own harness-tuned default model; override only via env. - model: process.env.CODEX_STAGEHAND_MODEL ?? "", - sdk, - logger, - thread: { - // The browser work happens in the MCP server; the local sandbox can stay - // read-only. - sandboxMode: "read-only", - // Headless policy chosen empirically: "on-failure" lets MCP tool calls - // complete; "never" and "untrusted" auto-cancel them ("user cancelled - // MCP tool call"). - approvalPolicy: "on-failure", - skipGitRepoCheck: true, - }, - }); - if (result.status !== "completed") { - // Surface any answer the agent produced before the abnormal stop; a - // budget/max-turns stop usually still carries a useful final message. - throw ( - result.iterationError ?? - new Error( - `Agent did not finish: ${result.stopReason ?? result.status}` + - (result.finalMessage ? `\nLast agent message: ${result.finalMessage}` : ""), - ) - ); + const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "stagehand-codex-example-")); + try { + const env = await isolatedCodexEnv(cwd); + const sdk = await loadCodexSdk({ + env, + ...(process.env.OPENAI_API_KEY ? { apiKey: process.env.OPENAI_API_KEY } : {}), + // pnpm can skip the SDK's vendored-binary postinstall; point at a locally + // installed codex when that happens (same escape hatch the evals harness + // uses). + ...(process.env.CODEX_PATH_OVERRIDE + ? { codexPathOverride: process.env.CODEX_PATH_OVERRIDE } + : {}), + extraConfig: buildCodexConfig(), + }); + const result = await runCodexSession({ + prompt: `Use only the mounted Stagehand MCP tools for browser interaction.\n\n${instruction}`, + allowedMcpServers: ["stagehand"], + codexHome: env.CODEX_HOME, + diagnosticDirectory: process.env.STAGEHAND_CODEX_DIAGNOSTICS_DIR, + // Codex picks its own harness-tuned default model; override only via env. + model: process.env.CODEX_STAGEHAND_MODEL ?? "", + sdk, + logger, + thread: { + workingDirectory: cwd, + // The browser work happens in the MCP server; the local sandbox can stay + // read-only. + sandboxMode: "read-only", + // Headless policy chosen empirically: "on-failure" lets MCP tool calls + // complete; "never" and "untrusted" auto-cancel them ("user cancelled + // MCP tool call"). + approvalPolicy: "on-failure", + skipGitRepoCheck: true, + }, + }); + if (result.status !== "completed") { + if (result.diagnosticPath) + process.stderr.write(`Codex diagnostic saved: ${result.diagnosticPath}\n`); + // Surface any answer the agent produced before the abnormal stop; a + // budget/max-turns stop usually still carries a useful final message. + throw ( + result.iterationError ?? + new Error( + `Agent did not finish: ${result.stopReason ?? result.status}` + + (result.finalMessage ? `\nLast agent message: ${result.finalMessage}` : ""), + ) + ); + } + // oxlint-disable-next-line no-console -- CLI example prints the agent result. + console.log(result.finalMessage); + } finally { + await fs.rm(cwd, { recursive: true, force: true }); } - // oxlint-disable-next-line no-console -- CLI example prints the agent result. - console.log(result.finalMessage); } if (import.meta.main) { diff --git a/packages/integrations/core/src/facade/logging.ts b/packages/integrations/core/src/facade/logging.ts new file mode 100644 index 000000000..3009e53ee --- /dev/null +++ b/packages/integrations/core/src/facade/logging.ts @@ -0,0 +1,115 @@ +import { closeSync, constants, openSync, writeSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { sanitizeErrorMessage } from "../harness/redact.js"; + +export function redactToolLog(value: unknown, maxChars = 16_000): unknown { + const seen = new WeakSet(); + const text = + JSON.stringify(value, (key, item: unknown) => { + if (/^(?:authorization|cookie|password|secret|token|api[_-]?key|signingKey)$/i.test(key)) + return "[redacted]"; + if (key === "data" && typeof item === "string" && item.length > 256) + return `[binary omitted: ${item.length} characters]`; + if (typeof item === "string") + return sanitizeErrorMessage(item).replace( + /((?:password|api[_-]?key|secret|token)\s*[:=]\s*["']?)[^\s"',;}]+/gi, + "$1[redacted]", + ); + if (typeof item === "bigint") return String(item); + if (item && typeof item === "object") { + if (seen.has(item)) return "[Circular]"; + seen.add(item); + } + return item; + }) ?? "null"; + return text.length > maxChars + ? { preview: text.slice(0, maxChars), truncated: true, characters: text.length } + : JSON.parse(text); +} + +export function createFacadeLogger(env: NodeJS.ProcessEnv = process.env) { + const file = env.STAGEHAND_FACADE_LOG_FILE; + const level = env.STAGEHAND_FACADE_LOG_LEVEL ?? (file ? "calls" : "off"); + if (!["off", "calls", "debug"].includes(level)) + throw new Error("STAGEHAND_FACADE_LOG_LEVEL must be off, calls, or debug."); + const parsedLimit = Number(env.STAGEHAND_FACADE_LOG_MAX_CHARS ?? 16_000); + if (!Number.isSafeInteger(parsedLimit) || parsedLimit < 256 || parsedLimit > 1_000_000) + throw new Error("STAGEHAND_FACADE_LOG_MAX_CHARS must be between 256 and 1000000."); + const session = randomUUID(); + let descriptor: number | undefined; + if (file && level !== "off") + descriptor = openSync( + file, + constants.O_CREAT | constants.O_APPEND | constants.O_WRONLY | constants.O_NOFOLLOW, + 0o600, + ); + const emit = (event: string, details: Record) => { + if (level === "off") return; + const line = + JSON.stringify({ + timestamp: new Date().toISOString(), + session, + pid: process.pid, + event, + ...details, + }) + "\n"; + try { + if (descriptor !== undefined) writeSync(descriptor, line); + else process.stderr.write(line); + } catch { + if (descriptor !== undefined) { + try { + closeSync(descriptor); + } catch {} + } + descriptor = undefined; + process.stderr.write("Stagehand tool log write failed; subsequent log records use stderr.\n"); + } + }; + return { + emit, + async call( + id: string, + name: string, + args: unknown, + execute: () => Promise, + ): Promise { + const started = performance.now(); + emit("tool.start", { id, name, arguments: redactToolLog(args, parsedLimit) }); + try { + const result = await execute(); + const object = result as { + isError?: boolean; + content?: Array<{ type: string; text?: string; data?: string }>; + }; + emit("tool.end", { + id, + name, + durationMs: Math.round(performance.now() - started), + status: object?.isError ? "error" : "ok", + content: object?.content?.map((block) => ({ + type: block.type, + characters: block.text?.length ?? block.data?.length ?? 0, + })), + ...(level === "debug" || object?.isError + ? { result: redactToolLog(result, parsedLimit) } + : {}), + }); + return result; + } catch (error) { + emit("tool.end", { + id, + name, + durationMs: Math.round(performance.now() - started), + status: "error", + error: redactToolLog(String(error), parsedLimit), + }); + throw error; + } + }, + close() { + if (descriptor !== undefined) closeSync(descriptor); + descriptor = undefined; + }, + }; +} diff --git a/packages/integrations/core/src/facade/output.ts b/packages/integrations/core/src/facade/output.ts new file mode 100644 index 000000000..0986b6463 --- /dev/null +++ b/packages/integrations/core/src/facade/output.ts @@ -0,0 +1,6 @@ +export function transportSafeText(text: string): string { + return text.replace( + /[\u0085\u2028\u2029]/gu, + (character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`, + ); +} diff --git a/packages/integrations/core/src/facade/stdio-server.ts b/packages/integrations/core/src/facade/stdio-server.ts index 8197aa91c..fbc8db300 100644 --- a/packages/integrations/core/src/facade/stdio-server.ts +++ b/packages/integrations/core/src/facade/stdio-server.ts @@ -26,6 +26,8 @@ import { screenshotBase64BudgetFromArgs, } from "./screenshot-transport.js"; import { StagehandFacadeTools } from "./tools.js"; +import { createFacadeLogger } from "./logging.js"; +import { transportSafeText } from "./output.js"; type FacadeResources = { browser: StagehandBrowser; @@ -34,6 +36,7 @@ type FacadeResources = { }; const server = new McpServer({ name: "stagehand-facade", version: "4.0.0" }); +const toolLogger = createFacadeLogger(); const screenshotBase64Budget = screenshotBase64BudgetFromArgs(process.argv.slice(2)); const facadeTools = facadeToolsFor(facadeSurfaceFromArgs(process.argv.slice(2))); let resourcesPromise: Promise | undefined; @@ -58,70 +61,77 @@ server.registerTool( server.server.removeRequestHandler("tools/list"); server.server.setRequestHandler(ListToolsRequestSchema, () => ({ tools: [...facadeTools] })); server.server.removeRequestHandler("tools/call"); -server.server.setRequestHandler(CallToolRequestSchema, async (request) => { - try { - const args = request.params.arguments ?? {}; - switch (request.params.name) { - case "run": { - const input = CodeModeRunInputSchema.parse(args); - const tools = (await ensureResources()).tools; - const result = - input.code === undefined - ? await tools.runActions(input.actions!) - : await tools.run(input.code); - return textResult(stringifyResult(result)); - } - case "snapshot": { - const input = SnapshotInputSchema.parse(args); - const result = await (await ensureResources()).tools.snapshot(input); - return textResult(result); - } - case "screenshot": { - const input = ScreenshotInputSchema.parse(args); - const tools = (await ensureResources()).tools; - const screenshot = - screenshotBase64Budget === undefined - ? { image: await tools.screenshot(input), adjusted: false } - : await captureScreenshotWithinBase64Budget( - (options) => tools.screenshot(options), - input, - screenshotBase64Budget, - ); - return { - content: [ - { - type: "text" as const, - text: screenshot.adjusted - ? "Screenshot captured with transport-safe compression." - : "Screenshot captured.", - }, - { - type: "image" as const, - data: screenshot.image.data, - mimeType: screenshot.image.mimeType, - }, - ], - }; - } - case SESSION_INFO_TOOL_NAME: { - // Runner-side only (absent from tools/list): launches the browser if - // needed and reports where it lives so the harness can log the - // Browserbase session URL before the agent's first call. - const browser = (await ensureResources()).browser; - return textResult( - JSON.stringify({ - provider: browser.provider, - ...(browser.sessionId && { sessionId: browser.sessionId }), - }), - ); +server.server.setRequestHandler(CallToolRequestSchema, async (request, extra) => + toolLogger.call( + String(extra.requestId), + request.params.name, + request.params.arguments ?? {}, + async () => { + try { + const args = request.params.arguments ?? {}; + switch (request.params.name) { + case "run": { + const input = CodeModeRunInputSchema.parse(args); + const tools = (await ensureResources()).tools; + const result = + input.code === undefined + ? await tools.runActions(input.actions!) + : await tools.run(input.code); + return textResult(stringifyResult(result)); + } + case "snapshot": { + const input = SnapshotInputSchema.parse(args); + const result = await (await ensureResources()).tools.snapshot(input); + return textResult(result); + } + case "screenshot": { + const input = ScreenshotInputSchema.parse(args); + const tools = (await ensureResources()).tools; + const screenshot = + screenshotBase64Budget === undefined + ? { image: await tools.screenshot(input), adjusted: false } + : await captureScreenshotWithinBase64Budget( + (options) => tools.screenshot(options), + input, + screenshotBase64Budget, + ); + return { + content: [ + { + type: "text" as const, + text: screenshot.adjusted + ? "Screenshot captured with transport-safe compression." + : "Screenshot captured.", + }, + { + type: "image" as const, + data: screenshot.image.data, + mimeType: screenshot.image.mimeType, + }, + ], + }; + } + case SESSION_INFO_TOOL_NAME: { + // Runner-side only (absent from tools/list): launches the browser if + // needed and reports where it lives so the harness can log the + // Browserbase session URL before the agent's first call. + const browser = (await ensureResources()).browser; + return textResult( + JSON.stringify({ + provider: browser.provider, + ...(browser.sessionId && { sessionId: browser.sessionId }), + }), + ); + } + default: + throw new Error(`Unknown tool: ${request.params.name}`); + } + } catch (error) { + return errorResult(error); } - default: - throw new Error(`Unknown tool: ${request.params.name}`); - } - } catch (error) { - return errorResult(error); - } -}); + }, + ), +); async function ensureResources(): Promise { resourcesPromise ??= createResources().catch((error) => { @@ -163,6 +173,7 @@ async function createResources(): Promise { })}\n`, ), }); + toolLogger.emit("browser.ready", { provider: browser.provider, sessionId: browser.sessionId }); return { browser, stagehand, tools }; } catch (error) { await browser.close().catch(() => undefined); @@ -171,12 +182,12 @@ async function createResources(): Promise { } function textResult(text: string) { - return { content: [{ type: "text" as const, text }] }; + return { content: [{ type: "text" as const, text: transportSafeText(text) }] }; } function errorResult(error: unknown) { const message = sanitizeErrorMessage(error instanceof Error ? error.message : String(error)); - return { content: [{ type: "text" as const, text: message }], isError: true }; + return { ...textResult(message), isError: true }; } function stringifyResult(value: unknown): string { @@ -207,6 +218,7 @@ async function shutdown(code: number): Promise { server, ]); if (!clean) process.stderr.write("Failed to close Stagehand facade cleanly.\n"); + toolLogger.close(); process.exit(code === 0 && !clean ? 1 : code); } diff --git a/packages/integrations/core/tests/facade-logging.test.ts b/packages/integrations/core/tests/facade-logging.test.ts new file mode 100644 index 000000000..252f8f782 --- /dev/null +++ b/packages/integrations/core/tests/facade-logging.test.ts @@ -0,0 +1,60 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { createFacadeLogger, redactToolLog } from "../src/facade/logging.js"; + +describe("facade tool logging", () => { + it("redacts credentials, excludes images and bounds large records", () => { + const result = JSON.stringify( + redactToolLog({ + password: "private", + authorization: "secret", + code: 'const token="sensitive";', + data: "a".repeat(1000), + }), + ); + expect(result).not.toContain("private"); + expect(result).not.toContain("sensitive"); + expect(result).toContain("binary omitted"); + expect(redactToolLog({ code: "x".repeat(1000) }, 256)).toMatchObject({ truncated: true }); + }); + + it("pairs call IDs with code, status and timing without using stdout", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "facade-log-test-")); + const file = path.join(directory, "calls.jsonl"); + const logger = createFacadeLogger({ + STAGEHAND_FACADE_LOG_FILE: file, + STAGEHAND_FACADE_LOG_LEVEL: "debug", + }); + try { + await logger.call("42", "run", { code: "return 1" }, async () => ({ + content: [{ type: "text", text: "1" }], + })); + await logger.call("43", "snapshot", {}, async () => ({ + isError: true, + content: [{ type: "text", text: "unavailable" }], + })); + const records = (await fs.readFile(file, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(records[0]).toMatchObject({ + event: "tool.start", + id: "42", + arguments: { code: "return 1" }, + }); + expect(records[1]).toMatchObject({ + event: "tool.end", + id: "42", + status: "ok", + durationMs: expect.any(Number), + }); + expect(records[3]).toMatchObject({ id: "43", status: "error" }); + expect((await fs.stat(file)).mode & 0o777).toBe(0o600); + } finally { + logger.close(); + await fs.rm(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/integrations/core/tests/facade-output.test.ts b/packages/integrations/core/tests/facade-output.test.ts new file mode 100644 index 000000000..803d04182 --- /dev/null +++ b/packages/integrations/core/tests/facade-output.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from "vitest"; +import { transportSafeText } from "../src/facade/output.js"; + +describe("facade transport text", () => { + it("escapes Unicode line separators without changing structured JSON values", () => { + const value = { text: "people.\u2028Next\u2029paragraph\u0085line 🏈" }; + const text = transportSafeText(JSON.stringify(value)); + expect(text).not.toMatch(/[\u0085\u2028\u2029]/u); + expect(JSON.parse(text)).toEqual(value); + expect(transportSafeText("ordinary text")).toBe("ordinary text"); + }); +}); diff --git a/packages/integrations/core/tests/facade-server.test.ts b/packages/integrations/core/tests/facade-server.test.ts index b60638427..10fb9ee88 100644 --- a/packages/integrations/core/tests/facade-server.test.ts +++ b/packages/integrations/core/tests/facade-server.test.ts @@ -1,4 +1,7 @@ import { fileURLToPath } from "node:url"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import type { Stream } from "node:stream"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; @@ -11,12 +14,19 @@ const readyMessage = "Stagehand facade MCP host listening on stdio"; describe("built Stagehand facade stdio server", () => { let client: Client; let transport: StdioClientTransport; + let directory: string; beforeEach(async () => { + directory = await fs.mkdtemp(path.join(os.tmpdir(), "facade-stdio-logs-")); transport = new StdioClientTransport({ command: process.execPath, args: [entrypoint], - env: { PATH: process.env.PATH ?? "", STAGEHAND_BROWSER: "invalid" }, + env: { + PATH: process.env.PATH ?? "", + STAGEHAND_BROWSER: "invalid", + STAGEHAND_FACADE_LOG_LEVEL: "debug", + STAGEHAND_FACADE_LOG_FILE: path.join(directory, "tools.jsonl"), + }, stderr: "pipe", }); if (!transport.stderr) throw new Error("stdio transport did not expose stderr"); @@ -27,6 +37,7 @@ describe("built Stagehand facade stdio server", () => { afterEach(async () => { await client.close(); + await fs.rm(directory, { recursive: true, force: true }); }); it("initializes and lists the exact tools without launching a browser", async () => { @@ -43,6 +54,20 @@ describe("built Stagehand facade stdio server", () => { const unknown = await client.callTool({ name: "missing", arguments: {} }); expect(unknown.isError).toBe(true); + const logs = (await fs.readFile(path.join(directory, "tools.jsonl"), "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(logs.map((record) => record.event)).toEqual([ + "tool.start", + "tool.end", + "tool.start", + "tool.end", + ]); + expect(logs[0]).toMatchObject({ name: "run", arguments: {} }); + expect(logs[1]).toMatchObject({ id: logs[0].id, status: "error" }); + expect(logs[3].result.isError).toBe(true); + await expect(client.ping()).resolves.toBeDefined(); }); diff --git a/packages/integrations/core/tests/facade-stdio-session-age.test.ts b/packages/integrations/core/tests/facade-stdio-session-age.test.ts index d4a2a3f8d..94882611c 100644 --- a/packages/integrations/core/tests/facade-stdio-session-age.test.ts +++ b/packages/integrations/core/tests/facade-stdio-session-age.test.ts @@ -69,10 +69,13 @@ it("includes browser launch and Stagehand initialization in measured session age vi.spyOn(process, "once").mockReturnValue(process); vi.spyOn(process.stdin, "once").mockReturnValue(process.stdin); await import("../src/facade/stdio-server.js"); - const handler = mocks.setRequestHandler.mock.calls.at(-1)?.[1] as (request: { - params: { name: string; arguments: object }; - }) => Promise; - await handler({ params: { name: "snapshot", arguments: {} } }); + const handler = mocks.setRequestHandler.mock.calls.at(-1)?.[1] as ( + request: { + params: { name: string; arguments: object }; + }, + extra: { requestId: string }, + ) => Promise; + await handler({ params: { name: "snapshot", arguments: {} } }, { requestId: "snapshot-1" }); const telemetry = output.mock.calls .map(([chunk]) => String(chunk)) .find((line) => line.startsWith(SESSION_LOST_TELEMETRY_PREFIX));