diff --git a/.changeset/eval-capture-deadlines.md b/.changeset/eval-capture-deadlines.md new file mode 100644 index 000000000..94edd0cd9 --- /dev/null +++ b/.changeset/eval-capture-deadlines.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Bound experimental batch and RPC deadlines so callers can stop waiting without replaying actions or accepting late capture state. diff --git a/.changeset/eval-cdp-heartbeat.md b/.changeset/eval-cdp-heartbeat.md new file mode 100644 index 000000000..73125c9a6 --- /dev/null +++ b/.changeset/eval-cdp-heartbeat.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Add configurable bounded CDP heartbeats and sanitized disconnect diagnostics with cleanup on shutdown. diff --git a/.changeset/eval-frame-snapshot-maps.md b/.changeset/eval-frame-snapshot-maps.md new file mode 100644 index 000000000..56ded681b --- /dev/null +++ b/.changeset/eval-frame-snapshot-maps.md @@ -0,0 +1,7 @@ +--- +"@browserbasehq/stagehand-extension": patch +"@browserbasehq/stagehand-go": patch +--- + +Snapshot references remain valid across same-origin and out-of-process frame captures, including when shadow-root piercing is disabled. +Refresh the Go SDK's embedded extension to include the fix. diff --git a/packages/evals/core/contracts/tool.ts b/packages/evals/core/contracts/tool.ts index e6dbdc0f5..15a8b0c76 100644 --- a/packages/evals/core/contracts/tool.ts +++ b/packages/evals/core/contracts/tool.ts @@ -140,6 +140,11 @@ export interface BrowserSessionLoss { cause: string; tool?: string; at?: string; + provider?: "local" | "browserbase"; + sessionId?: string; + /** Elapsed time since the facade started browser launch, including initialization. */ + sessionAgeMs?: number; + sessionTimeoutMs?: number; } /** MCP content returned unchanged by a runner call into its existing surface. */ diff --git a/packages/evals/core/tools/browserSessionLoss.ts b/packages/evals/core/tools/browserSessionLoss.ts index 8ea5c804d..206125bf3 100644 --- a/packages/evals/core/tools/browserSessionLoss.ts +++ b/packages/evals/core/tools/browserSessionLoss.ts @@ -33,6 +33,22 @@ export function parseSessionLossTelemetry(line: string): BrowserSessionLoss | un cause: sanitizeErrorMessage(parsed.cause), ...(typeof parsed.tool === "string" && { tool: parsed.tool }), ...(typeof parsed.at === "string" && { at: parsed.at }), + ...((parsed.provider === "local" || parsed.provider === "browserbase") && { + provider: parsed.provider, + }), + ...(typeof parsed.sessionId === "string" && { + sessionId: sanitizeErrorMessage(parsed.sessionId), + }), + ...(typeof parsed.sessionAgeMs === "number" && + Number.isFinite(parsed.sessionAgeMs) && + parsed.sessionAgeMs >= 0 && { + sessionAgeMs: parsed.sessionAgeMs, + }), + ...(typeof parsed.sessionTimeoutMs === "number" && + Number.isFinite(parsed.sessionTimeoutMs) && + parsed.sessionTimeoutMs >= 0 && { + sessionTimeoutMs: parsed.sessionTimeoutMs, + }), }; } catch { return undefined; diff --git a/packages/evals/docs/verifier-gates.md b/packages/evals/docs/verifier-gates.md new file mode 100644 index 000000000..187a5ce9b --- /dev/null +++ b/packages/evals/docs/verifier-gates.md @@ -0,0 +1,29 @@ +# Verifier evidence gates + +External harnesses use the existing V3 verifier with the task's precomputed rubric when available. A requested verification that errors or returns a verifier-uncertainty sentinel fails closed: `_success` is false, `verifierError` explains the failure, and `agentReportedSuccess` preserves the original self-report. Such rows are ungraded and must not be presented as verified benchmark outcomes. + +For completed grades, the raw judge verdict is retained under `judge` in `scores/result.json`. The top-level result, `task_data.json`, and the run row carry the same adjusted outcome. Failed or uncertain verification instead persists `{ graded: false, verifierError, judge? }`, without top-level outcome or process scores; `judge` is present only if the verifier returned a response. `scores/gates.json` records diagnostics for completed grades when trajectory persistence is enabled. + +## Outcome + +A judge pass is rejected when the final answer is empty or, if a mounted-tool matcher is available, the trajectory has no browser tool calls. Grounding checks are advisory by default. `EVAL_REQUIRE_GROUNDING=1` additionally rejects an answer whose checked numeric findings all lack matching observations from a known non-search page. Captured step and terminal accessibility trees are included; unknown-page text cannot establish grounding. A terminal match is recorded as `groundedAtFinalObservation`, while untrusted matches can be recorded as `seenOnUnknownPage`. The check is a text heuristic: images, paraphrases and valid snippet sources can escape its matching, so it is not a replacement for rubric verification. + +Execution state remains separate. When populated by the runner or adapter, `harnessStatus`, `harnessStopReason` and `terminationReason` describe completion, budget exhaustion, abort, SDK error and browser loss. The verifier does not synthesize missing lifecycle metadata; its absence means unavailable. A supported task completion can still pass after a disconnect; execution error alone does not erase earlier evidence. An unfinished task must fail its rubric. + +## Process + +`processScoreStrict` recomputes the weighted process score with explicit `evidenceInsufficient` criteria earning zero while retaining their maximum points in the denominator. `processScore` uses this score; `processScoreLenient` preserves the judge's aggregate. Not-applicable criteria are excluded. Without a criterion breakdown, the judge's aggregate is retained and `scoringIncomplete` flags a short result against the rubric. + +Blocker wording is recorded as `blockerMentioned` on criterion diagnostics. It never changes points by itself: permitted fallback and stop-boundary explanations can correctly mention a blocker. The rubric and observed evidence determine whether the requirement was satisfied. This replaces the campaign's overbroad blocker substring heuristic. + +## Reporting + +These fields depend on the producing runner; this verifier layer forwards them but does not make every harness emit them: + +- Where supplied, `facade_tool_calls` and `facade_tool_call_failures` count attempted and failed browser work. Missing counters are unknown, not measured zero. Run-level browser loss comes from runner-owned telemetry. Normalized steps do not provide trusted per-call loss attribution, so tool-output text cannot exclude failures or synthesize a count after session loss. A graded pass with an explicit zero browser-call count is shown in the batch summary; with `EVAL_MAX_UNVERIFIABLE_CRITERIA` enabled, it fails the batch gate. +- Separate agent, evidence-capture and verifier wall times are available only when recorded by the producer. +- Usage must be interpreted with the producer's presence marker and cache convention. Legacy runners may supply zero placeholders; without an explicit presence marker, zero does not establish measured usage. Historical Cursor CLI usage remains unreported. +- A producer's `cost_source` distinguishes reported dollars from a catalog estimate (`computed`). Shared runner estimates use the dated catalog in `pricing/pricing.json`; they are not invoices. This verifier layer does not compute estimates. Without provenance, cost origin is unavailable; unknown, tier-dependent or subscription costs must not be inferred as zero. +- `harnessImplementation` records adapter and SDK versions when supplied. Its absence means unknown implementation; historical labels are preserved. + +Use `VERIFIER_PERSIST_TRAJECTORIES=1` for reviewable evidence. HardBench's compatibility gate rejects verifier errors, uncertainty sentinels, missing criteria and self-report fallbacks before accepting a result. Offline transport checks establish integration compatibility; live rubric accuracy still requires the separately recorded live fixtures. diff --git a/packages/evals/framework/agentToolRuntime.ts b/packages/evals/framework/agentToolRuntime.ts index b5d41f4e0..c786c441d 100644 --- a/packages/evals/framework/agentToolRuntime.ts +++ b/packages/evals/framework/agentToolRuntime.ts @@ -3,6 +3,7 @@ import { prepareCoreBrowserTarget } from "../core/targets/index.js"; import { getCoreTool } from "../core/tools/registry.js"; import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; +import { browserSessionFromMetadata, type BrowserSessionInfo } from "./browserSession.js"; export interface AgentToolRuntimeInput { toolSurface: ToolSurface; @@ -13,6 +14,12 @@ export interface AgentToolRuntimeInput { export interface StartedAgentToolRuntime { running: ToolStartResult; + /** + * Browser behind the surface, whether the runner provided it (Browserbase + * CDP target) or the tool created it (facade, stagehand_code). Known before + * the agent starts so the session URL can head the task log. + */ + browserSession: BrowserSessionInfo; /** Closes the tool-owned runtime, then the runner-owned browser target. */ cleanup: () => Promise; } @@ -49,6 +56,10 @@ export async function startAgentToolRuntime( let cleanupPromise: Promise | undefined; return { running, + browserSession: browserSessionFromMetadata( + { ...running.metadata, ...target.metadata }, + input.environment, + ), cleanup: async () => { cleanupPromise ??= (async () => { try { diff --git a/packages/evals/framework/benchHarness.ts b/packages/evals/framework/benchHarness.ts index 99c8f3d61..71f9766fd 100644 --- a/packages/evals/framework/benchHarness.ts +++ b/packages/evals/framework/benchHarness.ts @@ -1,5 +1,6 @@ import { V3, normalizeRubric, type AvailableModel, type TaskSpec } from "stagehand-v3"; import { EvalsError } from "../errors.js"; +import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; import type { EvalLogger } from "../logger.js"; import type { StagehandInitResult } from "../initStagehand.js"; import type { EvalInput } from "../types/evals.js"; @@ -26,7 +27,13 @@ import { buildExternalHarnessTaskPlan, type ExternalHarnessTaskPlan, } from "./externalHarnessPlan.js"; +import { + logBrowserSession, + withBrowserSession, + type BrowserSessionInfo, +} from "./browserSession.js"; import { withHarnessAgentSpan } from "./otel.js"; +import { verifierTraceEnabled } from "./verifierTrace.js"; import type { DiscoveredTask, TaskResult } from "./types.js"; import type { BenchMatrixRow, BenchTaskKind, Harness } from "./benchTypes.js"; import { DEFAULT_BENCH_HARNESS } from "./benchTypes.js"; @@ -69,7 +76,7 @@ export interface BenchHarness { supportsApi: boolean; /** * Tool surfaces this harness can mount for the agent, in display order; the - * first entry is the default when --tool is omitted. An empty list means the + * facade is preferred when --tool is omitted, otherwise the first entry. An empty list means the * harness does not mount tool surfaces and the planner passes the requested * surface/profile through unchanged as row metadata (stagehand harness). */ @@ -105,7 +112,14 @@ export interface ExternalHarnessRunInput { verifier: ExternalHarnessVerifierConfig; } -export interface ExternalHarnessDefinition Promise }> { +/** What every prepared external-harness adapter must expose to the shared lifecycle. */ +export interface ExternalHarnessAdapterBase { + cleanup: () => Promise; + /** Browser behind the mounted surface; logged before the agent starts. */ + browserSession?: BrowserSessionInfo; +} + +export interface ExternalHarnessDefinition { harness: string; supportedToolSurfaces: ToolSurface[]; defaultModels: AvailableModel[]; @@ -119,7 +133,7 @@ export interface ExternalHarnessDefinition Pro * Define the lifecycle common to external agent harnesses without registering * it; registry ownership stays explicit so list order remains deterministic. */ -export function defineExternalHarness Promise }>( +export function defineExternalHarness( definition: ExternalHarnessDefinition, ): BenchHarness { const { @@ -148,6 +162,9 @@ export function defineExternalHarness Promise< // the adapter and the carrier. const carrierV3 = buildVerifierCarrierV3(logger); let toolAdapter: TAdapter | undefined; + let browserSession: BrowserSessionInfo = { + provider: row.config.environment === "BROWSERBASE" ? "browserbase" : "local", + }; try { toolAdapter = await prepareToolAdapter({ toolSurface: row.config.toolSurface, @@ -157,7 +174,9 @@ export function defineExternalHarness Promise< logger, }); const preparedAdapter = toolAdapter; - return await withHarnessAgentSpan( + browserSession = preparedAdapter.browserSession ?? browserSession; + logBrowserSession(logger, browserSession); + const result = await withHarnessAgentSpan( { harness, model: input.modelName, @@ -178,6 +197,18 @@ export function defineExternalHarness Promise< }, }), ); + return withBrowserSession(result, browserSession); + } catch (error) { + return withBrowserSession( + { + _success: false, + error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), + harnessStatus: "sdk_error", + terminationReason: "sdk_error", + logs: logger.getLogs(), + }, + browserSession, + ); } finally { try { await toolAdapter?.cleanup(); @@ -208,7 +239,9 @@ function buildVerifierCarrierV3(logger: EvalLogger): V3 { disablePino: true, disableAPI: true, experimental: true, - verbose: 0, + // verbose 2 surfaces the judge's LLM request/response lines (level 2), + // which verifierAdapter routes to scores/verifier-trace.jsonl. + verbose: verifierTraceEnabled() ? 2 : 0, }); } diff --git a/packages/evals/framework/benchPlanner.ts b/packages/evals/framework/benchPlanner.ts index ad35a6f79..595ab6023 100644 --- a/packages/evals/framework/benchPlanner.ts +++ b/packages/evals/framework/benchPlanner.ts @@ -1,5 +1,6 @@ import type { AvailableModel } from "stagehand-v3"; import { EvalsError } from "../errors.js"; +import { explicitSnapshotActionsEnabled } from "@browserbasehq/stagehand-integrations/facade"; import { buildOnlineMind2WebTestcases } from "../suites/onlineMind2Web.js"; import { buildHardBenchmarkTestcases } from "../suites/hardbenchmark.js"; import { buildWebTailBenchTestcases } from "../suites/webtailbench.js"; @@ -366,11 +367,15 @@ function withBenchMetadata( } function buildToolMetadata(row: BenchMatrixRow): Partial { + const promptVariant = + row.toolSurface === "stagehand_facade" && explicitSnapshotActionsEnabled() + ? { promptVariant: "explicit_snapshot_actions" } + : {}; if ( getBenchHarness(row.harness).supportedToolSurfaces.includes("browse_cli") && row.toolSurface === "browse_cli" ) { - return getBrowseCliToolMetadata(); + return { ...getBrowseCliToolMetadata(), ...promptVariant }; } - return {}; + return promptVariant; } diff --git a/packages/evals/framework/browserSession.ts b/packages/evals/framework/browserSession.ts new file mode 100644 index 000000000..855fd80f3 --- /dev/null +++ b/packages/evals/framework/browserSession.ts @@ -0,0 +1,89 @@ +import type { LogLine } from "stagehand-v3"; +import type { TaskResult } from "./types.js"; + +export const BROWSER_SESSION_LOG_CATEGORY = "session"; + +/** Where the browser behind a run lives, resolved before the agent starts. */ +export interface BrowserSessionInfo { + provider: "browserbase" | "local"; + sessionId?: string; + sessionUrl?: string; + debugUrl?: string; +} + +export function browserbaseSessionUrl(sessionId: string): string { + return `https://www.browserbase.com/sessions/${encodeURIComponent(sessionId)}`; +} + +/** + * Read the session fields core tools and runner-provided targets publish on + * their `metadata` (`browserbaseSessionId` / `browserbaseSessionUrl` / + * `browserbaseDebugUrl`). Falls back to the bare provider when a Browserbase + * surface does not report its session id (browse_cli). + */ +export function browserSessionFromMetadata( + metadata: Record | undefined, + environment: "LOCAL" | "BROWSERBASE", +): BrowserSessionInfo { + if (environment !== "BROWSERBASE") return { provider: "local" }; + const rawUrl = readString(metadata?.browserbaseSessionUrl); + const sessionId = + readString(metadata?.browserbaseSessionId) ?? rawUrl?.match(/\/sessions\/([^/?#]+)/u)?.[1]; + const sessionUrl = rawUrl ?? (sessionId ? browserbaseSessionUrl(sessionId) : undefined); + const debugUrl = readString(metadata?.browserbaseDebugUrl); + return { + provider: "browserbase", + ...(sessionId && { sessionId }), + ...(sessionUrl && { sessionUrl }), + ...(debugUrl && { debugUrl }), + }; +} + +export function formatBrowserSessionMessage(info: BrowserSessionInfo): string { + if (info.provider === "local") return "Browser: local"; + if (!info.sessionUrl) return "Browser: browserbase (session id not reported by this surface)"; + return `Browserbase session: ${info.sessionUrl}`; +} + +/** Level-0 lines so the session pointer survives every log filter. */ +export function buildBrowserSessionLogLines(info: BrowserSessionInfo): LogLine[] { + const lines: LogLine[] = [ + { + category: BROWSER_SESSION_LOG_CATEGORY, + level: 0, + message: formatBrowserSessionMessage(info), + auxiliary: { + provider: { value: info.provider, type: "string" }, + ...(info.sessionId && { sessionId: { value: info.sessionId, type: "string" } }), + ...(info.sessionUrl && { sessionUrl: { value: info.sessionUrl, type: "string" } }), + }, + }, + ]; + if (info.debugUrl) { + lines.push({ + category: BROWSER_SESSION_LOG_CATEGORY, + level: 0, + message: `Browserbase debugger: ${info.debugUrl}`, + }); + } + return lines; +} + +export function logBrowserSession(sink: { log(line: LogLine): void }, info: BrowserSessionInfo) { + for (const line of buildBrowserSessionLogLines(info)) sink.log(line); +} + +/** Surface the session on the TaskResult row so Braintrust output is filterable. */ +export function withBrowserSession(result: TaskResult, info: BrowserSessionInfo): TaskResult { + return { + ...result, + browserProvider: info.provider, + ...(info.sessionId && { browserbaseSessionId: info.sessionId }), + ...(info.sessionUrl && { sessionUrl: result.sessionUrl || info.sessionUrl }), + ...(info.debugUrl && { debugUrl: result.debugUrl || info.debugUrl }), + }; +} + +function readString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value : undefined; +} diff --git a/packages/evals/framework/costEstimate.ts b/packages/evals/framework/costEstimate.ts new file mode 100644 index 000000000..afbf56598 --- /dev/null +++ b/packages/evals/framework/costEstimate.ts @@ -0,0 +1,273 @@ +import fs from "node:fs"; +import path from "node:path"; +import { getPackageRootDir } from "../runtimePaths.js"; +import type { NormalizedUsage } from "./usageNormalization.js"; + +/** USD per million tokens. `null` marks a model the owner has not priced yet. */ +export interface ModelPrice { + input_per_m: number | null; + cached_input_per_m: number | null; + /** Cache-write rate where the provider bills one; falls back to `input_per_m`. */ + cache_write_input_per_m?: number | null; + output_per_m: number | null; + source: string; + note?: string; +} + +export interface PriceMap { + as_of: string; + models: Record; +} + +/** + * Where `cost_usd` came from: + * - `reported`: the harness's own billing channel reported dollars. + * - `computed`: the harness called the provider API directly with our key, so + * the estimate is normalized tokens × the dated catalog price in pricing.json. + * - `unavailable`: neither — subscription-billed cells (cursor, claude_code on + * a plan) or a model missing from the price map. No cost metric is emitted; + * token efficiency stays on the usage_* metrics. + */ +export type CostSource = "reported" | "computed" | "unavailable"; + +export interface BilledCost { + cost_usd?: number; + cost_source: CostSource; + /** Who billed the tokens, e.g. "anthropic_api", "ai_gateway", "pi_catalog", "subscription". */ + billing_channel: string; + /** Catalog provenance for a list-price estimate, never for a reported bill. */ + cost_pricing?: { as_of: string; model: string; source: string }; +} + +/** + * Billing channel per harness. + * + * | harness | reports dollars? | channel when reported | when not reported | + * |-------------|-----------------------------------------|-----------------------|-------------------------------------------------------| + * | claude_code | total_cost_usd on the result message | anthropic_api | subscription (Claude plan; no dollars, unavailable) | + * | eve | costUsd per step (gateway-routed models)| ai_gateway | first-party creators run direct → computed

_api | + * | pi | usage.cost.total from pi's model catalog| pi_catalog | direct provider call → computed _api | + * | fx | total_cost in usage-v2.json | fx_gateway | fx always bills via its gateway → unavailable | + * | codex | never (turn.completed has tokens only) | — | OpenAI API with our key → computed openai_api | + * | mastra | never (AI SDK usage has no dollars) | — | provider SDK with our key → computed _api | + * | deepagents | never (LangChain usage_metadata) | — | provider SDK with our key → computed _api | + * | cursor | never | — | subscription → unavailable | + */ +const REPORTED_CHANNEL: Readonly> = { + claude_code: "anthropic_api", + claude_cua: "anthropic_api", + gemini_cua: "google_api", + eve: "ai_gateway", + pi: "pi_catalog", + fx: "fx_gateway", +}; + +/** Harnesses whose unreported bill is our own provider-API spend, priceable at list. */ +const DIRECT_PROVIDER_HARNESSES: ReadonlySet = new Set([ + "codex", + "claude_cua", + "gemini_cua", + "mastra", + "deepagents", + "eve", + "pi", +]); + +const SUBSCRIPTION_HARNESSES: ReadonlySet = new Set(["cursor", "claude_code"]); + +export interface ResolveBilledCostInput { + harness: string; + model: string | undefined; + usage: NormalizedUsage; + /** Dollars the harness's own channel reported for the run, when any. */ + reportedCostUsd?: number; + priceMap?: PriceMap; +} + +/** + * The cost column: reported dollars win; + * otherwise a direct-provider harness is estimated at catalog price; otherwise the + * cost is unavailable rather than zero. + */ +export function resolveBilledCost({ + harness, + model, + usage, + reportedCostUsd, + priceMap = loadPriceMap(), +}: ResolveBilledCostInput): BilledCost { + if ( + typeof reportedCostUsd === "number" && + Number.isFinite(reportedCostUsd) && + reportedCostUsd >= 0 + ) { + return { + cost_usd: reportedCostUsd, + cost_source: "reported", + billing_channel: REPORTED_CHANNEL[harness] ?? `${harness}_reported`, + }; + } + const provider = providerOf(model); + const channel = + harness === "fx" + ? "fx_gateway" + : SUBSCRIPTION_HARNESSES.has(harness) + ? "subscription" + : provider + ? `${provider}_api` + : "none"; + if (!DIRECT_PROVIDER_HARNESSES.has(harness) || usage.convention === "unreported") { + return { cost_source: "unavailable", billing_channel: channel }; + } + const matched = resolveModelPrice(model, priceMap); + const computed = computeListCost(usage, model, priceMap); + return computed === undefined || !matched + ? { cost_source: "unavailable", billing_channel: channel } + : { + cost_usd: computed, + cost_source: "computed", + billing_channel: channel, + cost_pricing: { + as_of: priceMap.as_of, + model: matched.key, + source: matched.price.source, + }, + }; +} + +/** Provider segment of a configured model id, normalized to the price-map spelling. */ +export function providerOf(model: string | undefined): string | undefined { + if (!model) return undefined; + const [candidate] = modelPriceCandidates(model); + if (!candidate?.includes("/")) return undefined; + const provider = candidate.slice(0, candidate.indexOf("/")); + return provider === "spacexai" || provider === "x-ai" ? "xai" : provider; +} + +/** + * cost = uncached·p_in + cached·p_cached + cache_write·(p_write ?? p_in) + output·p_out, + * plus reasoning at the output rate only when the SDK reports it outside output. + */ +export function computeListCost( + usage: NormalizedUsage, + model: string | undefined, + priceMap: PriceMap = loadPriceMap(), +): number | undefined { + if (usage.convention === "unreported") return undefined; + const price = resolveModelPrice(model, priceMap)?.price; + if (!price) return undefined; + const cacheWriteRate = price.cache_write_input_per_m ?? price.input_per_m!; + const billedOutput = usage.output + (usage.reasoning_in_output ? 0 : usage.reasoning); + const cost = + (usage.input_uncached * price.input_per_m! + + usage.input_cached * price.cached_input_per_m! + + usage.input_cache_write * cacheWriteRate + + billedOutput * price.output_per_m!) / + 1_000_000; + return Number.isFinite(cost) && cost >= 0 ? Number(cost.toFixed(6)) : undefined; +} + +const PRICE_MAP_FILE = "pricing/pricing.json"; +let cachedPriceMap: PriceMap | undefined; + +/** The versioned price map shipped with the package (empty when the file is missing). */ +export function loadPriceMap(filePath?: string): PriceMap { + if (filePath) return readPriceMap(filePath); + cachedPriceMap ??= readPriceMap(path.join(getPackageRootDir(), PRICE_MAP_FILE)); + return cachedPriceMap; +} + +function readPriceMap(filePath: string): PriceMap { + try { + const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")) as Partial; + return { as_of: parsed.as_of ?? "unknown", models: parsed.models ?? {} }; + } catch { + return { as_of: "unknown", models: {} }; + } +} + +/** Find a fully priced entry for the model id, trying each alias in turn. */ +export function resolveModelPrice( + model: string | undefined, + priceMap: PriceMap, +): { key: string; price: ModelPrice } | undefined { + if (!model) return undefined; + const keys = Object.keys(priceMap.models); + for (const candidate of modelPriceCandidates(model)) { + const key = candidate.includes("/") + ? keys.find((entry) => entry === candidate) + : uniqueMatch(keys, (entry) => entry.slice(entry.indexOf("/") + 1) === candidate); + if (!key) continue; + const price = priceMap.models[key]; + return isPriced(price) ? { key, price } : undefined; + } + return undefined; +} + +function isPriced(price: ModelPrice | undefined): price is ModelPrice { + return ( + !!price && + (price.cache_write_input_per_m == null || + (typeof price.cache_write_input_per_m === "number" && + Number.isFinite(price.cache_write_input_per_m) && + price.cache_write_input_per_m >= 0)) && + [price.input_per_m, price.cached_input_per_m, price.output_per_m].every( + (rate) => typeof rate === "number" && Number.isFinite(rate) && rate >= 0, + ) + ); +} + +const HARNESS_DEFAULT_MODELS: Record = { + "codex/default": "openai/gpt-5.4-mini", +}; + +const PROVIDER_ALIASES: Record = { + xai: ["spacexai", "x-ai"], + spacexai: ["xai", "x-ai"], + "x-ai": ["spacexai", "xai"], + zai: ["z-ai"], + "z-ai": ["zai"], + alibaba: ["qwen"], + qwen: ["alibaba"], +}; + +/** + * Alias forms of a configured model id, most specific first: + * `gateway/` prefixes dropped, harness defaults expanded, version dashes as + * dots (`claude-sonnet-4-6` → `claude-sonnet-4.6`), `-preview` and trailing + * date segments stripped, provider spellings swapped, and finally the + * bare model name (matched only when one provider carries it). + */ +export function modelPriceCandidates(model: string): string[] { + let id = model.trim(); + while (id.startsWith("gateway/")) id = id.slice("gateway/".length); + id = HARNESS_DEFAULT_MODELS[id] ?? id; + const slash = id.indexOf("/"); + const provider = slash >= 0 ? id.slice(0, slash) : undefined; + const name = slash >= 0 ? id.slice(slash + 1) : id; + + // Only suffixes that never distinguish one priced model from another are + // stripped; version segments (`-4-6`, `-mini`) stay so a sibling model's + // price is never borrowed. + const names = new Set(); + for (const base of [name, name.replace(/(\d)-(?=\d)/g, "$1.")]) { + names.add(base); + const undated = base.replace(/-\d{8}$/u, "").replace(/-\d{2}-\d{4}$/u, ""); + names.add(undated); + names.add(undated.replace(/-preview$/u, "")); + } + + const providers = provider ? [provider, ...(PROVIDER_ALIASES[provider] ?? [])] : []; + const candidates: string[] = []; + for (const candidateName of names) { + for (const candidateProvider of providers) + candidates.push(`${candidateProvider}/${candidateName}`); + } + for (const candidateName of names) candidates.push(candidateName); + return [...new Set(candidates)]; +} + +function uniqueMatch(items: T[], predicate: (item: T) => boolean): T | undefined { + const matches = items.filter(predicate); + return matches.length === 1 ? matches[0] : undefined; +} diff --git a/packages/evals/framework/evalSystemPrompt.ts b/packages/evals/framework/evalSystemPrompt.ts new file mode 100644 index 000000000..e923e647e --- /dev/null +++ b/packages/evals/framework/evalSystemPrompt.ts @@ -0,0 +1,3 @@ +/** Evaluations have no operator available to answer follow-up questions. */ +export const EVAL_SYSTEM_PROMPT = + "Do not ask for clarification. Make a reasonable assumption and proceed."; diff --git a/packages/evals/framework/harnesses/externalRunner.ts b/packages/evals/framework/harnesses/externalRunner.ts index 82dcfc122..46d0a18cd 100644 --- a/packages/evals/framework/harnesses/externalRunner.ts +++ b/packages/evals/framework/harnesses/externalRunner.ts @@ -1,11 +1,17 @@ import type { ProbeEvidence, TaskSpec, Trajectory } from "stagehand-v3"; +import type { HarnessTrajectory, TerminationReason } from "./trajectoryAdapter.js"; import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; +import type { BrowserSessionLoss } from "../../core/contracts/tool.js"; import type { EvalLogger } from "../../logger.js"; +import { EVAL_SYSTEM_PROMPT } from "../evalSystemPrompt.js"; import { datasetPromptGuidance } from "../externalHarnessPlan.js"; import type { ExternalHarnessTaskPlan } from "../externalHarnessPlan.js"; import type { StepObservation } from "../observationRecorder.js"; import type { TaskResult } from "../types.js"; import { gradeExternalTrajectory, type ExternalHarnessVerifierConfig } from "../verifierAdapter.js"; +import { emitTrajectoryTrace } from "./traceLog.js"; +import { resolveBilledCost, type BilledCost } from "../costEstimate.js"; +import { normalizeUsage, type NormalizedUsage } from "../usageNormalization.js"; export type MetricValue = { count: number; value: number }; @@ -45,10 +51,13 @@ export function parseEvalResult(raw: string): ParsedEvalResult { const markerIndex = markerMatch?.index ?? -1; const resultText = markerIndex >= 0 ? raw.slice(markerIndex + (markerMatch?.[0].length ?? 0)).trim() : raw.trim(); + // Without a marker the report may trail free-form narration ("I'll open + // the site...\n\n{...}"): a report-shaped object that ends the message is + // the agent's conclusion. One quoted mid-prose is not. const candidates = markerIndex >= 0 ? [resultText, resultText.split(/\r?\n/, 1)[0]?.trim(), extractFirstJsonObject(resultText)] - : [resultText, resultText.split(/\r?\n/, 1)[0]?.trim()]; + : [resultText, resultText.split(/\r?\n/, 1)[0]?.trim(), trailingEvalResultJson(resultText)]; for (const candidate of candidates) { if (!candidate) continue; @@ -58,6 +67,30 @@ export function parseEvalResult(raw: string): ParsedEvalResult { return { success: false, raw }; } +/** + * The answer the verifier should grade: the structured report's finalAnswer + * when the agent produced one, otherwise its last message. Remove recognized + * eval reports without discarding legitimate JSON task deliverables. + */ +export function resolveFinalAnswer( + parsed: Pick, + lastMessage: string | undefined, +): string | undefined { + if (parsed.finalAnswer !== undefined) return parsed.finalAnswer; + if (!lastMessage) return undefined; + const stripped = stripEmbeddedEvalReports(lastMessage).trim(); + return stripped || undefined; +} + +/** Remove eval report envelopes, preserving unrelated JSON content. */ +export function stripEmbeddedEvalReports(text: string): string { + let output = text; + for (const span of extractJsonObjects(text)) { + if (isEvalResultJson(span)) output = output.replace(span, ""); + } + return output.replace(/\n{3,}/gu, "\n\n"); +} + export interface ExternalHarnessPromptInput { plan: ExternalHarnessTaskPlan; toolInstructions?: string; @@ -107,6 +140,12 @@ export interface ExternalHarnessUsage { cacheCreationInputTokens?: number; reasoningOutputTokens?: number; totalTokens: number; + /** + * `false` when the SDK exposed no usage at all (cursor; codex after an + * aborted turn with no rollout to recover from). Zeros with `reported: false` + * are treated as unknown, never as a free run. + */ + reported?: boolean; } export interface ExternalHarnessSessionOutcome { @@ -126,6 +165,26 @@ export interface ExternalHarnessToolAdapterLike { captureEvidence?: () => Promise; drainStepObservations?: () => Promise; observedToolMatcher?: (name: string) => boolean; + browserSessionLoss?: () => BrowserSessionLoss | undefined; +} + +/** harnessStopReason recorded when the mounted browser died before the agent finished. */ +export const BROWSER_SESSION_LOST_STOP_REASON = "browser_session_lost"; + +/** + * Collapse a harness's normalized status + stop reason into why the run ended. + * Every SDK reports `completed | max_turns | sdk_error`; the stop reason is the + * only place aborts and browser loss are distinguishable from other errors. + */ +export function deriveTerminationReason( + outcome: Pick, "status" | "stopReason">, +): TerminationReason { + if (outcome.status === "completed") return "completed"; + if (outcome.status === "max_turns") return "step_budget"; + const stopReason = outcome.stopReason ?? ""; + if (stopReason === BROWSER_SESSION_LOST_STOP_REASON) return "browser_session_lost"; + if (/\b(aborted|interrupted)\b/iu.test(stopReason)) return "aborted"; + return "sdk_error"; } export interface ExternalHarnessTrajectoryInput { @@ -140,15 +199,32 @@ export interface ExternalHarnessTrajectoryInput { export interface RunExternalHarnessTaskInput { harness: string; + /** Execution implementation; absence on historical rows means unknown. */ + implementation?: { name: string; version: number; sdkVersion?: string }; plan: ExternalHarnessTaskPlan; + /** Configured model id, used to price the normalized token usage. */ + model?: string; logger: EvalLogger; toolAdapter?: ExternalHarnessToolAdapterLike; verifier?: ExternalHarnessVerifierConfig; resultContract: EvalResultContract; fallbackErrorMessage: string; + /** + * The step (or turn) budget the session was started with, emitted as the + * `step_budget` metric so rows can be grouped by it in Braintrust. + */ + stepBudget?: number; + /** The harness's actual budget counter; different SDKs count different units. */ + stepBudgetUnit?: "tool_calls" | "successful_tool_calls" | "model_steps" | "agent_steps" | "turns"; + configuration?: Record; /** Harness-specific result parser; defaults to the strict parseEvalResult. */ parseResult?: (raw: string) => ParsedEvalResult; - runSession: (prompt: string) => Promise>; + /** Use a native system/developer channel when the harness exposes one. */ + systemPromptMode?: "native" | "task_prefix"; + runSession: ( + prompt: string, + systemPrompt: string, + ) => Promise>; toTrajectory: (input: ExternalHarnessTrajectoryInput, taskSpec: TaskSpec) => Trajectory; } @@ -159,13 +235,19 @@ export interface RunExternalHarnessTaskInput { */ export async function runExternalHarnessTask({ harness, + implementation, plan, + model, logger, toolAdapter, verifier, resultContract, fallbackErrorMessage, + stepBudget, + stepBudgetUnit, + configuration, parseResult, + systemPromptMode = "task_prefix", runSession, toTrajectory, }: RunExternalHarnessTaskInput): Promise { @@ -174,20 +256,55 @@ export async function runExternalHarnessTask({ toolInstructions: toolAdapter?.promptInstructions, resultContract, }); - const outcome = await runSession(prompt); + const harnessConfiguration = { + ...configuration, + evalPolicyVersion: 1, + systemPromptMode, + ...(stepBudget !== undefined && { stepBudget }), + ...(stepBudgetUnit && { stepBudgetUnit }), + }; + const startedAt = performance.now(); + // CLI-only harnesses without a system-instruction channel receive the same + // eval policy as a task prefix. Native channels must not also get the prefix. + const sessionOutcome = await runSession( + systemPromptMode === "native" ? prompt : `${EVAL_SYSTEM_PROMPT}\n\n${prompt}`, + systemPromptMode === "native" ? EVAL_SYSTEM_PROMPT : "", + ); + const agentWallMs = performance.now() - startedAt; + // A run that outlived its browser has no trustworthy self-report: the agent + // was answering terminal "Browser session lost" errors, not the task. + const browserSessionLoss = toolAdapter?.browserSessionLoss?.(); + if (browserSessionLoss) { + logger.warn({ + category: "stagehand_facade", + level: 1, + message: `browser session lost before the agent finished: ${browserSessionLoss.cause}`, + }); + } + const outcome: ExternalHarnessSessionOutcome = browserSessionLoss + ? { + ...sessionOutcome, + status: "sdk_error", + stopReason: BROWSER_SESSION_LOST_STOP_REASON, + iterationError: `Browser session lost (${browserSessionLoss.cause})`, + } + : sessionOutcome; const iterationErrorMessage = stringifyError(outcome.iterationError); const rawResult = [outcome.resultText, outcome.transcriptText, iterationErrorMessage] .filter(Boolean) .join("\n\n"); const trustedResultText = outcome.resultText.trim(); const parsed = { ...(parseResult ?? parseEvalResult)(trustedResultText), raw: rawResult }; + parsed.finalAnswer = resolveFinalAnswer(parsed, trustedResultText); const sanitizedStopReason = outcome.stopReason ? sanitizeErrorMessage(outcome.stopReason) : undefined; const sanitizedIterationError = iterationErrorMessage ? sanitizeErrorMessage(iterationErrorMessage) : undefined; - const sdkErrorMessage = sanitizedStopReason ?? sanitizedIterationError; + const sdkErrorMessage = browserSessionLoss + ? sanitizedIterationError + : (sanitizedStopReason ?? sanitizedIterationError); const errorMessage = sanitizeErrorMessage( outcome.status === "sdk_error" ? (sdkErrorMessage ?? fallbackErrorMessage) @@ -199,6 +316,22 @@ export async function runExternalHarnessTask({ fallbackErrorMessage, ); const prefix = legacyHarnessFieldPrefix(harness); + const terminationReason = deriveTerminationReason(outcome); + const usage = normalizeUsage({ harness, raw: outcome.usage }); + const cost = resolveBilledCost({ harness, model, usage, reportedCostUsd: outcome.costUsd }); + if (cost.cost_source === "unavailable" && usage.convention !== "unreported") { + logger.log({ + category: "cost", + level: 1, + message: `cost unavailable for ${harness} on ${model ?? "(unknown model)"} (channel ${cost.billing_channel}); cost_usd omitted`, + }); + } + const baseMetrics: Record = { + ...buildNormalizedHarnessMetrics(outcome), + ...buildUsageCostMetrics(usage, cost), + ...(stepBudget !== undefined && { step_budget: metricValue(stepBudget) }), + agent_wall_ms: metricValue(agentWallMs), + }; const baseResult: TaskResult = { _success: outcome.status === "sdk_error" ? false : parsed.success, error: outcome.status === "sdk_error" || !parsed.success ? errorMessage : undefined, @@ -206,48 +339,193 @@ export async function runExternalHarnessTask({ finalAnswer: parsed.finalAnswer, rawResult: parsed.raw, harnessStatus: outcome.status, + harnessConfiguration, + usageConvention: usage.convention, + ...(implementation && { harnessImplementation: implementation }), ...(sanitizedStopReason && { harnessStopReason: sanitizedStopReason }), + terminationReason, + agent_wall_ms: Math.round(agentWallMs), + cost_source: cost.cost_source, + billing_channel: cost.billing_channel, + ...(cost.cost_pricing && { cost_pricing: cost.cost_pricing }), + ...(cost.cost_usd !== undefined && { cost_usd: cost.cost_usd }), // Deprecated compatibility aliases; consumers should use the normalized // harnessStatus / harnessStopReason fields for newly registered harnesses. [`${prefix}Status`]: outcome.status, ...(sanitizedStopReason && { [`${prefix}StopReason`]: sanitizedStopReason }), logs: logger.getLogs(), - metrics: buildNormalizedHarnessMetrics(outcome), + metrics: baseMetrics, }; - if (!verifier) return baseResult; + if (!verifier) { + return { ...baseResult, metrics: { ...baseMetrics, total_wall_ms: metricValue(agentWallMs) } }; + } + const isFacadeTool = toolAdapter?.observedToolMatcher; const evidenceTimeoutMs = readPositiveIntEnv("EVAL_CAPTURE_EVIDENCE_TIMEOUT_MS", 15_000); + const evidenceStartedAt = performance.now(); const finalObservation = toolAdapter?.captureEvidence ? await bestEffort(toolAdapter.captureEvidence(), evidenceTimeoutMs) : undefined; const stepObservations = toolAdapter?.drainStepObservations ? await bestEffort(toolAdapter.drainStepObservations(), evidenceTimeoutMs) : undefined; + const evidenceMs = performance.now() - evidenceStartedAt; + let trajectory: HarnessTrajectory | undefined; + const verifierStartedAt = performance.now(); const gradedResult = await gradeExternalTrajectory({ - buildTrajectory: () => - toTrajectory( - { - raw: outcome.raw, - parsed, - outcome, - ...(finalObservation && { finalObservation }), - ...(stepObservations?.length && { stepObservations }), - ...(toolAdapter?.observedToolMatcher && { - observedToolName: toolAdapter.observedToolMatcher, - }), - status: outcome.status === "completed" ? "complete" : "error", - }, - verifier.taskSpec, - ), + buildTrajectory: () => { + trajectory = withTerminationReason( + toTrajectory( + { + raw: outcome.raw, + parsed, + outcome, + ...(finalObservation && { finalObservation }), + ...(stepObservations?.length && { stepObservations }), + ...(toolAdapter?.observedToolMatcher && { + observedToolName: toolAdapter.observedToolMatcher, + }), + status: outcome.status === "completed" ? "complete" : "error", + }, + verifier.taskSpec, + ), + terminationReason, + ); + if (implementation) trajectory.harnessImplementation = implementation; + trajectory.harnessConfiguration = harnessConfiguration; + if (cost.cost_pricing) trajectory.cost_pricing = cost.cost_pricing; + // The readable step trace is derived from the normalized trajectory so + // every harness logs the same shape; a formatting bug must never fail + // the grade. + try { + emitTrajectoryTrace(logger, { + trajectory, + outcome: { ...outcome, stopReason: sanitizedStopReason }, + usage, + agentWallMs, + isFacadeTool, + report: { + summary: parsed.summary, + finalAnswer: parsed.finalAnswer, + success: parsed.success, + }, + }); + } catch (traceError) { + logger.warn({ + category: "trace", + level: 1, + message: `step trace failed: ${stringifyError(traceError)}`, + }); + } + return trajectory; + }, verifier, baseResult, errorMessage, category: harness, logger, + isFacadeTool, + }); + const verifierWallMs = performance.now() - verifierStartedAt; + const timing = buildTimingMetrics({ agentWallMs, evidenceMs, verifierWallMs }); + logger.log({ + category: "trace", + level: 1, + message: [ + "timing", + `agent=${formatSeconds(agentWallMs)}`, + `evidence=${formatSeconds(evidenceMs)}`, + `verifier=${formatSeconds(verifierWallMs)}`, + `total=${formatSeconds(timing.total_wall_ms.value)}`, + ...(cost.cost_usd !== undefined ? [`cost=$${cost.cost_usd} (${cost.cost_source})`] : []), + ].join(" · "), }); - return outcome.status === "sdk_error" - ? { ...gradedResult, _success: false, error: errorMessage } - : gradedResult; + const facadeMetrics = + trajectory && isFacadeTool ? buildFacadeToolCallMetrics(trajectory, isFacadeTool) : {}; + const gradedMetrics = (gradedResult.metrics ?? {}) as Record; + const result: TaskResult = { + ...gradedResult, + metrics: { ...gradedMetrics, ...facadeMetrics, ...timing }, + // Re-read so the trace and verifier lines logged after baseResult was + // built ship with the row. + logs: logger.getLogs(), + }; + // Execution failures remain visible in harnessStatus/terminationReason. The + // verifier can still establish a completed task from the captured evidence. + return result; +} + +function withTerminationReason( + trajectory: Trajectory, + terminationReason: TerminationReason, +): HarnessTrajectory { + return { ...trajectory, terminationReason }; +} + +/** + * How often the agent actually reached the mounted browser surface. A run that + * "passes" with zero facade calls answered from somewhere else (curl, another + * MCP server, prior knowledge), which the rubric verifier cannot see. + * + * Failed calls remain failures even when their text claims session loss. The + * run-level loss signal is runner-owned; normalized steps do not currently + * carry trusted per-call loss attribution. + */ +export function buildFacadeToolCallMetrics( + trajectory: Pick, + isFacadeTool: (name: string) => boolean, +): Record { + let calls = 0; + let failures = 0; + for (const step of trajectory.steps) { + if (!isFacadeTool(step.actionName)) continue; + calls += 1; + if (step.toolOutput?.ok !== false) continue; + failures += 1; + } + return { + facade_tool_calls: metricValue(calls), + facade_tool_call_failures: metricValue(failures), + }; +} + +/** Wall-clock split so agent speed is never confounded with verifier speed. */ +export function buildTimingMetrics(timing: { + agentWallMs: number; + evidenceMs: number; + verifierWallMs: number; +}): Record<"agent_wall_ms" | "evidence_ms" | "verifier_wall_ms" | "total_wall_ms", MetricValue> { + return { + agent_wall_ms: metricValue(timing.agentWallMs), + evidence_ms: metricValue(timing.evidenceMs), + verifier_wall_ms: metricValue(timing.verifierWallMs), + total_wall_ms: metricValue(timing.agentWallMs + timing.evidenceMs + timing.verifierWallMs), + }; +} + +export function formatSeconds(ms: number): string { + return `${(ms / 1000).toFixed(1)}s`; +} + +/** + * Convention-independent token buckets (the efficiency axis) plus the single + * billed cost column `cost_usd` — reported by the harness's channel, else + * computed at provider list price for direct-API harnesses, else absent + * rather than zero. Unreported usage gets no usage_* metrics rather than zeros. + */ +export function buildUsageCostMetrics( + usage: NormalizedUsage, + cost: BilledCost, +): Record { + return { + ...(usage.convention !== "unreported" && { + usage_input_total: metricValue(usage.input_total), + usage_input_cached: metricValue(usage.input_cached), + usage_output: metricValue(usage.output), + usage_reasoning: metricValue(usage.reasoning), + }), + ...(cost.cost_usd !== undefined && { cost_usd: metricValue(cost.cost_usd) }), + }; } /** Convert a registered harness id to its deprecated TaskResult field prefix. */ @@ -290,13 +568,25 @@ function toFiniteNumber(value: unknown): number { } function extractFirstJsonObject(value: string): string | undefined { - const start = value.indexOf("{"); - if (start < 0) return undefined; + return extractJsonObjects(value)[0]; +} + +/** Every top-level balanced `{...}` span in document order (not validated). */ +function extractJsonObjects(value: string): string[] { + const spans: string[] = []; + let start = -1; let depth = 0; let inString = false; let escaped = false; - for (let index = start; index < value.length; index += 1) { + for (let index = 0; index < value.length; index += 1) { const character = value[index]; + if (start < 0) { + if (character === "{") { + start = index; + depth = 1; + } + continue; + } if (inString) { if (escaped) escaped = false; else if (character === "\\") escaped = true; @@ -307,10 +597,34 @@ function extractFirstJsonObject(value: string): string | undefined { else if (character === "{") depth += 1; else if (character === "}") { depth -= 1; - if (depth === 0) return value.slice(start, index + 1); + if (depth === 0) { + spans.push(value.slice(start, index + 1)); + start = -1; + } } } - return undefined; + return spans; +} + +function trailingEvalResultJson(text: string): string | undefined { + const last = extractJsonObjects(text).at(-1); + return last && text.endsWith(last) && isEvalResultJson(last) ? last : undefined; +} + +function isEvalResultJson(candidate: string): boolean { + try { + const parsed: unknown = JSON.parse(candidate); + return ( + typeof parsed === "object" && + parsed !== null && + typeof (parsed as { success?: unknown }).success === "boolean" && + (typeof (parsed as { summary?: unknown }).summary === "string" || + typeof (parsed as { finalAnswer?: unknown }).finalAnswer === "string") && + Object.keys(parsed).every((key) => ["success", "summary", "finalAnswer"].includes(key)) + ); + } catch { + return false; + } } function tryParseEvalJson(candidate: string): Omit | undefined { diff --git a/packages/evals/framework/harnesses/persistTrajectory.ts b/packages/evals/framework/harnesses/persistTrajectory.ts index 0eb9353ab..ff637f6ba 100644 --- a/packages/evals/framework/harnesses/persistTrajectory.ts +++ b/packages/evals/framework/harnesses/persistTrajectory.ts @@ -7,13 +7,15 @@ import { resolveTrajectoryRoot, writeTrajectoryMetadata, } from "../trajectoryGroup.js"; -import type { EvaluationResult, TaskSpec, Trajectory } from "stagehand-v3"; +import type { EvaluationResult, TaskSpec } from "stagehand-v3"; +import type { HarnessTrajectory } from "./trajectoryAdapter.js"; +import type { UngradedVerifierResult } from "../verifierAdapter.js"; export interface PersistAdapterTrajectoryOptions { - trajectory: Trajectory; + trajectory: HarnessTrajectory; taskSpec: TaskSpec; - /** EvaluationResult from V3Evaluator.verify(). Written to scores/result.json. */ - evaluationResult?: EvaluationResult; + /** Accepted grade or explicit ungraded audit record. Written to scores/result.json. */ + evaluationResult?: EvaluationResult | UngradedVerifierResult; /** * Output directory root. Final layout lives at * `////`. Entrypoints normally generate @@ -73,6 +75,16 @@ export async function persistAdapterTrajectory( runDir: path.basename(directory), attempt, status: opts.trajectory.status, + ...(opts.trajectory.cost_pricing && { cost_pricing: opts.trajectory.cost_pricing }), + ...(opts.trajectory.terminationReason && { + terminationReason: opts.trajectory.terminationReason, + }), + ...(opts.trajectory.harnessImplementation && { + harnessImplementation: opts.trajectory.harnessImplementation, + }), + ...(opts.trajectory.harnessConfiguration && { + harnessConfiguration: opts.trajectory.harnessConfiguration, + }), }); if (opts.evaluationResult) { @@ -86,6 +98,7 @@ export async function persistAdapterTrajectory( { task: opts.trajectory.task, status: opts.trajectory.status, + ...(opts.trajectory.cost_pricing && { cost_pricing: opts.trajectory.cost_pricing }), finalAnswer: opts.trajectory.finalAnswer ?? null, result: opts.evaluationResult, }, diff --git a/packages/evals/framework/harnesses/piAdapter.ts b/packages/evals/framework/harnesses/piAdapter.ts index 3d828dc25..dc10bdabe 100644 --- a/packages/evals/framework/harnesses/piAdapter.ts +++ b/packages/evals/framework/harnesses/piAdapter.ts @@ -133,12 +133,14 @@ function normalizeResult(value: unknown): { for (const block of value.content) { if (!isRecord(block)) continue; if (block.type === "text" && typeof block.text === "string") text.push(block.text); - if ( - block.type === "image" && - typeof block.data === "string" && - typeof block.mimeType === "string" - ) { - images.push({ bytes: Buffer.from(block.data, "base64"), mediaType: block.mimeType }); + if (block.type === "image" && typeof block.mimeType === "string") { + // The pi session decodes screenshots to a single Buffer when it retains + // the event; raw base64 only arrives from callers that bypass it. + if (Buffer.isBuffer(block.bytes)) { + images.push({ bytes: block.bytes, mediaType: block.mimeType }); + } else if (typeof block.data === "string") { + images.push({ bytes: Buffer.from(block.data, "base64"), mediaType: block.mimeType }); + } } } } diff --git a/packages/evals/framework/harnesses/toolSurfaceResolution.ts b/packages/evals/framework/harnesses/toolSurfaceResolution.ts index 977b12601..47cb31c14 100644 --- a/packages/evals/framework/harnesses/toolSurfaceResolution.ts +++ b/packages/evals/framework/harnesses/toolSurfaceResolution.ts @@ -11,6 +11,8 @@ function formatList(values: ToolSurface[]): string { return `${values.slice(0, -1).join(", ")}, or ${values.at(-1)}`; } +const DEFAULT_TOOL_SURFACE_PREFERENCE: ToolSurface[] = ["stagehand_facade"]; + /** Resolve the tool surface for a row on `harness`. */ export function resolveToolSurface( harness: Pick, @@ -18,7 +20,14 @@ export function resolveToolSurface( ): ToolSurface | undefined { const supported = harness.supportedToolSurfaces; if (supported.length === 0) return requested; - if (requested === undefined) return supported[0]; + // Default to the facade wherever the harness supports it: it is the surface + // every benchmark comparison is run on, and the list's first entry + // (browse_cli / playwright_code) has silently produced invalid runs when + // --tool was omitted. + if (requested === undefined) + return ( + DEFAULT_TOOL_SURFACE_PREFERENCE.find((surface) => supported.includes(surface)) ?? supported[0] + ); if (supported.includes(requested)) return requested; throw new EvalsError( `Harness "${harness.harness}" supports --tool ${formatList(supported)}; received "${requested}".`, diff --git a/packages/evals/framework/harnesses/traceLog.ts b/packages/evals/framework/harnesses/traceLog.ts new file mode 100644 index 000000000..3d5d30940 --- /dev/null +++ b/packages/evals/framework/harnesses/traceLog.ts @@ -0,0 +1,294 @@ +import type { LogLine, Trajectory, TrajectoryStep } from "stagehand-v3"; +import type { ExternalHarnessSessionOutcome } from "./externalRunner.js"; +import { formatNormalizedUsage, type NormalizedUsage } from "../usageNormalization.js"; + +/** + * Readable per-step trace shared by every external harness. It is derived + * from the normalized Trajectory, so claude_code, codex, mastra, pi, eve, + * deepagents, fx and cursor all log the exact same shape: + * + * step 3 · think · + * step 3 · run · ok · await page.goto('https://…'); return page.title() → "Recreation.gov…" + * step 4 · screenshot · ok → [image 42 KB] + * step 5 · run · ERR · await page.click('#nope') → Timeout 30000ms exceeded + * summary · + * answer · (or "answer · (none — max_turns)") + * result · completed · steps=5 · facade_calls=4 · in=12345 (cached 9000) out=678 · agent=42.0s + * + * The verifier runs after this trace, so its wall-clock lands on a separate + * `timing · agent=… evidence=… verifier=… total=…` line once grading is done. + * + * Full code and results travel in `auxiliary` so Braintrust / parseLogLine + * keep the detail expandable without polluting the message text. + */ + +export const TRACE_LOG_CATEGORY = "trace"; +export const TRACE_CLIP_CHARS = 200; +/** Cap on full-detail auxiliary payloads so a 300 KB snapshot cannot bloat a row. */ +export const TRACE_AUXILIARY_MAX_CHARS = 16_000; + +const SEPARATOR = " · "; +const ARROW = " → "; + +export interface TraceLogSink { + log(line: LogLine): void; +} + +export interface TrajectoryTraceInput { + trajectory: Pick; + outcome: Pick, "status" | "stopReason" | "usage">; + /** Convention-normalized token buckets; falls back to the raw SDK usage when absent. */ + usage?: NormalizedUsage; + /** Wall-clock of the agent session alone (ms), excluding evidence capture and grading. */ + agentWallMs?: number; + /** Optional per-step wall-clock durations (ms), indexed like `trajectory.steps`. */ + stepDurationsMs?: ReadonlyArray; + /** Which action names count as calls into the mounted browser surface. */ + isFacadeTool?: (name: string) => boolean; + /** The agent's parsed self-report (EVAL_RESULT), when available. */ + report?: { summary?: string; finalAnswer?: string; success?: boolean }; +} + +/** Build every trace line for a completed run, in emission order. */ +export function buildTrajectoryTraceLines(input: TrajectoryTraceInput): LogLine[] { + const lines: LogLine[] = []; + input.trajectory.steps.forEach((step, index) => { + lines.push(...buildStepTraceLines(step, index + 1, input.stepDurationsMs?.[index])); + }); + lines.push(...buildAnswerTraceLines(input)); + lines.push(buildResultTraceLine(input)); + return lines; +} + +/** + * What the agent said it did and concluded. A missing answer is stated + * explicitly (with the stop status) — silence here hides budget/error stops + * where the agent never produced a final message. + */ +export function buildAnswerTraceLines(input: TrajectoryTraceInput): LogLine[] { + const report = input.report ?? {}; + const lines: LogLine[] = []; + const summary = report.summary?.trim(); + if (summary) { + lines.push({ + category: TRACE_LOG_CATEGORY, + level: 1, + message: ["summary", clip(singleLine(summary))].join(SEPARATOR), + auxiliary: { summary: { value: capped(summary), type: "string" } }, + }); + } + const answer = report.finalAnswer?.trim(); + if (answer) { + lines.push({ + category: TRACE_LOG_CATEGORY, + level: 1, + message: ["answer", clip(singleLine(answer))].join(SEPARATOR), + auxiliary: { answer: { value: capped(answer), type: "string" } }, + }); + } else { + const why = input.outcome.status === "completed" ? "agent reported none" : input.outcome.status; + lines.push({ + category: TRACE_LOG_CATEGORY, + level: input.outcome.status === "completed" ? 1 : 0, + message: ["answer", `(none — ${why})`].join(SEPARATOR), + }); + } + return lines; +} + +/** Emit the trace through an EvalLogger-compatible sink. */ +export function emitTrajectoryTrace(sink: TraceLogSink, input: TrajectoryTraceInput): void { + for (const line of buildTrajectoryTraceLines(input)) sink.log(line); +} + +export function buildStepTraceLines( + step: TrajectoryStep, + ordinal: number, + durationMs?: number, +): LogLine[] { + const lines: LogLine[] = []; + const reasoning = singleLine(step.reasoning); + if (reasoning) { + lines.push({ + category: TRACE_LOG_CATEGORY, + level: 1, + message: ["step " + ordinal, "think", clip(reasoning)].join(SEPARATOR), + auxiliary: { reasoning: { value: capped(step.reasoning), type: "string" } }, + }); + } + + const tool = shortToolName(step.actionName); + const ok = step.toolOutput?.ok !== false; + const code = describeArgs(tool, step.actionArgs); + const result = ok ? describeResult(tool, step) : describeError(step); + const head = [ + "step " + ordinal, + tool, + ok ? "ok" : "ERR", + ...(durationMs !== undefined ? [formatDuration(durationMs)] : []), + ...(code ? [clip(code)] : []), + ].join(SEPARATOR); + + lines.push({ + category: TRACE_LOG_CATEGORY, + level: ok ? 1 : 0, + message: result ? head + ARROW + clip(result) : head, + auxiliary: buildStepAuxiliary(step, code), + }); + return lines; +} + +export function buildResultTraceLine(input: TrajectoryTraceInput): LogLine { + const { outcome, trajectory } = input; + const facadeCalls = input.isFacadeTool + ? trajectory.steps.filter((step) => input.isFacadeTool!(step.actionName)).length + : undefined; + const raw = outcome.usage; + const tokens = input.usage + ? formatNormalizedUsage(input.usage) + : [ + `in=${raw.inputTokens}`, + `out=${raw.outputTokens}`, + ...(raw.cachedInputTokens !== undefined ? [`cached=${raw.cachedInputTokens}`] : []), + ].join(" "); + const message = [ + "result", + outcome.status, + ...(outcome.stopReason ? [singleLine(outcome.stopReason)] : []), + `steps=${trajectory.steps.length}`, + ...(facadeCalls !== undefined ? [`facade_calls=${facadeCalls}`] : []), + tokens, + ...(input.agentWallMs !== undefined ? [`agent=${(input.agentWallMs / 1000).toFixed(1)}s`] : []), + ].join(SEPARATOR); + return { + category: TRACE_LOG_CATEGORY, + level: 1, + message: clip(message, 400), + auxiliary: { + usage: { value: JSON.stringify(raw), type: "object" }, + ...(input.usage && { + normalized_usage: { value: JSON.stringify(input.usage), type: "object" }, + }), + }, + }; +} + +/** + * Collapse harness-specific tool naming onto the surface's own tool name: + * `mcp__stagehand__run`, `stagehand.run`, `stagehand_run` all become `run`. + * Names that carry no server prefix (Bash, node, web_search) pass through. + */ +export function shortToolName(actionName: string): string { + let name = actionName.trim() || "tool"; + if (name.startsWith("mcp__")) { + const separator = name.lastIndexOf("__"); + if (separator > 4) name = name.slice(separator + 2); + } else if (name.includes(".")) { + name = name.slice(name.lastIndexOf(".") + 1); + } + if (/^(?:mcp_)?stagehand_(?:browser_)?(run|snapshot|screenshot)$/u.test(name)) { + name = name.replace(/^(?:mcp_)?stagehand_(?:browser_)?/u, ""); + } + return name || "tool"; +} + +function describeArgs(tool: string, args: Record): string { + if (!args || Object.keys(args).length === 0) return ""; + if (tool === "run") { + if (typeof args.code === "string") return singleLine(args.code); + if (args.actions !== undefined) return "actions " + (safeJson(args.actions) ?? ""); + } + if (typeof args.command === "string") return singleLine(args.command); + return safeJson(args) ?? ""; +} + +function describeResult(tool: string, step: TrajectoryStep): string { + const imageBytes = imageEvidenceBytes(step); + if (imageBytes > 0) return `[image ${formatKb(imageBytes)}]`; + const result = step.toolOutput?.result; + if (result === undefined || result === null) return ""; + if (Buffer.isBuffer(result)) return `[image ${formatKb(result.length)}]`; + if (typeof result === "string") { + if (tool === "screenshot") return "[image]"; + if (tool === "snapshot") return describeSnapshot(result); + return singleLine(result); + } + return safeJson(result) ?? String(result); +} + +function describeError(step: TrajectoryStep): string { + const error = step.toolOutput?.error; + if (error) return singleLine(error); + const result = step.toolOutput?.result; + if (typeof result === "string" && result) return singleLine(result); + return safeJson(result) ?? "error"; +} + +function describeSnapshot(text: string): string { + const lines = text.split(/\r?\n/u); + const nodes = lines.filter((line) => /^\s*\[\d+(?:-\d+)?\]/u.test(line)).length; + const first = lines.find((line) => line.trim())?.trim() ?? ""; + return nodes > 0 ? `[snapshot ${nodes} nodes] ${first}`.trim() : singleLine(text); +} + +function imageEvidenceBytes(step: TrajectoryStep): number { + let total = 0; + for (const modality of step.agentEvidence?.modalities ?? []) { + if (modality.type === "image") total += modality.bytes.length; + } + return total; +} + +function buildStepAuxiliary(step: TrajectoryStep, code: string): LogLine["auxiliary"] { + const auxiliary: NonNullable = { + tool: { value: step.actionName, type: "string" }, + }; + if (code) auxiliary.code = { value: capped(code), type: "string" }; + const result = step.toolOutput?.result; + if (result !== undefined && result !== null && !Buffer.isBuffer(result)) { + auxiliary.result = + typeof result === "string" + ? { value: capped(result), type: "string" } + : { value: capped(safeJson(result) ?? String(result)), type: "object" }; + } + if (step.toolOutput?.error) { + auxiliary.error = { value: capped(step.toolOutput.error), type: "string" }; + } + return auxiliary; +} + +function formatDuration(ms: number): string { + return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms)}ms`; +} + +function formatKb(bytes: number): string { + return `${Math.max(1, Math.round(bytes / 1024))} KB`; +} + +function singleLine(value: string | undefined): string { + return (value ?? "").replace(/\s+/gu, " ").trim(); +} + +function clip(value: string, max = TRACE_CLIP_CHARS): string { + return value.length > max ? value.slice(0, max - 1) + "…" : value; +} + +/** + * Capped auxiliary payload. Objects are capped as JSON text, which can leave a + * truncated string in a type:"object" entry — parseLogLine falls back to the + * raw string in that case. + */ +function capped(value: string): string { + return value.length > TRACE_AUXILIARY_MAX_CHARS + ? value.slice(0, TRACE_AUXILIARY_MAX_CHARS) + + `…[truncated ${value.length - TRACE_AUXILIARY_MAX_CHARS} chars]` + : value; +} + +function safeJson(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} diff --git a/packages/evals/framework/harnesses/trajectoryAdapter.ts b/packages/evals/framework/harnesses/trajectoryAdapter.ts index 2d43cf6d5..4dd86f2ba 100644 --- a/packages/evals/framework/harnesses/trajectoryAdapter.ts +++ b/packages/evals/framework/harnesses/trajectoryAdapter.ts @@ -7,6 +7,26 @@ import type { TrajectoryStep, } from "stagehand-v3"; +/** + * Why a run stopped. `status` on the Trajectory only says whether it ended + * cleanly; this says what ended it, so a budget-exhausted run can be told + * apart from a crash offline. + */ +export type TerminationReason = + | "completed" + | "step_budget" + | "browser_session_lost" + | "sdk_error" + | "aborted"; + +/** A Trajectory as external harnesses persist it. */ +export type HarnessTrajectory = Trajectory & { + terminationReason?: TerminationReason; + harnessImplementation?: { name: string; version: number; sdkVersion?: string }; + harnessConfiguration?: Record; + cost_pricing?: { as_of: string; model: string; source: string }; +}; + /** * Pure converter from a harness-specific result to a verifier Trajectory. * Implementations must be deterministic (no I/O, no mutation of input). diff --git a/packages/evals/framework/reasoningSummary.ts b/packages/evals/framework/reasoningSummary.ts new file mode 100644 index 000000000..e4f8c27c2 --- /dev/null +++ b/packages/evals/framework/reasoningSummary.ts @@ -0,0 +1,44 @@ +/** + * OpenAI Responses-style models only return reasoning text when a summary is + * requested explicitly (`reasoning: { summary }`); without it every harness + * reports reasoning tokens but an empty step.reasoning. The request is on by + * default and shared across harnesses so a run can be compared step for step. + */ +export const REASONING_SUMMARY_ENV = "EVAL_REASONING_SUMMARY"; + +export type ReasoningSummaryMode = "auto" | "concise" | "detailed"; + +export const DEFAULT_REASONING_SUMMARY: ReasoningSummaryMode = "detailed"; + +const MODES = new Set(["auto", "concise", "detailed"]); + +/** Requested summary mode, or undefined when `EVAL_REASONING_SUMMARY=off`. */ +export function readReasoningSummary( + env: NodeJS.ProcessEnv = process.env, +): ReasoningSummaryMode | undefined { + const raw = env[REASONING_SUMMARY_ENV]?.trim().toLowerCase(); + if (!raw) return DEFAULT_REASONING_SUMMARY; + if (raw === "off" || raw === "none" || raw === "false" || raw === "0") return undefined; + return MODES.has(raw) ? (raw as ReasoningSummaryMode) : DEFAULT_REASONING_SUMMARY; +} + +export function isOpenAiModel(model: string): boolean { + return !model.includes("/") || model.startsWith("openai/"); +} + +/** + * AI SDK `providerOptions` that make the OpenAI provider request reasoning + * summaries. Empty for other providers, whose reasoning text (Anthropic + * thinking) streams without being asked. + */ +export function openAiReasoningProviderOptions( + model: string, + env: NodeJS.ProcessEnv = process.env, +): Record> | undefined { + const summary = readReasoningSummary(env); + const name = model.replace(/^openai\//u, ""); + const supportsReasoning = + /^(?:gpt-[56]|o[134])(?:[.-]|$)/u.test(name) && !name.startsWith("gpt-5-chat"); + if (!summary || !isOpenAiModel(model) || !supportsReasoning) return undefined; + return { openai: { reasoningSummary: summary } }; +} diff --git a/packages/evals/framework/runner.ts b/packages/evals/framework/runner.ts index 6f74b1da9..9aadf6fad 100644 --- a/packages/evals/framework/runner.ts +++ b/packages/evals/framework/runner.ts @@ -287,6 +287,49 @@ export interface RunEvalsResult { }>; } +export type SummaryResult = RunEvalsResult["results"][number] & { categories?: string[] }; + +/** + * Normalize one Braintrust Eval result row for the run summary. Braintrust + * leaves `output` undefined when the task function threw (or its span failed + * after the task returned) and reports the failure in `error`; that row must + * still count as a failure so the summary, experiment link, and per-model + * table are written for the rest of the run. + */ +export function toSummaryResult(result: { + input: EvalInput; + output?: unknown; + error?: unknown; + metadata?: Record; +}): SummaryResult { + const output: SummaryResult["output"] = + typeof result.output === "boolean" + ? { _success: result.output } + : result.output !== null && typeof result.output === "object" + ? (result.output as SummaryResult["output"]) + : { + _success: false, + error: + formatProgressError(result.error) ?? + (result.output === undefined + ? "Braintrust reported no output for this task" + : String(result.output)), + }; + const categories = Array.isArray(result.metadata?.categories) + ? result.metadata.categories.filter( + (category): category is string => typeof category === "string", + ) + : undefined; + + return { + input: result.input, + output, + name: result.input.name, + score: output._success ? 1 : 0, + ...(categories && { categories }), + }; +} + function formatProgressError(error: unknown): string | undefined { if (error === undefined || error === null) return undefined; if (typeof error === "string") return error; @@ -299,6 +342,57 @@ function formatProgressError(error: unknown): string | undefined { } } +/** + * Experiment-level metadata for Braintrust/LangSmith filtering. Tool surface + * and model are always present (derived from the planned rows when not set + * globally) so cells can be grouped without opening a row; a single value + * is emitted as a scalar, several as a list. + */ +export function buildExperimentMetadata(input: { + environment: "LOCAL" | "BROWSERBASE"; + tier: "core" | "bench"; + coreToolSurface?: string; + coreStartupProfile?: string; + harness?: Harness; + modelOverride?: string; + useApi?: boolean; + testcases: Testcase[]; +}): Record { + const distinct = (pick: (tc: Testcase) => unknown): string[] => { + const values = new Set(); + for (const tc of input.testcases) { + const value = pick(tc); + if (typeof value === "string" && value) values.add(value); + } + return [...values].sort(); + }; + const scalarOrList = (values: string[]): string | string[] | undefined => + values.length === 0 ? undefined : values.length === 1 ? values[0] : values; + + const toolSurface = + input.coreToolSurface ?? scalarOrList(distinct((tc) => tc.metadata?.toolSurface)); + const startupProfile = + input.coreStartupProfile ?? scalarOrList(distinct((tc) => tc.metadata?.startupProfile)); + const model = + input.modelOverride ?? + scalarOrList(distinct((tc) => tc.metadata?.model).filter((m) => m !== "none")); + const provider = scalarOrList(distinct((tc) => tc.metadata?.provider)); + const dataset = scalarOrList(distinct((tc) => tc.metadata?.dataset)); + + return { + environment: input.environment, + tier: input.tier, + ...(toolSurface && { tool_surface: toolSurface, toolSurface }), + ...(startupProfile && { startup_profile: startupProfile, startupProfile }), + ...(input.harness && { harness: input.harness }), + ...(model && { model }), + ...(provider && { provider }), + ...(dataset && { dataset }), + task_count: input.testcases.length, + ...(input.useApi && { api: true }), + }; +} + const MAX_SPAN_PAYLOAD_BYTES = 2_000_000; export function capForSpan(value: Record): Record { @@ -423,19 +517,16 @@ export async function runEvals(options: RunEvalsOptions): Promise testcases, task: async (input: EvalInput): Promise => { // Cooperative abort: skip any testcase that hasn't started yet @@ -482,6 +573,7 @@ export async function runEvals(options: RunEvalsOptions): Promise { - const output = - typeof result.output === "boolean" ? { _success: result.output } : result.output; - const categories = Array.isArray(result.metadata?.categories) - ? result.metadata.categories.filter( - (category): category is string => typeof category === "string", - ) - : undefined; - - return { - input: result.input, - output, - name: result.input.name, - score: output._success ? 1 : 0, - ...(categories && { categories }), - }; - }); + const summaryResults = evalResult.results.map((result) => toSummaryResult(result)); const resolvedExperimentName = evalResult.summary?.experimentName ?? experimentName; const resolvedExperimentUrl = evalResult.summary?.experimentUrl; diff --git a/packages/evals/framework/stepBudget.ts b/packages/evals/framework/stepBudget.ts new file mode 100644 index 000000000..1dc6d87dc --- /dev/null +++ b/packages/evals/framework/stepBudget.ts @@ -0,0 +1,46 @@ +/** Env var honored by every external harness when its own key is unset. */ +export const SHARED_STEP_BUDGET_ENV = "AGENT_EVAL_MAX_STEPS"; + +/** + * Per-dataset execution budgets. Reaching a budget records step_budget; + * the verifier still determines task completion from the captured evidence. + */ +export const DATASET_STEP_BUDGETS: Readonly> = { + hardbenchmark: 100, +}; + +export interface ResolveStepBudgetInput { + /** Harness-specific env key (e.g. EVAL_CODEX_MAX_STEPS, EVAL_CLAUDE_CODE_MAX_TURNS). */ + harnessEnvKey: string; + dataset: string | undefined; + /** The harness's historical default, used when no env or dataset budget applies. */ + harnessDefault: number; + env?: NodeJS.ProcessEnv; +} + +/** + * Resolve the agent step budget for one run. Precedence: + * harness env key → AGENT_EVAL_MAX_STEPS → DATASET_STEP_BUDGETS[dataset] → harnessDefault. + * + * The unit is whatever the harness counts (tool steps for most, turns for + * claude_code and pi). A shared number does not equate those units; runners + * record the effective value and unit for comparison. + */ +export function resolveStepBudget({ + harnessEnvKey, + dataset, + harnessDefault, + env = process.env, +}: ResolveStepBudgetInput): number { + for (const key of [harnessEnvKey, SHARED_STEP_BUDGET_ENV]) { + const parsed = readPositiveInt(env[key]); + if (parsed !== undefined) return parsed; + } + const datasetBudget = dataset ? DATASET_STEP_BUDGETS[dataset] : undefined; + return datasetBudget ?? harnessDefault; +} + +function readPositiveInt(raw: string | undefined): number | undefined { + const parsed = Number(raw); + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined; +} diff --git a/packages/evals/framework/usageNormalization.ts b/packages/evals/framework/usageNormalization.ts new file mode 100644 index 000000000..4f991238d --- /dev/null +++ b/packages/evals/framework/usageNormalization.ts @@ -0,0 +1,158 @@ +import type { ExternalHarnessUsage } from "./harnesses/externalRunner.js"; + +/** + * How a harness SDK lays out the token buckets it reports. + * + * - `openai_cached_subset`: `inputTokens` is the whole prompt; cached (and any + * cache-write) tokens are a subset of it. Reasoning tokens are a subset of + * `outputTokens`. This is the OpenAI Responses shape and also what the AI SDK + * (v6+) and LangChain normalize every provider onto. + * - `anthropic_cache_separate`: `inputTokens` excludes cache reads and cache + * writes; both are reported in their own fields. The raw Anthropic Messages + * API shape, surfaced unchanged by the Claude Agent SDK. + * - `uncached_only`: the harness re-normalizes every provider onto an + * Anthropic-style split, so `inputTokens` is always the uncached remainder + * (pi: `usage.input` = 40 alongside 127k `cacheRead` observed). + * - `unreported`: the SDK exposes no token usage at all; every bucket is 0 and + * must not be priced. + */ +export type UsageConvention = + | "openai_cached_subset" + | "anthropic_cache_separate" + | "uncached_only" + | "unreported"; + +export interface NormalizedUsage { + /** Every prompt token billed on any input rate: uncached + cached + cache writes. */ + input_total: number; + /** Prompt tokens served from a prompt cache. */ + input_cached: number; + /** Prompt tokens written into a prompt cache (0 when the SDK does not split them out). */ + input_cache_write: number; + /** Prompt tokens billed at the plain input rate. */ + input_uncached: number; + /** Completion tokens, including reasoning when `reasoning_in_output`. */ + output: number; + /** Reasoning / thinking tokens the SDK reported (0 when it does not). */ + reasoning: number; + /** Whether `reasoning` is already counted inside `output`. */ + reasoning_in_output: boolean; + convention: UsageConvention; +} + +export interface NormalizeUsageInput { + harness: string; + /** + * Model provider (e.g. "anthropic"). Accepted for forward compatibility; today + * every harness SDK reports one shape regardless of provider, so the + * convention is decided per harness. + */ + provider?: string; + raw: ExternalHarnessUsage; +} + +/** + * Convention per registered harness, decided from what each SDK actually + * reports (see the runner's usage extraction, not its docs): + * + * | harness | evidence | + * |-------------|---------------------------------------------------------------------------------------| + * | claude_code | result message `usage.input_tokens` + separate `cache_read/creation_input_tokens` | + * | codex | `turn.completed` usage: `cached_input_tokens` ⊂ `input_tokens`, reasoning ⊂ output | + * | mastra | @mastra/core 1.57 on ai@7: `inputTokens` total, `cachedInputTokens` subset | + * | eve | eve 0.29 reads AI SDK 7 `usage.inputTokens` + `inputTokenDetails.cacheReadTokens` | + * | deepagents | LangChain `usage_metadata.input_tokens` total, `input_token_details.cache_read` subset | + * | fx | `usage-v2.json`: cached input is a subset; reasoning_tokens is separate from output_tokens | + * | pi | pi-ai `usage.input` is the uncached remainder; `cacheRead`/`cacheWrite` separate | + * | cursor | SDK input excludes cache; historical CLI records explicitly report no usage | + * | cursor_sdk | SDK `totalTokens` sums input, output, cacheRead and cacheWrite; input excludes cache | + * | claude_cua | raw Messages API `usage.input_tokens` + separate `cache_read/creation_input_tokens` | + */ +const HARNESS_CONVENTIONS: Readonly> = { + claude_code: "anthropic_cache_separate", + codex: "openai_cached_subset", + mastra: "openai_cached_subset", + eve: "openai_cached_subset", + deepagents: "openai_cached_subset", + fx: "openai_cached_subset", + pi: "uncached_only", + cursor: "uncached_only", + cursor_sdk: "uncached_only", + claude_cua: "anthropic_cache_separate", + gemini_cua: "openai_cached_subset", +}; + +/** Unknown harnesses get the most common SDK shape; the metric is still labelled. */ +const DEFAULT_CONVENTION: UsageConvention = "openai_cached_subset"; + +export function usageConventionFor(harness: string): UsageConvention { + return Object.hasOwn(HARNESS_CONVENTIONS, harness) + ? HARNESS_CONVENTIONS[harness] + : DEFAULT_CONVENTION; +} + +export function normalizeUsage({ harness, raw }: NormalizeUsageInput): NormalizedUsage { + const input = nonNegative(raw.inputTokens); + const cached = nonNegative(raw.cachedInputTokens); + const cacheWrite = nonNegative(raw.cacheCreationInputTokens); + const output = nonNegative(raw.outputTokens); + const reasoning = nonNegative(raw.reasoningOutputTokens); + // Legacy runners initialize absent telemetry to zero. Without an explicit + // presence flag, zeros cannot establish a free run. Nonzero component + // buckets remain usable for legacy records; reported:true preserves an + // actual zero-valued usage event. Total alone cannot price missing buckets. + const declaredConvention = usageConventionFor(harness); + const hasUsage = + raw.reported === true || + input > 0 || + output > 0 || + (declaredConvention !== "openai_cached_subset" && (cached > 0 || cacheWrite > 0)); + const convention = raw.reported === false || !hasUsage ? "unreported" : declaredConvention; + + switch (convention) { + case "openai_cached_subset": + return { + input_total: input, + input_cached: Math.min(cached, input), + input_cache_write: Math.min(cacheWrite, Math.max(0, input - cached)), + input_uncached: Math.max(0, input - cached - cacheWrite), + output, + reasoning, + reasoning_in_output: harness !== "fx", + convention, + }; + case "anthropic_cache_separate": + case "uncached_only": + return { + input_total: input + cached + cacheWrite, + input_cached: cached, + input_cache_write: cacheWrite, + input_uncached: input, + output, + reasoning, + reasoning_in_output: true, + convention, + }; + case "unreported": + return { + input_total: 0, + input_cached: 0, + input_cache_write: 0, + input_uncached: 0, + output: 0, + reasoning: 0, + reasoning_in_output: true, + convention, + }; + } +} + +/** `in= (cached ) out=` for the trace result line. */ +export function formatNormalizedUsage(usage: NormalizedUsage): string { + if (usage.convention === "unreported") return "in=? out=? (usage unreported)"; + return `in=${usage.input_total} (cached ${usage.input_cached}) out=${usage.output}`; +} + +function nonNegative(value: number | undefined): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0; +} diff --git a/packages/evals/framework/verifierAdapter.ts b/packages/evals/framework/verifierAdapter.ts index d7f1a3d8c..791d39060 100644 --- a/packages/evals/framework/verifierAdapter.ts +++ b/packages/evals/framework/verifierAdapter.ts @@ -10,31 +10,107 @@ import { type V3, } from "stagehand-v3"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; + import type { EvalLogger } from "../logger.js"; import { tracedSpan } from "./braintrust.js"; import { persistAdapterTrajectory } from "./harnesses/persistTrajectory.js"; +import { + selectVerifierTraceLines, + verifierTraceEnabled, + writeVerifierTrace, +} from "./verifierTrace.js"; +import type { HarnessTrajectory } from "./harnesses/trajectoryAdapter.js"; import { RubricCache } from "./rubricCache.js"; import type { TaskResult } from "./types.js"; +import { applyVerdictGates, resolveRequireGrounding, type VerdictGates } from "./verifierGates.js"; + +/** + * What scores/result.json holds: the judge's EvaluationResult shape with the + * gated verdict at the top level, so a reader of result.json alone sees the + * same outcome as the Braintrust row. The judge's untouched verdict is kept + * under `judge` (and as `judgeOutcomeSuccess` / `processScoreLenient`). + */ +export interface PersistedEvaluationResult extends EvaluationResult { + judgeOutcomeSuccess: boolean; + outcomeGates: VerdictGates["outcomeGates"]; + processScoreStrict: number | undefined; + processScoreLenient: number | undefined; + judge: EvaluationResult; +} + +export interface UngradedVerifierResult { + graded: false; + verifierError: string; + judge?: EvaluationResult; +} + +export function buildUngradedVerifierResult( + verifierError: string, + judge?: EvaluationResult, +): UngradedVerifierResult { + return { + graded: false, + verifierError: sanitizeErrorMessage(verifierError), + ...(judge && { judge }), + }; +} + +/** Retain a failed judge response for audit without presenting its synthetic scores as a grade. */ +export function getUngradedVerifierResult( + evaluation: EvaluationResult, +): { graded: false; verifierError: string; judge: EvaluationResult } | undefined { + if (!evaluation.findings?.some((finding) => finding.category === "verifier_uncertainty")) { + return undefined; + } + return { + graded: false, + verifierError: "Verifier returned an uncertainty result; no trustworthy grade was produced.", + judge: evaluation, + }; +} + +export function buildPersistedEvaluationResult( + evaluation: EvaluationResult, + gates: VerdictGates, +): PersistedEvaluationResult { + return { + ...evaluation, + outcomeSuccess: gates.outcomeSuccess, + processScore: gates.processScore, + ...(gates.perCriterion && { perCriterion: gates.perCriterion }), + judgeOutcomeSuccess: gates.judgeOutcomeSuccess, + outcomeGates: gates.outcomeGates, + processScoreStrict: gates.processScoreStrict, + processScoreLenient: gates.processScoreLenient, + judge: evaluation, + }; +} const VERIFIER_MODEL_ENV = "EVAL_VERIFIER_MODEL"; const KEYLESS_VERIFIER_PROVIDERS = new Set(["bedrock", "ollama"]); +/** Shared default; callers can pin the judge independently. */ +export const DEFAULT_VERIFIER_MODEL = "google/gemini-3.5-flash"; /** - * Build the shared rubric verifier. By default V3Evaluator keeps its existing - * model selection; EVAL_VERIFIER_MODEL makes the verifier independently - * selectable for external harnesses and normal Stagehand runs alike. + * Build the shared rubric verifier. EVAL_VERIFIER_MODEL makes the verifier + * independently selectable for external harnesses and normal Stagehand runs + * alike; otherwise DEFAULT_VERIFIER_MODEL applies. A command-line modelOverride + * takes precedence over the environment without mutating process-wide policy. */ -export function createVerifierEvaluator(v3: V3): V3Evaluator { - const modelName = process.env[VERIFIER_MODEL_ENV]?.trim(); - if (!modelName) { - return new V3Evaluator(v3, { backend: "verifier" }); - } +export function createVerifierEvaluator(v3: V3, modelOverride?: string): V3Evaluator { + const explicitModel = modelOverride?.trim() || process.env[VERIFIER_MODEL_ENV]?.trim(); + const modelName = explicitModel || DEFAULT_VERIFIER_MODEL; const provider = modelName.includes("/") ? modelName.slice(0, modelName.indexOf("/")) : undefined; const apiKey = loadApiKeyFromEnv(provider, () => {}); - if (!apiKey && !KEYLESS_VERIFIER_PROVIDERS.has(provider ?? "")) { + // Only an explicit override fails loudly on a missing key; the default lets + // V3Evaluator resolve credentials itself (tests and keyless environments). + if (explicitModel && !apiKey && !KEYLESS_VERIFIER_PROVIDERS.has(provider ?? "")) { throw new Error( - `${VERIFIER_MODEL_ENV} is set to "${modelName}", but no API key was found for provider "${provider ?? "unknown"}".`, + `Verifier model is explicitly set to "${modelName}", but no API key was found for provider "${provider ?? "unknown"}".`, ); } @@ -135,13 +211,18 @@ export async function verifyTraced( async (span) => { const v = await evaluator.verify(trajectory); const rawSteps = asRecord(v.rawSteps); + const ungraded = getUngradedVerifierResult(v); span.log({ output: v, - scores: { - outcome: v.outcomeSuccess ? 1 : 0, - process: v.processScore, - }, + ...(!ungraded && { + scores: { + outcome: v.outcomeSuccess ? 1 : 0, + process: v.processScore, + }, + }), metadata: { + graded: !ungraded, + ...(ungraded && { verifierError: ungraded.verifierError }), taskId: meta.taskId, dataset: meta.dataset, stepCount: trajectory.steps.length, @@ -192,7 +273,7 @@ export interface ExternalHarnessVerifierConfig { export interface GradeExternalTrajectoryOptions { /** Builds the harness-specific Trajectory; runs inside the guarded block. */ - buildTrajectory: () => Trajectory; + buildTrajectory: () => HarnessTrajectory; verifier: ExternalHarnessVerifierConfig; /** The agent's self-reported result to fold the verdict into. */ baseResult: TaskResult; @@ -201,13 +282,18 @@ export interface GradeExternalTrajectoryOptions { /** Logger category ("claude_code" | "codex"). */ category: string; logger: EvalLogger; + /** + * Matcher for mounted-browser (facade) tool names. When present, a judge + * pass with zero facade steps is gated (`no_browser_use`). + */ + isFacadeTool?: (name: string) => boolean; } /** * Grade an external-harness run with the rubric verifier and fold the verdict * into the TaskResult. Never throws: on any failure in the verifier path the - * self-reported result is returned with `verifierError` set, so downstream - * consumers can tell an ungraded run apart from a graded one. + * result fails closed with `verifierError` set and the agent report preserved + * separately, so downstream consumers can distinguish ungraded runs. */ export async function gradeExternalTrajectory({ buildTrajectory, @@ -216,9 +302,14 @@ export async function gradeExternalTrajectory({ errorMessage, category, logger, + isFacadeTool, }: GradeExternalTrajectoryOptions): Promise { + let capturedTrajectory: HarnessTrajectory | undefined; + let rawEvaluation: EvaluationResult | undefined; + let savedDirectory: string | undefined; try { const trajectory = buildTrajectory(); + capturedTrajectory = trajectory; const evaluator = createVerifierEvaluator(verifier.v3); // Hydrate rubric — use precomputed if present, otherwise cache-or-generate. @@ -230,42 +321,126 @@ export async function gradeExternalTrajectory({ ...verifier.taskSpec, precomputedRubric: rubric, }; - const hydratedTrajectory = { ...trajectory, task: hydratedSpec }; + const hydratedTrajectory: HarnessTrajectory = { ...trajectory, task: hydratedSpec }; + capturedTrajectory = hydratedTrajectory; + const traceOn = verifierTraceEnabled(); + const logCountBefore = traceOn ? logger.getLogs({ maxLevel: 2 }).length : 0; const evaluationResult = await verifyTraced(evaluator, hydratedTrajectory, { taskId: hydratedSpec.id, dataset: verifier.dataset, }); + rawEvaluation = evaluationResult; + const ungraded = getUngradedVerifierResult(evaluationResult); + if (ungraded) { + throw new Error(ungraded.verifierError); + } + const traceLines = traceOn + ? selectVerifierTraceLines(logger.getLogs({ maxLevel: 2 }), logCountBefore) + : []; + // The judge's verdict is not the final word: deterministic gates fold in + // what the trajectory itself proves (an answer exists, the run finished, + // the browser was used, the numbers came from the target site) and a + // strict process score that does not credit blocker-walled criteria. See + // verifierGates.ts for why each exists. + const gates = applyVerdictGates({ + evaluation: evaluationResult, + trajectory: hydratedTrajectory, + isFacadeTool, + requireGrounding: resolveRequireGrounding( + verifier.dataset, + Boolean(verifier.taskSpec.precomputedRubric), + ), + rubricItemCount: rubric.items.length, + }); const successMode = verifier.successMode ?? process.env.EVAL_SUCCESS_MODE; - const verifiedSuccess = evaluationResultToSuccess(evaluationResult, successMode); + const verifiedSuccess = evaluationResultToSuccess( + { + ...evaluationResult, + outcomeSuccess: gates.outcomeSuccess, + processScore: gates.processScore, + }, + successMode, + ); - const { directory: trajectoryDir } = await persistAdapterTrajectory({ + const { directory: trajectoryDir, persisted } = await persistAdapterTrajectory({ trajectory: hydratedTrajectory, taskSpec: hydratedSpec, - evaluationResult, + evaluationResult: buildPersistedEvaluationResult(evaluationResult, gates), outputRoot: verifier.trajectoryRoot, runId: verifier.runId, }); + savedDirectory = trajectoryDir; + if (persisted) await writeGatesFile(trajectoryDir, gates); + if (persisted && traceLines.length) { + const file = await writeVerifierTrace(trajectoryDir, traceLines); + if (file) logger.log({ category, message: `verifier trace: ${file}`, level: 1 }); + } + const gateSuffix = gates.outcomeGates.length ? ` gated=${gates.outcomeGates.join(",")}` : ""; logger.log({ category, - message: `result: outcome=${evaluationResult.outcomeSuccess} process=${formatProcessScore(evaluationResult.processScore)} steps=${hydratedTrajectory.steps.length}`, + message: `result: outcome=${gates.outcomeSuccess} (judge=${gates.judgeOutcomeSuccess}${gateSuffix}) process=${formatProcessScore(gates.processScore)} (lenient=${formatProcessScore(gates.processScoreLenient)}) steps=${hydratedTrajectory.steps.length}`, level: 1, }); return { ...baseResult, _success: verifiedSuccess, - error: verifiedSuccess ? undefined : (baseResult.error ?? errorMessage), - outcomeSuccess: evaluationResult.outcomeSuccess, - processScore: evaluationResult.processScore, + error: verifiedSuccess + ? undefined + : gates.outcomeGates.length > 0 && gates.judgeOutcomeSuccess + ? // The judge passed this row; a deterministic gate flipped it. Say + // so where the row error is read, instead of echoing the agent's + // (often confident) self-report. + `${describeOutcomeGates(gates)} (judge passed; agent said: ${clipError(String(baseResult.error ?? errorMessage))})` + : (baseResult.error ?? errorMessage), + outcomeSuccess: gates.outcomeSuccess, + judgeOutcomeSuccess: gates.judgeOutcomeSuccess, + outcomeGates: gates.outcomeGates, + processScore: gates.processScore, + processScoreStrict: gates.processScoreStrict, + processScoreLenient: gates.processScoreLenient, + perCriterion: gates.perCriterion, evidenceInsufficient: evaluationResult.evidenceInsufficient, + ...(gates.grounding && { grounding: gates.grounding }), + scoringIncomplete: gates.scoringIncomplete, criterionCount: rubric.items.length, stepCount: hydratedTrajectory.steps.length, trajectoryDir, + metrics: { + ...(asRecord(baseResult.metrics) ?? {}), + ...gateMetrics(gates), + }, }; } catch (verifyError) { - const message = stringifyVerifierError(verifyError); + const message = sanitizeErrorMessage(stringifyVerifierError(verifyError)); + // Failed verification still needs the captured browser evidence and raw + // judge response for diagnosis. This path never accepts the result as a grade. + if (capturedTrajectory && !savedDirectory) { + try { + const saved = await persistAdapterTrajectory({ + trajectory: capturedTrajectory, + taskSpec: capturedTrajectory.task ?? verifier.taskSpec, + evaluationResult: buildUngradedVerifierResult(message, rawEvaluation), + outputRoot: verifier.trajectoryRoot, + runId: verifier.runId, + }); + savedDirectory = saved.directory; + if (saved.persisted) { + await fs.writeFile( + path.join(saved.directory, "scores", "verifier-error.json"), + JSON.stringify({ verifierError: message, graded: false }, null, 2), + ); + } + } catch (persistenceError) { + logger.warn({ + category, + level: 1, + message: `could not persist failed verification: ${sanitizeErrorMessage(stringifyVerifierError(persistenceError))}`, + }); + } + } logger.warn({ category, message: `verifier integration failed: ${message}`, @@ -274,10 +449,16 @@ export async function gradeExternalTrajectory({ error: { value: message, type: "string" }, }, }); - // Surface the failure on the result — `_success` falls back to the - // agent's self-report, and downstream consumers must be able to tell - // this run apart from one the verifier actually graded. - return { ...baseResult, verifierError: message }; + // A requested verification that failed cannot produce a verified pass. + // Preserve the agent's report separately for diagnosis. + return { + ...baseResult, + _success: false, + agentReportedSuccess: baseResult._success, + verifierError: message, + error: `Verification failed: ${message}`, + ...(savedDirectory && { trajectoryDir: savedDirectory }), + }; } } @@ -285,6 +466,38 @@ function formatProcessScore(score: number | undefined): string { return typeof score === "number" ? score.toFixed(2) : "n/a"; } +/** + * Braintrust-filterable 0/1 metrics for the gates. `answer_grounded` is only + * emitted when the answer had numeric datums to check, so its average is not + * diluted by rows the check skipped. + */ +function gateMetrics(gates: VerdictGates): Record { + const flag = (value: boolean) => ({ count: 1, value: value ? 1 : 0 }); + return { + outcome_gated: flag(gates.outcomeGates.length > 0), + scoring_incomplete: flag(gates.scoringIncomplete), + blocked_criteria: { count: 1, value: gates.blockedCriteria }, + ...(typeof gates.processScoreLenient === "number" && { + process_score_lenient: { count: 1, value: gates.processScoreLenient }, + }), + ...(gates.grounding && { + answer_grounded: flag(!gates.grounding.gatesOutcome), + }), + }; +} + +/** Sidecar next to scores/result.json so audits can diff judge vs gated verdicts. */ +async function writeGatesFile(trajectoryDir: string, gates: VerdictGates): Promise { + try { + await fs.writeFile( + path.join(trajectoryDir, "scores", "gates.json"), + JSON.stringify(gates, null, 2), + ); + } catch { + // Best-effort: the TaskResult already carries the same data. + } +} + /** Always non-empty, so a set `verifierError` is reliably truthy downstream. */ function stringifyVerifierError(value: unknown): string { if (value instanceof Error) return value.message || value.name || "Error"; @@ -342,3 +555,20 @@ export function evaluationResultToSuccess( return outcomeOk && processOk; } } + +const GATE_DESCRIPTIONS: Record = { + no_final_answer: "agent produced no final answer", + trajectory_error: "trajectory ended in error", + no_browser_use: "no browser tool calls", + ungrounded_answer: "answer datums only in search-engine results, never on a target page", +}; + +function describeOutcomeGates(gates: { outcomeGates: string[] }): string { + const parts = gates.outcomeGates.map((g) => GATE_DESCRIPTIONS[g] ?? g); + return `gated: ${gates.outcomeGates.join(",")} — ${parts.join("; ")}`; +} + +function clipError(value: string | undefined): string { + const text = (value ?? "").replace(/\s+/g, " ").trim(); + return text.length > 160 ? `${text.slice(0, 159)}…` : text; +} diff --git a/packages/evals/framework/verifierGate.ts b/packages/evals/framework/verifierGate.ts index a2fd2a469..e0f4e10a1 100644 --- a/packages/evals/framework/verifierGate.ts +++ b/packages/evals/framework/verifierGate.ts @@ -15,16 +15,31 @@ export interface ArmVerifiability { gradedRuns: number; /** * Verifier-backed runs the verifier failed to grade (`verifierError` set) — - * their `_success` is the agent's self-report, so they must never hide - * inside a gated batch. + * keep these separate from completed grades, including historical rows + * whose `_success` may still contain a self-report. */ ungradedRuns: number; unverifiableCriteria: number; totalCriteria: number; + /** + * Runs marked successful whose `facade_tool_calls` metric is 0 — the agent + * never reached the mounted browser surface, so the pass came from + * somewhere the verifier cannot see (another tool, prior knowledge). + */ + passesWithoutBrowserUse: number; } const GATE_ENV = "EVAL_MAX_UNVERIFIABLE_CRITERIA"; +function readFacadeToolCalls(output: Record): number | undefined { + const metrics = output.metrics; + if (typeof metrics !== "object" || metrics === null) return undefined; + const metric = (metrics as Record).facade_tool_calls; + if (typeof metric !== "object" || metric === null) return undefined; + const value = (metric as { value?: unknown }).value; + return typeof value === "number" ? value : undefined; +} + export function summarizeArmVerifiability( results: Array<{ input: EvalInput; output: Record }>, harness: string, @@ -43,6 +58,7 @@ export function summarizeArmVerifiability( ungradedRuns: 0, unverifiableCriteria: 0, totalCriteria: 0, + passesWithoutBrowserUse: 0, }; if (graded) { arm.gradedRuns += 1; @@ -50,6 +66,9 @@ export function summarizeArmVerifiability( arm.unverifiableCriteria += Array.isArray(output.evidenceInsufficient) ? output.evidenceInsufficient.length : 0; + if (output._success === true && readFacadeToolCalls(output) === 0) { + arm.passesWithoutBrowserUse += 1; + } } else { arm.ungradedRuns += 1; } @@ -78,3 +97,8 @@ export function armsOverLimit(arms: ArmVerifiability[], limit: number): ArmVerif export function armsWithUngradedRuns(arms: ArmVerifiability[]): ArmVerifiability[] { return arms.filter((arm) => arm.ungradedRuns > 0); } + +/** Arms where at least one pass never touched the mounted browser surface. */ +export function armsWithPassesWithoutBrowserUse(arms: ArmVerifiability[]): ArmVerifiability[] { + return arms.filter((arm) => arm.passesWithoutBrowserUse > 0); +} diff --git a/packages/evals/framework/verifierGates.ts b/packages/evals/framework/verifierGates.ts new file mode 100644 index 000000000..67f2c041a --- /dev/null +++ b/packages/evals/framework/verifierGates.ts @@ -0,0 +1,530 @@ +/** + * Deterministic diagnostics and evidence gates over the current V3 verifier. + * Preserve the judge's verdict for auditing. Execution status and blocker + * wording alone cannot establish whether a rubric requirement was completed. + */ +import type { + CriterionScore, + EvaluationResult, + ProbeEvidence, + Trajectory, + TrajectoryStep, +} from "stagehand-v3"; + +export type OutcomeGate = "no_final_answer" | "no_browser_use" | "ungrounded_answer"; + +export type GroundingDatumKind = "currency" | "percent" | "time" | "decimal" | "integer" | "entity"; + +export interface GroundingDatum { + /** The token as written in the final answer. */ + text: string; + kind: GroundingDatumKind; + /** Index of the first non-search-engine step whose output contains it. */ + groundedAtStep?: number; + /** Matching captured text came from the terminal observation, not a tool step. */ + groundedAtFinalObservation?: true; + /** Matching text existed but its page URL was unknown, so it could not ground the answer. */ + seenOnUnknownPage?: true; + /** True when the datum only ever appeared in search-engine step outputs. */ + onlyInSearchEngine: boolean; + /** Echoed from the task instruction; reported but never gates. */ + fromInstruction?: boolean; +} + +export interface GroundingResult { + checked: GroundingDatum[]; + /** Subset of `checked` that no non-search-engine step output contains. */ + ungrounded: GroundingDatum[]; + /** + * Counts over datums of a kind allowed to gate the outcome (currency / + * percent / time / decimal / integer with 4+ digits). Entity and + * short-integer misses are advisory only. + */ + groundedNumeric: number; + ungroundedNumeric: number; + /** True when the answer has numeric datums and none was verified off-search. */ + gatesOutcome: boolean; +} + +export interface GatedCriterionScore extends CriterionScore { + /** + * Flagged by the judge as lacking evidence. Zeroed in the strict + * process score. + */ + blocked?: boolean; + /** Advisory wording match; a sanctioned fallback or stop boundary may mention a blocker. */ + blockerMentioned?: boolean; +} + +export interface VerdictGates { + /** Gated verdict: judge AND every deterministic gate. */ + outcomeSuccess: boolean; + /** The judge's raw verdict, untouched. */ + judgeOutcomeSuccess: boolean; + /** Gates that flipped a judge pass to a fail. Empty when nothing fired. */ + outcomeGates: OutcomeGate[]; + /** Same as `processScoreStrict`; the score `--success process` reads. */ + processScore: number | undefined; + processScoreStrict: number | undefined; + /** The judge's processScore, untouched. */ + processScoreLenient: number | undefined; + perCriterion: GatedCriterionScore[] | undefined; + /** Count of criteria zeroed in the strict score. */ + blockedCriteria: number; + /** Undefined when grounding was not checked (no final answer / no datums). */ + grounding?: GroundingResult; + /** Rubric has more items than the judge returned criterion scores for. */ + scoringIncomplete: boolean; + rubricItemCount?: number; +} + +export interface ApplyVerdictGatesInput { + evaluation: EvaluationResult; + trajectory: Pick & { + task?: { instruction?: string }; + }; + /** Matcher for mounted-browser (facade) tool names; enables `no_browser_use`. */ + isFacadeTool?: (name: string) => boolean; + /** Whether an ungrounded numeric datum in the answer fails the outcome. */ + requireGrounding: boolean; + /** `task_data.precomputed_rubric.items.length`; enables `scoringIncomplete`. */ + rubricItemCount?: number; +} + +const BLOCKER_EXPLANATION = /uncontrollable|blocker|could not be attempted|blocked/i; + +const SEARCH_ENGINE_HOST = /(^|\.)(google|bing|duckduckgo|yahoo|scribd|reddit)\./i; + +/** Datasets whose rubrics are precomputed and whose answers are factual lookups. */ + +export function applyVerdictGates({ + evaluation, + trajectory, + isFacadeTool, + requireGrounding, + rubricItemCount, +}: ApplyVerdictGatesInput): VerdictGates { + const judgeOutcomeSuccess = evaluation.outcomeSuccess === true; + const finalAnswer = (trajectory.finalAnswer ?? "").trim(); + const outcomeGates: OutcomeGate[] = []; + + if (!finalAnswer) outcomeGates.push("no_final_answer"); + // A disconnect after the required actions does not erase their evidence. + // Execution status is retained separately; the judge must assess completion. + if (isFacadeTool && !trajectory.steps.some((step) => isFacadeTool(step.actionName))) { + outcomeGates.push("no_browser_use"); + } + + const grounding = finalAnswer + ? checkAnswerGrounding( + finalAnswer, + trajectory.steps, + trajectory.task?.instruction ?? "", + trajectory.finalObservation, + ) + : undefined; + if (requireGrounding && grounding?.gatesOutcome) { + outcomeGates.push("ungrounded_answer"); + } + + const { perCriterion, strict, blockedCriteria } = strictProcessScore(evaluation); + + const scoringIncomplete = + typeof rubricItemCount === "number" && + rubricItemCount > 0 && + rubricItemCount > (evaluation.perCriterion?.length ?? 0); + + return { + // Gates only ever flip a pass to a fail; a judge fail stays a fail even + // when no gate fires, so the gate list is only meaningful on a judge pass. + outcomeSuccess: judgeOutcomeSuccess && outcomeGates.length === 0, + judgeOutcomeSuccess, + outcomeGates: judgeOutcomeSuccess ? outcomeGates : [], + processScore: strict, + processScoreStrict: strict, + processScoreLenient: evaluation.processScore, + perCriterion, + blockedCriteria, + ...(grounding && { grounding }), + scoringIncomplete, + ...(rubricItemCount !== undefined && { rubricItemCount }), + }; +} + +/** + * Resolve EVAL_REQUIRE_GROUNDING. Grounding is advisory unless explicitly enabled. + */ +export function resolveRequireGrounding( + dataset: string, + hasPrecomputedRubric: boolean, + env: NodeJS.ProcessEnv = process.env, +): boolean { + const raw = env.EVAL_REQUIRE_GROUNDING?.trim(); + if (raw === "1" || raw?.toLowerCase() === "true") return true; + if (raw === "0" || raw?.toLowerCase() === "false") return false; + // Advisory by default: a factually correct answer sourced from a search + // snippet still counts as a pass (owner decision, 2026-08-31 — the F1 race + // time answered from Google was ruled a pass). The check still runs and is + // recorded as `grounding` + metric answer_grounded so snippet-sourced passes + // remain filterable; opt into gating with EVAL_REQUIRE_GROUNDING=1. + void dataset; + void hasPrecomputedRubric; + return false; +} + +/** + * Recompute the process score with explicitly evidenceInsufficient-credited + * criteria zeroed. Blocked criteria stay in the denominator: dropping them + * would score a fully bot-walled run 0/0 and, with any fallback, let it pass + * `--success process` again — exactly the leak this exists to close. + * Not-applicable criteria (earnedPoints === null) are excluded, as the judge + * does. + */ +export function strictProcessScore( + evaluation: Pick, +): { + perCriterion: GatedCriterionScore[] | undefined; + strict: number | undefined; + blockedCriteria: number; +} { + const source = evaluation.perCriterion; + // Without a per-criterion breakdown there is nothing to zero; the judge's + // aggregate is the best available strict score. + if (!source) + return { perCriterion: undefined, strict: evaluation.processScore, blockedCriteria: 0 }; + + const insufficient = new Set(evaluation.evidenceInsufficient ?? []); + let earned = 0; + let max = 0; + let blockedCriteria = 0; + const perCriterion = source.map((criterion): GatedCriterionScore => { + const blocked = + criterion.evidenceInsufficient === true || insufficient.has(criterion.criterion); + const blockerMentioned = BLOCKER_EXPLANATION.test(criterion.explanation ?? ""); + // V3 also uses null for a missing judge score, with explicit insufficiency. + // Only genuinely inapplicable criteria leave the denominator. + if (criterion.earnedPoints === null && !blocked) return { ...criterion }; + max += criterion.maxPoints; + if (blocked) blockedCriteria += 1; + else earned += criterion.earnedPoints ?? 0; + return { + ...criterion, + ...(blocked && { blocked: true }), + ...(blockerMentioned && { blockerMentioned: true }), + }; + }); + + return { + perCriterion, + strict: max > 0 ? earned / max : undefined, + blockedCriteria, + }; +} + +// --------------------------------------------------------------------------- +// Grounding +// --------------------------------------------------------------------------- + +const CURRENCY_CODES = "SGD|USD|MYR|EUR|GBP|AUD|CAD|INR|JPY|CNY|HKD|NZD|CHF|THB|IDR|PHP|KRW"; +const CURRENCY_PREFIX = new RegExp( + `(?:(${CURRENCY_CODES}|S\\$|US\\$|A\\$|C\\$|\\$|€|£|¥)\\s?)`, + "i", +); +const TIME_TOKEN = /\b\d{1,2}:\d{2}(?::\d{2})?(?:\.\d{1,3})?\b/g; +const YEAR_TOKEN = /^(?:1[89]|20)\d{2}$/; +const NUMBER_TOKEN = new RegExp( + `${CURRENCY_PREFIX.source}?(\\d[\\d,]*(?:\\.\\d+)?)(?:\\s?(${CURRENCY_CODES}|%|\\$))?`, + "gi", +); +// Two or more Capitalised words, allowing a lowercase joiner in between. +const ENTITY_TOKEN = + /\b[A-Z][\w’'&-]*(?:\s+(?:of|the|and|de|for|&)\s+|\s+)(?:[A-Z][\w’'&-]*)(?:\s+(?:[A-Z][\w’'&-]*))*/g; + +const CURRENCY_ALIASES: Record = { + $: ["$", "usd", "us$"], + usd: ["usd", "us$", "$"], + us$: ["us$", "usd", "$"], + sgd: ["sgd", "s$"], + s$: ["s$", "sgd"], + aud: ["aud", "a$"], + a$: ["a$", "aud"], + cad: ["cad", "c$"], + c$: ["c$", "cad"], + "€": ["€", "eur"], + eur: ["eur", "€"], + "£": ["£", "gbp"], + gbp: ["gbp", "£"], + "¥": ["¥", "jpy", "cny"], + jpy: ["jpy", "¥"], + cny: ["cny", "¥", "rmb"], +}; + +interface ParsedDatum { + text: string; + kind: GroundingDatumKind; + /** Matchers tried against a normalized step text; any hit grounds the datum. */ + patterns: RegExp[]; + /** Whether an ungrounded miss may gate the outcome. */ + gates: boolean; + /** Echoed verbatim from the task instruction — task context, not a finding. */ + fromInstruction?: boolean; +} + +/** + * Lowercase, drop thousands separators, collapse whitespace to one space. Space + * is kept (not removed) so digit boundaries survive: "1:27:02.624 2 Pits" must + * not read as "...6242". + */ +export function normalizeForGrounding(text: string): string { + return text.toLowerCase().replace(/,/g, "").replace(/\s+/g, " ").trim(); +} + +/** Regex source matching `text` with any run of whitespace between its tokens. */ +function spaceTolerant(text: string): string { + return normalizeForGrounding(text).split(" ").map(escapeRegExp).join("\\s*"); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Extract the datums worth grounding from a final answer. Datums that also + * appear in the task instruction (dates, budgets, quantities the task set) are + * kept for reporting but never gate: the agent did not find them anywhere. + */ +export function extractGroundingDatums(finalAnswer: string, instruction = ""): ParsedDatum[] { + const datums: ParsedDatum[] = []; + const seen = new Set(); + const normalizedInstruction = normalizeForGrounding(instruction); + const push = (datum: ParsedDatum) => { + const normalized = normalizeForGrounding(datum.text); + const key = `${datum.kind}:${normalized}`; + if (seen.has(key)) return; + seen.add(key); + const fromInstruction = + normalizedInstruction.length > 0 && normalizedInstruction.includes(normalized); + datums.push(fromInstruction ? { ...datum, gates: false, fromInstruction } : datum); + }; + + let remaining = finalAnswer; + for (const match of finalAnswer.matchAll(TIME_TOKEN)) { + const text = match[0]; + push({ + text, + kind: "time", + patterns: [new RegExp(`(?= 3; + const barePattern = new RegExp(`(? { + const a = escapeRegExp(alias); + // "sgd5", "sgd 5.00", "5sgd", "5.00 sgd" + return [ + new RegExp(`${a}\\s?${coreRe}(?:\\.0+)?(?!\\d)`), + new RegExp(`(?= 4 && !YEAR_TOKEN.test(core), + }); + } + + for (const match of finalAnswer.matchAll(ENTITY_TOKEN)) { + const text = match[0].trim(); + if (text.split(/\s+/).length < 2) continue; + push({ + text, + kind: "entity", + patterns: [new RegExp(spaceTolerant(text))], + gates: false, + }); + } + + return datums; +} + +/** Best-effort page URL for a step, carried forward from earlier steps when absent. */ +export function stepUrlHint(step: TrajectoryStep): string | undefined { + const probeUrl = step.probeEvidence?.url; + if (typeof probeUrl === "string" && probeUrl) return probeUrl; + const fromArgs = firstUrl(safeStringify(step.actionArgs)); + if (fromArgs) return fromArgs; + return firstUrl(toolOutputText(step)); +} + +function firstUrl(text: string): string | undefined { + const match = /https?:\/\/[^\s"'\\)\]}>,]+/i.exec(text); + return match?.[0]; +} + +export function isSearchEngineUrl(url: string | undefined): boolean { + if (!url) return false; + let host: string; + try { + host = new URL(url).hostname; + } catch { + const match = /^https?:\/\/([^/?#]+)/i.exec(url); + host = match?.[1] ?? url; + } + return SEARCH_ENGINE_HOST.test(host); +} + +function safeStringify(value: unknown): string { + if (value === undefined || value === null) return ""; + if (typeof value === "string") return value; + try { + return JSON.stringify(value) ?? ""; + } catch { + return String(value); + } +} + +function toolOutputText(step: TrajectoryStep): string { + const parts = [ + safeStringify(step.toolOutput?.result), + step.toolOutput?.error ?? "", + step.probeEvidence?.ariaTree ?? "", + ]; + for (const modality of step.agentEvidence?.modalities ?? []) { + if (modality.type === "text") parts.push(modality.content); + else if (modality.type === "json") parts.push(safeStringify(modality.content)); + } + return parts.join("\n"); +} + +/** + * Check that every key datum in the final answer appears in at least one step + * output whose page was not a search engine. A number that only shows up in + * Google/Bing snippets (or nowhere) was never verified on the target site. + */ +export function checkAnswerGrounding( + finalAnswer: string, + steps: TrajectoryStep[], + instruction = "", + finalObservation?: ProbeEvidence, +): GroundingResult | undefined { + const datums = extractGroundingDatums(finalAnswer, instruction); + if (datums.length === 0) return undefined; + + // Steps with no URL hint (a snapshot after navigation) inherit the page of + // the step before them. + const evidenceTexts: Array<{ + text: string; + source: "unknown" | "search" | "page"; + stepIndex?: number; + }> = []; + const source = (url: string | undefined) => + !url ? "unknown" : isSearchEngineUrl(url) ? "search" : "page"; + let currentUrl: string | undefined; + for (const [stepIndex, step] of steps.entries()) { + const hint = stepUrlHint(step); + if (hint) currentUrl = hint; + evidenceTexts.push({ + text: normalizeForGrounding(toolOutputText(step)), + source: source(currentUrl), + stepIndex, + }); + } + if (finalObservation?.ariaTree) { + evidenceTexts.push({ + text: normalizeForGrounding(finalObservation.ariaTree), + source: source(finalObservation.url || currentUrl), + }); + } + + const checked: GroundingDatum[] = datums.map((datum) => { + let groundedAtStep: number | undefined; + let groundedAtFinalObservation: true | undefined; + let seenInSearch = false; + let seenOnUnknownPage = false; + for (const { text, source, stepIndex } of evidenceTexts) { + if (!datum.patterns.some((pattern) => pattern.test(text))) continue; + if (source === "unknown") { + seenOnUnknownPage = true; + continue; + } + if (source === "search") { + seenInSearch = true; + continue; + } + if (stepIndex === undefined) groundedAtFinalObservation = true; + else groundedAtStep = stepIndex; + break; + } + return { + text: datum.text, + kind: datum.kind, + ...(groundedAtStep !== undefined && { groundedAtStep }), + ...(groundedAtFinalObservation && { groundedAtFinalObservation }), + ...(seenOnUnknownPage && { seenOnUnknownPage: true as const }), + onlyInSearchEngine: + groundedAtStep === undefined && + !groundedAtFinalObservation && + seenInSearch && + !seenOnUnknownPage, + ...(datum.fromInstruction && { fromInstruction: true }), + }; + }); + + const isGrounded = (datum: GroundingDatum) => + datum.groundedAtStep !== undefined || datum.groundedAtFinalObservation === true; + const ungrounded = checked.filter((datum) => !isGrounded(datum)); + const gating = new Set(datums.filter((d) => d.gates).map((d) => `${d.kind}:${d.text}`)); + const isGating = (d: GroundingDatum) => gating.has(`${d.kind}:${d.text}`); + const groundedNumeric = checked.filter((d) => isGating(d) && isGrounded(d)).length; + const ungroundedNumeric = ungrounded.filter(isGating).length; + return { + checked, + ungrounded, + groundedNumeric, + ungroundedNumeric, + // Secondary numbers (a comparison price read off a screenshot, a figure + // paraphrased from a snippet) must not sink a row whose headline datum was + // verified on the target site, so the gate needs every numeric datum to be + // ungrounded. Tightening this to "any" flips ~1 in 5 legitimate passes. + gatesOutcome: ungroundedNumeric > 0 && groundedNumeric === 0, + }; +} diff --git a/packages/evals/framework/verifierTrace.ts b/packages/evals/framework/verifierTrace.ts new file mode 100644 index 000000000..3828df4c4 --- /dev/null +++ b/packages/evals/framework/verifierTrace.ts @@ -0,0 +1,66 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { LogLine } from "stagehand-v3"; +import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; + +type TraceLine = LogLine & { parsedAuxiliary?: unknown }; + +/** + * Full verifier trace (EVAL_VERIFIER_TRACE=1). The v3 evaluator itself logs + * only failures, but its LLM client logs every request and response at + * level 2 (category "aisdk"): the batchedRelevance prompt with each evidence + * item, the top-K selection the judge actually reads, and the fusedJudgment + * response. With the switch on, the verifier carrier runs at verbose 2 and + * the grading-phase lines are written to scores/verifier-trace.jsonl instead + * of the row logs (they are megabytes per task). + */ +export const VERIFIER_TRACE_ENV = "EVAL_VERIFIER_TRACE"; +export const VERIFIER_TRACE_FILE = "verifier-trace.jsonl"; +const TRACE_CATEGORIES = new Set(["aisdk", "AISDK error", "verifier", "llm", "flow"]); + +export function verifierTraceEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const v = env[VERIFIER_TRACE_ENV]?.trim().toLowerCase(); + return v === "1" || v === "true" || v === "yes" || v === "on"; +} + +/** Grading-phase lines worth keeping: the judge's LLM traffic and verifier notes. */ +export function selectVerifierTraceLines( + linesAfter: TraceLine[], + countBefore: number, +): TraceLine[] { + return linesAfter + .slice(countBefore) + .filter((line) => TRACE_CATEGORIES.has(String(line.category ?? ""))); +} + +export function validateVerifierLabel(label: string | undefined): void { + if (label !== undefined && (/[\\/\0]/u.test(label) || label === "." || label === "..")) { + throw new Error("Verifier label must be a single filename component without path separators."); + } +} + +export async function writeVerifierTrace( + trajectoryDir: string, + lines: TraceLine[], + label?: string, +): Promise { + validateVerifierLabel(label); + if (lines.length === 0) return undefined; + const file = path.join( + trajectoryDir, + "scores", + label ? `verifier-trace_${label}.jsonl` : VERIFIER_TRACE_FILE, + ); + try { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, lines.map((line) => JSON.stringify(line)).join("\n") + "\n"); + return file; + } catch (error) { + console.warn( + sanitizeErrorMessage( + `Could not write verifier trace ${file}: ${error instanceof Error ? error.message : String(error)}`, + ), + ); + return undefined; + } +} diff --git a/packages/evals/initStagehand.ts b/packages/evals/initStagehand.ts index dd7d60faa..bbc43eaa6 100644 --- a/packages/evals/initStagehand.ts +++ b/packages/evals/initStagehand.ts @@ -10,6 +10,7 @@ import { import type { EvalLogger } from "./logger.js"; import { launchRunnerProvidedBrowserbaseChrome } from "./core/targets/browserbase.js"; import { onceAsync, registerActiveRunCleanup } from "./framework/activeRunCleanup.js"; +import { EVAL_SYSTEM_PROMPT } from "./framework/evalSystemPrompt.js"; import { resolveKey } from "./tui/welcomeStatus.js"; export type InitStagehandArgs = { @@ -136,6 +137,7 @@ export async function initStagehand({ try { stagehand = await Stagehand.create({ browser, + systemPrompt: EVAL_SYSTEM_PROMPT, // selfHeal defaults off on the server; without it the heal_* benchmarks // pass while measuring nothing. selfHeal: true, diff --git a/packages/evals/logger.ts b/packages/evals/logger.ts index 48767010c..400baf6d9 100644 --- a/packages/evals/logger.ts +++ b/packages/evals/logger.ts @@ -122,11 +122,13 @@ export class EvalLogger { /** * getLogs: - * Retrieves the array of stored log lines. - * Useful for returning logs after a task completes, for analysis or debugging. + * Retrieves the stored log lines at or below `maxLevel` (default 1, so + * level-2 debug lines stay out of persisted task output). Lines without a + * level count as level 1. Pass `{ maxLevel: 2 }` for everything. */ - getLogs(): LogLineEval[] { - return this.logs || []; + getLogs(options: { maxLevel?: number } = {}): LogLineEval[] { + const maxLevel = options.maxLevel ?? 1; + return (this.logs || []).filter((line) => (line.level ?? 1) <= maxLevel); } /** diff --git a/packages/evals/pricing/pricing.json b/packages/evals/pricing/pricing.json new file mode 100644 index 000000000..58bffb4b2 --- /dev/null +++ b/packages/evals/pricing/pricing.json @@ -0,0 +1,180 @@ +{ + "as_of": "2026-08-31", + "models": { + "openai/gpt-5.4-mini": { + "input_per_m": 0.75, + "cached_input_per_m": 0.075, + "output_per_m": 4.5, + "source": "https://ai-gateway.vercel.sh/v1/models (openai/gpt-5.4-mini)" + }, + "openai/gpt-5.5": { + "input_per_m": 5, + "cached_input_per_m": 0.5, + "output_per_m": 30, + "source": "https://ai-gateway.vercel.sh/v1/models (openai/gpt-5.5)" + }, + "anthropic/claude-sonnet-4.6": { + "input_per_m": 3, + "cached_input_per_m": 0.3, + "cache_write_input_per_m": 3.75, + "output_per_m": 15, + "source": "https://ai-gateway.vercel.sh/v1/models (anthropic/claude-sonnet-4.6)" + }, + "anthropic/claude-opus-4.8": { + "input_per_m": 5, + "cached_input_per_m": 0.5, + "cache_write_input_per_m": 6.25, + "output_per_m": 25, + "source": "https://ai-gateway.vercel.sh/v1/models (anthropic/claude-opus-4.8)" + }, + "anthropic/claude-sonnet-5": { + "input_per_m": 2, + "cached_input_per_m": 0.2, + "cache_write_input_per_m": 2.5, + "output_per_m": 10, + "source": "https://ai-gateway.vercel.sh/v1/models (anthropic/claude-sonnet-5)" + }, + "anthropic/claude-opus-5": { + "input_per_m": 5, + "cached_input_per_m": 0.5, + "cache_write_input_per_m": 6.25, + "output_per_m": 25, + "source": "https://ai-gateway.vercel.sh/v1/models (anthropic/claude-opus-5)" + }, + "google/gemini-3-flash": { + "input_per_m": 0.5, + "cached_input_per_m": 0.05, + "output_per_m": 3, + "source": "https://ai-gateway.vercel.sh/v1/models (google/gemini-3-flash)" + }, + "google/gemini-3.5-flash": { + "input_per_m": 1.5, + "cached_input_per_m": 0.15, + "output_per_m": 9, + "source": "https://ai-gateway.vercel.sh/v1/models (google/gemini-3.5-flash)" + }, + "google/gemini-3.6-flash": { + "input_per_m": 0.75, + "cached_input_per_m": 0.075, + "output_per_m": 3.75, + "source": "https://ai-gateway.vercel.sh/v1/models (google/gemini-3.6-flash)" + }, + "google/gemini-3.7-flash": { + "input_per_m": 0.75, + "cached_input_per_m": 0.075, + "output_per_m": 3.75, + "source": "https://ai-gateway.vercel.sh/v1/models (google/gemini-3.7-flash)" + }, + "spacexai/grok-4.5": { + "input_per_m": 2, + "cached_input_per_m": 0.3, + "output_per_m": 6, + "source": "https://ai-gateway.vercel.sh/v1/models (spacexai/grok-4.5)" + }, + "spacexai/grok-4.6": { + "input_per_m": 2, + "cached_input_per_m": 0.5, + "output_per_m": 6, + "source": "https://ai-gateway.vercel.sh/v1/models (spacexai/grok-4.6)" + }, + "zai/glm-5.3": { + "input_per_m": 1.4, + "cached_input_per_m": 0.14, + "output_per_m": 4.4, + "source": "https://ai-gateway.vercel.sh/v1/models (zai/glm-5.3)" + }, + "zai/glm-5.3-flash": { + "input_per_m": 0.15, + "cached_input_per_m": 0.03, + "output_per_m": 0.5, + "source": "https://ai-gateway.vercel.sh/v1/models (zai/glm-5.3-flash)" + }, + "alibaba/qwen3.8-flash": { + "input_per_m": 0.16, + "cached_input_per_m": 0.016, + "cache_write_input_per_m": 0.2, + "output_per_m": 0.47, + "source": "https://ai-gateway.vercel.sh/v1/models (alibaba/qwen3.8-flash)" + }, + "deepseek/deepseek-v4-pro": { + "input_per_m": 0.66, + "cached_input_per_m": 0.022, + "output_per_m": 1.98, + "source": "https://ai-gateway.vercel.sh/v1/models (deepseek/deepseek-v4-pro)" + }, + "deepseek/deepseek-v4-flash": { + "input_per_m": 0.13, + "cached_input_per_m": 0.028, + "output_per_m": 0.26, + "source": "https://ai-gateway.vercel.sh/v1/models (deepseek/deepseek-v4-flash)" + }, + "openai/gpt-5.6-luna": { + "input_per_m": 0.2, + "cached_input_per_m": 0.02, + "output_per_m": 1.2, + "source": "https://developers.openai.com/api/docs/pricing (fetched 2026-09-02)", + "note": "public list price; matches gateway listing of 2026-08-31" + }, + "openai/gpt-5.6-sol": { + "input_per_m": 4.0, + "cached_input_per_m": 0.4, + "output_per_m": 20.0, + "source": "https://developers.openai.com/api/docs/pricing (fetched 2026-09-02)", + "note": "public list price; gateway listed out=10 on 2026-08-31, official page says 20" + }, + "openai/gpt-5.6-terra": { + "input_per_m": 2.0, + "cached_input_per_m": 0.2, + "output_per_m": 12.0, + "source": "https://developers.openai.com/api/docs/pricing (fetched 2026-09-02)", + "note": "public list price; matches gateway listing of 2026-08-31" + }, + "anthropic/claude-fable-5": { + "input_per_m": 10.0, + "cached_input_per_m": 1.0, + "output_per_m": 50.0, + "source": "https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-09-02)", + "note": "public list price; 5m cache write 12.50, 1h 20", + "cache_write_input_per_m": 12.5 + }, + "anthropic/claude-fable-5-1": { + "input_per_m": 10.0, + "cached_input_per_m": 0.25, + "output_per_m": 50.0, + "source": "https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-09-02)", + "note": "public list price; cache hits at 0.025x base input on Fable 5.1 (other models 0.1x); 5m cache write 12.50, 1h 20", + "cache_write_input_per_m": 12.5 + }, + "google/gemini-3.8-flash": { + "input_per_m": 0.75, + "cached_input_per_m": 0.075, + "output_per_m": 3.75, + "source": "https://ai.google.dev/gemini-api/docs/pricing (Gemini 3.8 Flash, promotional rate through 2026-12-31; output includes thinking tokens)" + }, + "zai/glm-5.3-promo-50": { + "input_per_m": 0.7, + "output_per_m": 2.2, + "source": "https://ai-gateway.vercel.sh/v1/models (zai/glm-5.3-promo-50)", + "cached_input_per_m": 0.13 + }, + "alibaba/qwen3.8-max-0902": { + "input_per_m": 2.0, + "output_per_m": 6.0, + "source": "https://ai-gateway.vercel.sh/v1/models (alibaba/qwen3.8-max-0902)", + "cached_input_per_m": 0.25 + }, + "meta/muse-spark-1.3": { + "input_per_m": 1.25, + "output_per_m": 4.25, + "source": "https://ai-gateway.vercel.sh/v1/models (meta/muse-spark-1.3)", + "cached_input_per_m": 0.15 + }, + "openai/gpt-6-astra": { + "input_per_m": null, + "cached_input_per_m": null, + "output_per_m": null, + "source": "needs owner input", + "note": "Astra context-length tiers require per-request accounting; aggregate run usage cannot select the correct rate. Observed base rates were input=10 cached=1 output=50 USD/M. Estimates stay unavailable until tier-aware accounting is implemented." + } + } +} diff --git a/packages/evals/scripts/update-pricing.ts b/packages/evals/scripts/update-pricing.ts new file mode 100644 index 000000000..a173ef1e4 --- /dev/null +++ b/packages/evals/scripts/update-pricing.ts @@ -0,0 +1,193 @@ +/** + * Refresh the model entries already present in the versioned price map into pricing/pricing.json. + * + * Prereqs: network access. No API key: the Vercel AI Gateway model list is + * public; OpenRouter's public models endpoint is the fallback. + * Args: none. + * Env: EVAL_PRICING_SOURCE=gateway|openrouter to force one source. + * Example: pnpm exec tsx packages/evals/scripts/update-pricing.ts + * + * Entries already marked unpriced remain at null prices and + * "source": "needs owner input" so cost estimation reports them as unpriced + * instead of $0 — even when a public catalog happens to list a price for the + * id (the observed value is kept in `note` for the owner to confirm). + */ +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { EvalsError } from "../errors.js"; +import { getPackageRootDir } from "../runtimePaths.js"; +import type { ModelPrice, PriceMap } from "../framework/costEstimate.js"; + +const GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1/models"; +const OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"; + +/** OpenRouter spells a few creators differently from the gateway. */ +const OPENROUTER_ID_ALIASES: Record = { + "spacexai/": "x-ai/", + "zai/": "z-ai/", + "alibaba/": "qwen/", +}; + +type FetchedPrice = Omit & { source: string }; + +async function fetchCatalog( + url: string, + provider: string, +): Promise<{ data?: Array> }> { + let response: Response; + try { + response = await fetch(url); + } catch (cause) { + throw new EvalsError(`${provider} model catalog request failed.`, { cause }); + } + if (!response.ok) throw new EvalsError(`${provider} models: HTTP ${response.status}`); + try { + const body: unknown = await response.json(); + if (!body || typeof body !== "object" || !Array.isArray((body as { data?: unknown }).data)) { + throw new EvalsError(`${provider} model catalog has no model list.`); + } + return body as { data: Array> }; + } catch (cause) { + if (cause instanceof EvalsError) throw cause; + throw new EvalsError(`${provider} model catalog is not valid JSON.`, { cause }); + } +} + +async function fetchGatewayPrices(): Promise> { + const body = await fetchCatalog(GATEWAY_MODELS_URL, "gateway"); + const prices = new Map(); + for (const model of Array.isArray(body.data) ? body.data : []) { + if (!model || typeof model !== "object") continue; + const pricing = model.pricing as Record | undefined; + const id = typeof model.id === "string" ? model.id : undefined; + if (!id || !pricing) continue; + const input = perMillion(pricing.input); + const output = perMillion(pricing.output); + if (input === undefined || output === undefined) continue; + prices.set(id, { + input_per_m: input, + cached_input_per_m: perMillion(pricing.input_cache_read) ?? input, + ...(perMillion(pricing.input_cache_write) !== undefined && { + cache_write_input_per_m: perMillion(pricing.input_cache_write), + }), + output_per_m: output, + source: `${GATEWAY_MODELS_URL} (${id})`, + }); + } + if (prices.size === 0) throw new EvalsError("Model catalog contained no usable prices."); + return prices; +} + +async function fetchOpenRouterPrices(): Promise> { + const body = await fetchCatalog(OPENROUTER_MODELS_URL, "openrouter"); + const prices = new Map(); + for (const model of Array.isArray(body.data) ? body.data : []) { + if (!model || typeof model !== "object") continue; + const pricing = model.pricing as Record | undefined; + const id = typeof model.id === "string" ? model.id : undefined; + if (!id || !pricing) continue; + const input = perMillion(pricing.prompt); + const output = perMillion(pricing.completion); + if (input === undefined || output === undefined) continue; + prices.set(toGatewayId(id), { + input_per_m: input, + cached_input_per_m: perMillion(pricing.input_cache_read) ?? input, + ...(perMillion(pricing.input_cache_write) !== undefined && { + cache_write_input_per_m: perMillion(pricing.input_cache_write), + }), + output_per_m: output, + source: `${OPENROUTER_MODELS_URL} (${id})`, + }); + } + if (prices.size === 0) throw new EvalsError("Model catalog contained no usable prices."); + return prices; +} + +function toGatewayId(openRouterId: string): string { + for (const [gateway, openRouter] of Object.entries(OPENROUTER_ID_ALIASES)) { + if (openRouterId.startsWith(openRouter)) return gateway + openRouterId.slice(openRouter.length); + } + return openRouterId; +} + +/** Catalogs quote USD per token as strings; the price map stores USD per million. */ +function perMillion(value: unknown): number | undefined { + if (typeof value !== "number" && (typeof value !== "string" || !value.trim())) return undefined; + const perToken = Number(value); + if (!Number.isFinite(perToken) || perToken < 0) return undefined; + return Number((perToken * 1_000_000).toPrecision(10)); +} + +async function loadPrices(): Promise<{ prices: Map; source: string }> { + const forced = process.env.EVAL_PRICING_SOURCE; + if (forced !== "openrouter") { + try { + return { prices: await fetchGatewayPrices(), source: "gateway" }; + } catch (error) { + if (forced === "gateway") throw error; + console.warn("gateway model catalog unavailable; falling back to OpenRouter"); + } + } + return { prices: await fetchOpenRouterPrices(), source: "openrouter" }; +} + +export async function updatePricing( + target = path.join(getPackageRootDir(), "pricing", "pricing.json"), +): Promise { + const existing = JSON.parse(fs.readFileSync(target, "utf8")) as PriceMap; + const pricedModels = Object.keys(existing.models).filter( + (id) => existing.models[id].input_per_m !== null, + ); + const ownerInputModels = Object.keys(existing.models).filter( + (id) => existing.models[id].input_per_m === null, + ); + const { prices, source } = await loadPrices(); + if (pricedModels.length && !pricedModels.some((id) => prices.has(id))) { + throw new EvalsError( + "Model catalog contained no prices for currently priced models; file unchanged.", + ); + } + const models: PriceMap["models"] = {}; + const missing: string[] = []; + for (const id of pricedModels) { + const price = prices.get(id); + if (price) models[id] = price; + else { + missing.push(id); + models[id] = { + input_per_m: null, + cached_input_per_m: null, + output_per_m: null, + source: `not listed by ${source} on ${today()}; needs owner input`, + }; + } + } + for (const id of ownerInputModels) { + const observed = prices.get(id); + models[id] = { + input_per_m: null, + cached_input_per_m: null, + output_per_m: null, + source: "needs owner input", + ...(observed && { + note: `${source} lists in=${observed.input_per_m} cached=${observed.cached_input_per_m}${observed.cache_write_input_per_m !== undefined ? ` cache_write=${observed.cache_write_input_per_m}` : ""} out=${observed.output_per_m} USD/M on ${today()}; confirm before enabling this price`, + }), + }; + } + const priceMap: PriceMap = { as_of: today(), models }; + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, JSON.stringify(priceMap, null, 2) + "\n"); + console.log( + `wrote ${target}: ${pricedModels.length - missing.length} priced from ${source}, ${ownerInputModels.length} awaiting owner input` + + (missing.length ? `, ${missing.length} not listed: ${missing.join(", ")}` : ""), + ); +} + +function today(): string { + return new Date().toISOString().slice(0, 10); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + await updatePricing(); +} diff --git a/packages/evals/tests/cli.test.ts b/packages/evals/tests/cli.test.ts index 7aca4500c..fb106d7bb 100644 --- a/packages/evals/tests/cli.test.ts +++ b/packages/evals/tests/cli.test.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll, onTestFinished } from "vitest"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import fs from "node:fs"; @@ -9,6 +9,7 @@ const exec = promisify(execFile); const repoRoot = path.resolve(__dirname, "..", "..", ".."); const CLI_PATH = path.join(repoRoot, "packages", "evals", "cli.ts"); const SOURCE_CONFIG = path.join(repoRoot, "packages", "evals", "evals.config.json"); +const CLI_CHILD_TIMEOUT_MS = 15_000; // File-level snapshot/restore: any `evals run …` invocation through the // real CLI writes `_meta.firstRunCompletedAt` into the source config @@ -24,15 +25,20 @@ afterAll(() => { async function runCli(args: string[]): Promise<{ stdout: string; stderr: string; code: number }> { try { - const { stdout, stderr } = await exec( - process.execPath, - ["--import", "tsx", CLI_PATH, ...args], - { - cwd: repoRoot, - timeout: 15_000, - env: { ...process.env, NODE_NO_WARNINGS: "1" }, - }, - ); + const execution = exec(process.execPath, ["--import", "tsx", CLI_PATH, ...args], { + cwd: repoRoot, + timeout: CLI_CHILD_TIMEOUT_MS, + killSignal: "SIGKILL", + env: { ...process.env, NODE_NO_WARNINGS: "1" }, + }); + // A test timeout must also stop its own CLI child. SIGTERM enters the + // CLI's async cleanup path, which is not a bounded subprocess deadline. + onTestFinished(() => { + if (execution.child.exitCode === null && execution.child.signalCode === null) { + execution.child.kill("SIGKILL"); + } + }); + const { stdout, stderr } = await execution; return { stdout, stderr, code: 0 }; } catch (err: any) { return { @@ -55,15 +61,19 @@ function readSourceWelcomeCompletedAt(): string | undefined { } describe("CLI entrypoint", () => { - it("shows help", async () => { - const { stdout, code } = await runCli(["-h"]); - expect(code).toBe(0); - expect(stdout).toContain("Commands:"); - expect(stdout).toContain("run"); - expect(stdout).toContain("list"); - expect(stdout).toContain("config"); - expect(stdout).toContain("experiments"); - }); + it( + "shows help", + async () => { + const { stdout, code } = await runCli(["-h"]); + expect(code).toBe(0); + expect(stdout).toContain("Commands:"); + expect(stdout).toContain("run"); + expect(stdout).toContain("list"); + expect(stdout).toContain("config"); + expect(stdout).toContain("experiments"); + }, + CLI_CHILD_TIMEOUT_MS + 2_000, + ); it("shows experiments overview help", async () => { const { stdout, code } = await runCli(["experiments"]); diff --git a/packages/evals/tests/core/browserSessionLossTelemetry.test.ts b/packages/evals/tests/core/browserSessionLossTelemetry.test.ts new file mode 100644 index 000000000..6f36ec71c --- /dev/null +++ b/packages/evals/tests/core/browserSessionLossTelemetry.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { + parseSessionLossTelemetry, + SESSION_LOST_TELEMETRY_PREFIX, +} from "../../core/tools/browserSessionLoss.js"; + +function line(fields: Record) { + return ( + SESSION_LOST_TELEMETRY_PREFIX + JSON.stringify({ cause: "CDP connection closed", ...fields }) + ); +} + +describe("facade session loss diagnostics", () => { + it("retains browser identity, measured age and configured timeout", () => { + expect( + parseSessionLossTelemetry( + line({ + tool: "snapshot", + at: "2026-09-08T00:00:00.000Z", + provider: "browserbase", + sessionId: "session-123", + sessionAgeMs: 12_500, + sessionTimeoutMs: 3_600_000, + }), + ), + ).toEqual({ + cause: "CDP connection closed", + tool: "snapshot", + at: "2026-09-08T00:00:00.000Z", + provider: "browserbase", + sessionId: "session-123", + sessionAgeMs: 12_500, + sessionTimeoutMs: 3_600_000, + }); + }); + + it.each([-1, "12000", null, {}, 1e309])("drops invalid diagnostic durations: %j", (value) => { + expect( + parseSessionLossTelemetry(line({ sessionAgeMs: value, sessionTimeoutMs: value })), + ).toEqual({ cause: "CDP connection closed" }); + }); + + it("accepts zero age and omits invalid identity metadata", () => { + expect( + parseSessionLossTelemetry(line({ provider: "other", sessionId: 42, sessionAgeMs: 0 })), + ).toEqual({ cause: "CDP connection closed", sessionAgeMs: 0 }); + }); + + it("drops JSON numeric overflow without losing the terminal cause", () => { + expect( + parseSessionLossTelemetry( + SESSION_LOST_TELEMETRY_PREFIX + + '{"cause":"CDP connection closed","sessionAgeMs":1e309,"sessionTimeoutMs":1e309}', + ), + ).toEqual({ cause: "CDP connection closed" }); + }); + + it("sanitizes string diagnostics while retaining valid numeric metadata", () => { + const parsed = parseSessionLossTelemetry( + line({ + provider: "local", + sessionId: "wss://example.test/?apiKey=synthetic-key", + sessionAgeMs: 1, + }), + ); + expect(parsed?.provider).toBe("local"); + expect(parsed?.sessionAgeMs).toBe(1); + expect(JSON.stringify(parsed)).not.toContain("synthetic-key"); + }); +}); diff --git a/packages/evals/tests/framework/benchHarness.test.ts b/packages/evals/tests/framework/benchHarness.test.ts index 7c0b2e968..28ab356f2 100644 --- a/packages/evals/tests/framework/benchHarness.test.ts +++ b/packages/evals/tests/framework/benchHarness.test.ts @@ -227,6 +227,11 @@ describe("bench harness registry", () => { it("defines the shared external lifecycle and cleans up when the agent throws", async () => { let cleanupCalled = false; const adapter = { + browserSession: { + provider: "browserbase" as const, + sessionId: "session-a", + sessionUrl: "https://www.browserbase.com/sessions/session-a", + }, cleanup: async (): Promise => { cleanupCalled = true; }, @@ -289,7 +294,14 @@ describe("bench harness registry", () => { expect(harness.supportsApi).toBe(false); await expect( harness.execute?.({ task, input, row, logger: new EvalLogger(false) }), - ).rejects.toThrow("agent failed"); + ).resolves.toMatchObject({ + _success: false, + error: "agent failed", + harnessStatus: "sdk_error", + browserProvider: "browserbase", + browserbaseSessionId: "session-a", + sessionUrl: "https://www.browserbase.com/sessions/session-a", + }); expect(preparedInput).toMatchObject({ toolSurface: "browse_cli", startupProfile: "tool_create_browserbase", @@ -357,6 +369,140 @@ describe("bench harness registry", () => { } }); + it("logs the browser session as the first task log line and stamps the result", async () => { + const close = vi.spyOn(V3.prototype, "close").mockResolvedValue(undefined); + const logger = new EvalLogger(false); + let agentSawSessionLine = false; + const harness = defineExternalHarness({ + harness: "session_first_external", + supportedToolSurfaces: ["stagehand_facade"], + defaultModels: ["openai/x" as AvailableModel], + prepareToolAdapter: async (input) => { + input.logger.log({ category: "setup", message: "bridge started", level: 2 }); + return { + browserSession: { + provider: "browserbase" as const, + sessionId: "sess-1", + sessionUrl: "https://www.browserbase.com/sessions/sess-1", + }, + cleanup: async () => {}, + }; + }, + runAgent: async (input) => { + agentSawSessionLine = input.logger + .getLogs() + .some( + (line) => + line.message === "Browserbase session: https://www.browserbase.com/sessions/sess-1", + ); + input.logger.log({ category: "agent", message: "step 1 · run · ok", level: 1 }); + return { _success: true, logs: input.logger.getLogs() }; + }, + }); + const input: EvalInput = { + name: "agent/webvoyager", + modelName: "openai/x" as AvailableModel, + params: { id: "wv-1", web: "https://example.com", ques: "Find it" }, + }; + const task: DiscoveredTask = { + name: input.name, + tier: "bench", + primaryCategory: "agent", + categories: ["agent"], + tags: [], + filePath: "/tmp/fake.ts", + isLegacy: false, + }; + const row: BenchMatrixRow = { + harness: "session_first_external", + task: input.name, + category: "agent", + taskKind: "agent", + model: input.modelName, + environment: "BROWSERBASE", + useApi: false, + toolSurface: "stagehand_facade", + startupProfile: "tool_create_browserbase", + trial: 1, + config: { + harness: "session_first_external", + model: input.modelName, + environment: "BROWSERBASE", + useApi: false, + toolSurface: "stagehand_facade", + startupProfile: "tool_create_browserbase", + }, + }; + + try { + const result = await harness.execute!({ task, input, row, logger }); + expect(agentSawSessionLine).toBe(true); + // Level-2 setup chatter is filtered out, so the session pointer heads the row logs. + expect((result.logs ?? []).map((line) => line.message)).toEqual([ + "Browserbase session: https://www.browserbase.com/sessions/sess-1", + "step 1 · run · ok", + ]); + expect(result.logs?.[0]).toMatchObject({ category: "session", level: 0 }); + expect(result).toMatchObject({ + sessionUrl: "https://www.browserbase.com/sessions/sess-1", + browserbaseSessionId: "sess-1", + browserProvider: "browserbase", + }); + } finally { + close.mockRestore(); + } + }); + + it("logs a bare provider line when the adapter reports no session", async () => { + const close = vi.spyOn(V3.prototype, "close").mockResolvedValue(undefined); + const logger = new EvalLogger(false); + const harness = defineExternalHarness({ + harness: "session_fallback_external", + supportedToolSurfaces: ["browse_cli"], + defaultModels: ["openai/x" as AvailableModel], + prepareToolAdapter: async () => ({ cleanup: async () => {} }), + runAgent: async () => ({ _success: true }), + }); + const input: EvalInput = { + name: "agent/webvoyager", + modelName: "openai/x" as AvailableModel, + params: { id: "wv-1", web: "https://example.com", ques: "Find it" }, + }; + const task: DiscoveredTask = { + name: input.name, + tier: "bench", + primaryCategory: "agent", + categories: ["agent"], + tags: [], + filePath: "/tmp/fake.ts", + isLegacy: false, + }; + const row: BenchMatrixRow = { + harness: "session_fallback_external", + task: input.name, + category: "agent", + taskKind: "agent", + model: input.modelName, + environment: "LOCAL", + useApi: false, + trial: 1, + config: { + harness: "session_fallback_external", + model: input.modelName, + environment: "LOCAL", + useApi: false, + }, + }; + try { + const result = await harness.execute!({ task, input, row, logger }); + expect(logger.getLogs().map((line) => line.message)).toEqual(["Browser: local"]); + expect(result.browserProvider).toBe("local"); + expect(result.sessionUrl).toBeUndefined(); + } finally { + close.mockRestore(); + } + }); + it("rejects mismatched external harness config before preparing an adapter", async () => { let prepareCalled = false; const harness = defineExternalHarness({ diff --git a/packages/evals/tests/framework/benchPlanner.test.ts b/packages/evals/tests/framework/benchPlanner.test.ts index 33b9b022f..b3f5f34ea 100644 --- a/packages/evals/tests/framework/benchPlanner.test.ts +++ b/packages/evals/tests/framework/benchPlanner.test.ts @@ -212,12 +212,12 @@ describe("benchPlanner", () => { expect(testcases[0].input.isCUA).toBeUndefined(); expect(testcases[0].tags).toContain("harness/claude_code"); expect(testcases[0].metadata.harness).toBe("claude_code"); - expect(testcases[0].metadata.toolSurface).toBe("browse_cli"); + expect(testcases[0].metadata.toolSurface).toBe("stagehand_facade"); expect(testcases[0].metadata.startupProfile).toBe("tool_launch_local"); expect(testcases[0].metadata.agentMode).toBeUndefined(); }); - it("keeps codex as a harness-level matrix with browse_cli metadata", async () => { + it("keeps codex as a harness-level matrix with facade metadata", async () => { const testcases = await withEnvOverrides( { EVAL_MAX_K: "1", @@ -237,9 +237,9 @@ describe("benchPlanner", () => { expect(testcases[0].input.isCUA).toBeUndefined(); expect(testcases[0].tags).toContain("harness/codex"); expect(testcases[0].metadata.harness).toBe("codex"); - expect(testcases[0].metadata.toolSurface).toBe("browse_cli"); + expect(testcases[0].metadata.toolSurface).toBe("stagehand_facade"); expect(testcases[0].metadata.startupProfile).toBe("tool_launch_local"); - expect(testcases[0].metadata.toolCommand).toBe("browse"); + expect(testcases[0].metadata.toolCommand).toBeUndefined(); expect(testcases[0].metadata.agentMode).toBeUndefined(); }); diff --git a/packages/evals/tests/framework/browserSession.test.ts b/packages/evals/tests/framework/browserSession.test.ts new file mode 100644 index 000000000..5cbfbee81 --- /dev/null +++ b/packages/evals/tests/framework/browserSession.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { + browserSessionFromMetadata, + buildBrowserSessionLogLines, + withBrowserSession, +} from "../../framework/browserSession.js"; + +describe("browser session info", () => { + it("reads runner-provided target metadata", () => { + expect( + browserSessionFromMetadata( + { + browserbaseSessionId: "abc-123", + browserbaseSessionUrl: "https://www.browserbase.com/sessions/abc-123", + browserbaseDebugUrl: "https://debug.example/abc-123", + }, + "BROWSERBASE", + ), + ).toEqual({ + provider: "browserbase", + sessionId: "abc-123", + sessionUrl: "https://www.browserbase.com/sessions/abc-123", + debugUrl: "https://debug.example/abc-123", + }); + }); + + it("derives the missing half of id/url from the other", () => { + expect(browserSessionFromMetadata({ browserbaseSessionId: "abc" }, "BROWSERBASE")).toEqual({ + provider: "browserbase", + sessionId: "abc", + sessionUrl: "https://www.browserbase.com/sessions/abc", + }); + expect( + browserSessionFromMetadata( + { browserbaseSessionUrl: "https://www.browserbase.com/sessions/xyz?tab=logs" }, + "BROWSERBASE", + ), + ).toMatchObject({ sessionId: "xyz" }); + }); + + it("encodes a session identifier as one URL path segment", () => { + expect( + browserSessionFromMetadata({ browserbaseSessionId: "id/with?#spaces " }, "BROWSERBASE"), + ).toMatchObject({ + sessionUrl: "https://www.browserbase.com/sessions/id%2Fwith%3F%23spaces%20", + }); + }); + + it("falls back to the bare provider", () => { + expect(browserSessionFromMetadata({ browserbaseSessionId: "ignored" }, "LOCAL")).toEqual({ + provider: "local", + }); + expect(browserSessionFromMetadata(undefined, "BROWSERBASE")).toEqual({ + provider: "browserbase", + }); + }); + + it("formats level-0 session lines", () => { + expect(buildBrowserSessionLogLines({ provider: "local" })).toEqual([ + { + category: "session", + level: 0, + message: "Browser: local", + auxiliary: { provider: { value: "local", type: "string" } }, + }, + ]); + expect( + buildBrowserSessionLogLines({ provider: "browserbase" }).map((line) => line.message), + ).toEqual(["Browser: browserbase (session id not reported by this surface)"]); + expect( + buildBrowserSessionLogLines({ + provider: "browserbase", + sessionId: "abc", + sessionUrl: "https://www.browserbase.com/sessions/abc", + debugUrl: "https://debug.example/abc", + }).map((line) => line.message), + ).toEqual([ + "Browserbase session: https://www.browserbase.com/sessions/abc", + "Browserbase debugger: https://debug.example/abc", + ]); + }); + + it("stamps the task result without clobbering harness-reported urls", () => { + expect( + withBrowserSession( + { _success: true, sessionUrl: "https://reported.example" }, + { + provider: "browserbase", + sessionId: "abc", + sessionUrl: "https://www.browserbase.com/sessions/abc", + }, + ), + ).toEqual({ + _success: true, + browserProvider: "browserbase", + browserbaseSessionId: "abc", + sessionUrl: "https://reported.example", + }); + }); +}); diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 5acd0d939..c8fe7f7dc 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -22,8 +22,8 @@ describe("claude code tool adapter resolution", () => { delete process.env.EVAL_CLAUDE_CODE_ALLOW_UNSANDBOXED_LOCAL; }); - it("defaults Claude Code to browse_cli", () => { - expect(resolveToolSurface(claudeCodeHarness)).toBe("browse_cli"); + it("defaults Claude Code to the shared facade", () => { + expect(resolveToolSurface(claudeCodeHarness)).toBe("stagehand_facade"); }); it("defaults browse_cli startup by environment", () => { @@ -68,7 +68,7 @@ describe("claude code tool adapter resolution", () => { }); it("supports browse_cli and the code surfaces on Codex", () => { - expect(resolveToolSurface(codexHarness)).toBe("browse_cli"); + expect(resolveToolSurface(codexHarness)).toBe("stagehand_facade"); expect(resolveToolSurface(codexHarness, "browse_cli")).toBe("browse_cli"); expect(resolveToolSurface(codexHarness, "stagehand_code")).toBe("stagehand_code"); expect(resolveToolSurface(codexHarness, "playwright_code")).toBe("playwright_code"); diff --git a/packages/evals/tests/framework/costEstimate.test.ts b/packages/evals/tests/framework/costEstimate.test.ts new file mode 100644 index 000000000..be53e14a8 --- /dev/null +++ b/packages/evals/tests/framework/costEstimate.test.ts @@ -0,0 +1,471 @@ +import { describe, expect, it } from "vitest"; +import { + computeListCost, + loadPriceMap, + modelPriceCandidates, + providerOf, + resolveBilledCost, + resolveModelPrice, + type PriceMap, +} from "../../framework/costEstimate.js"; +import { normalizeUsage } from "../../framework/usageNormalization.js"; + +const priceMap: PriceMap = { + as_of: "2026-08-31", + models: { + "openai/gpt-5.4-mini": { + input_per_m: 1, + cached_input_per_m: 0.1, + output_per_m: 4, + source: "test", + }, + "anthropic/claude-sonnet-4.6": { + input_per_m: 3, + cached_input_per_m: 0.3, + cache_write_input_per_m: 3.75, + output_per_m: 15, + source: "test", + }, + "spacexai/grok-4.5": { + input_per_m: 2, + cached_input_per_m: 0.3, + output_per_m: 6, + source: "test", + }, + "google/gemini-3-flash": { + input_per_m: 0.5, + cached_input_per_m: 0.05, + output_per_m: 3, + source: "test", + }, + "anthropic/claude-fable-5": { + input_per_m: null, + cached_input_per_m: null, + output_per_m: null, + source: "needs owner input", + }, + }, +}; + +const compute = (usage: ReturnType, model: string) => + computeListCost(usage, model, priceMap); + +describe("computeListCost", () => { + it("keeps Astra estimates unavailable until per-request context tiers are represented", () => { + const usage = normalizeUsage({ + harness: "codex", + raw: { + inputTokens: 300_000, + outputTokens: 1000, + totalTokens: 301_000, + }, + }); + expect(computeListCost(usage, "openai/gpt-6-astra", loadPriceMap())).toBeUndefined(); + }); + + it("prices the OpenAI subset convention: uncached at input, cached at cache rate, reasoning inside output", () => { + const usage = normalizeUsage({ + harness: "codex", + raw: { + inputTokens: 1_000_000, + cachedInputTokens: 600_000, + outputTokens: 100_000, + reasoningOutputTokens: 40_000, + totalTokens: 1_100_000, + }, + }); + // 400k·1 + 600k·0.1 + 100k·4 = 0.4 + 0.06 + 0.4 + expect(compute(usage, "openai/gpt-5.4-mini")).toBe(0.86); + }); + + it("prices the Anthropic separate convention with cache writes at the write rate", () => { + const usage = normalizeUsage({ + harness: "claude_code", + raw: { + inputTokens: 100_000, + cachedInputTokens: 1_000_000, + cacheCreationInputTokens: 200_000, + outputTokens: 50_000, + totalTokens: 1_350_000, + }, + }); + // 100k·3 + 1M·0.3 + 200k·3.75 + 50k·15 = 0.3 + 0.3 + 0.75 + 0.75 + expect(compute(usage, "anthropic/claude-sonnet-4-6")).toBe(2.1); + }); + + it("prices pi's uncached-only input plus its separate cache buckets", () => { + const usage = normalizeUsage({ + harness: "pi", + raw: { + inputTokens: 40, + cachedInputTokens: 1_000_000, + cacheCreationInputTokens: 0, + outputTokens: 0, + totalTokens: 1_000_040, + }, + }); + expect(compute(usage, "openai/gpt-5.4-mini")).toBeCloseTo(0.10004, 6); + }); + + it("bills reasoning at the output rate only when it is reported outside output", () => { + const base = normalizeUsage({ + harness: "codex", + raw: { + inputTokens: 0, + outputTokens: 1_000_000, + reasoningOutputTokens: 500_000, + totalTokens: 0, + }, + }); + expect(compute(base, "openai/gpt-5.4-mini")).toBe(4); + expect(compute({ ...base, reasoning_in_output: false }, "openai/gpt-5.4-mini")).toBe(6); + }); + + it("rejects malformed or negative rates, including optional cache writes, and overflow", () => { + const usage = normalizeUsage({ + harness: "pi", + raw: { inputTokens: 1, cacheCreationInputTokens: 1, outputTokens: 1, totalTokens: 3 }, + }); + for (const key of [ + "input_per_m", + "cached_input_per_m", + "cache_write_input_per_m", + "output_per_m", + ] as const) { + for (const rate of [-1, NaN, Infinity, "2"]) { + const malformed = { + ...priceMap, + models: { + "openai/gpt-5.4-mini": { ...priceMap.models["openai/gpt-5.4-mini"], [key]: rate }, + }, + } as PriceMap; + expect( + computeListCost(usage, "openai/gpt-5.4-mini", malformed), + `${key}=${rate}`, + ).toBeUndefined(); + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + priceMap: malformed, + }).cost_source, + ).toBe("unavailable"); + } + } + const overflow = { ...usage, input_uncached: Number.MAX_VALUE }; + expect(computeListCost(overflow, "anthropic/claude-sonnet-4.6", priceMap)).toBeUndefined(); + }); + + it("returns nothing for null-price entries, unknown models and unreported usage", () => { + const usage = normalizeUsage({ + harness: "claude_code", + raw: { inputTokens: 10, outputTokens: 10, totalTokens: 20 }, + }); + expect(compute(usage, "anthropic/claude-fable-5")).toBeUndefined(); + expect(compute(usage, "example/unknown-model")).toBeUndefined(); + expect(computeListCost(usage, undefined, priceMap)).toBeUndefined(); + const unreported = normalizeUsage({ + harness: "cursor", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect(compute(unreported, "openai/gpt-5.4-mini")).toBeUndefined(); + }); +}); + +describe("resolveBilledCost", () => { + it.each([ + ["claude_cua", "anthropic/claude-sonnet-4.6", "anthropic_api"], + ["gemini_cua", "google/gemini-fixture", "google_api"], + ])( + "estimates native %s provider usage with dated price provenance", + (harness, model, channel) => { + const nativeMap: PriceMap = { + as_of: "2026-08-31", + models: { + [model]: { + input_per_m: 1, + cached_input_per_m: 0.1, + output_per_m: 4, + source: "fixture price source", + }, + }, + }; + const nativeUsage = normalizeUsage({ + harness, + raw: { inputTokens: 100, outputTokens: 10, totalTokens: 110, reported: true }, + }); + expect( + resolveBilledCost({ harness, model, usage: nativeUsage, priceMap: nativeMap }), + ).toMatchObject({ + cost_source: "computed", + billing_channel: channel, + cost_pricing: { as_of: "2026-08-31", model, source: "fixture price source" }, + }); + const missing = normalizeUsage({ + harness, + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect(resolveBilledCost({ harness, model, usage: missing, priceMap: nativeMap })).toEqual({ + cost_source: "unavailable", + billing_channel: channel, + }); + }, + ); + const usage = normalizeUsage({ + harness: "codex", + raw: { inputTokens: 1_000_000, outputTokens: 0, totalTokens: 1_000_000 }, + }); + + it("takes the harness-reported dollars first, naming the channel", () => { + expect( + resolveBilledCost({ + harness: "claude_code", + model: "anthropic/claude-sonnet-4-6", + usage, + reportedCostUsd: 4.2, + priceMap, + }), + ).toEqual({ cost_usd: 4.2, cost_source: "reported", billing_channel: "anthropic_api" }); + expect( + resolveBilledCost({ + harness: "eve", + model: "zai/glm-5.3", + usage, + reportedCostUsd: 0.01, + priceMap, + }).billing_channel, + ).toBe("ai_gateway"); + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: 0.5, + priceMap, + }).billing_channel, + ).toBe("pi_catalog"); + expect( + resolveBilledCost({ + harness: "fx", + model: "zai/glm-5.3", + usage, + reportedCostUsd: 0.02, + priceMap, + }).billing_channel, + ).toBe("fx_gateway"); + // A reported figure wins even for a priced model on a direct-API harness. + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: 1.3, + priceMap, + }), + ).toMatchObject({ cost_usd: 1.3, cost_source: "reported" }); + }); + + it("computes direct-provider harnesses at list price when nothing was reported", () => { + for (const harness of ["codex", "mastra", "deepagents", "eve", "pi"]) { + expect( + resolveBilledCost({ harness, model: "openai/gpt-5.4-mini", usage, priceMap }), + harness, + ).toEqual({ + cost_usd: 1, + cost_source: "computed", + billing_channel: "openai_api", + cost_pricing: { as_of: "2026-08-31", model: "openai/gpt-5.4-mini", source: "test" }, + }); + } + expect( + resolveBilledCost({ harness: "codex", model: "codex/default", usage, priceMap }) + .billing_channel, + ).toBe("openai_api"); + expect( + resolveBilledCost({ harness: "mastra", model: "xai/grok-4.5", usage, priceMap }), + ).toMatchObject({ + cost_usd: 2, + cost_source: "computed", + billing_channel: "xai_api", + }); + expect( + resolveBilledCost({ + harness: "deepagents", + model: "anthropic/claude-sonnet-4-6", + usage, + priceMap, + }).billing_channel, + ).toBe("anthropic_api"); + }); + + it("is unavailable, never zero, for subscription cells, unpriced models and unreported usage", () => { + expect( + resolveBilledCost({ harness: "cursor", model: "openai/gpt-5.4-mini", usage, priceMap }), + ).toEqual({ cost_source: "unavailable", billing_channel: "subscription" }); + expect( + resolveBilledCost({ + harness: "claude_code", + model: "anthropic/claude-sonnet-4-6", + usage, + priceMap, + }), + ).toEqual({ cost_source: "unavailable", billing_channel: "subscription" }); + expect(resolveBilledCost({ harness: "fx", model: "zai/glm-5.3", usage, priceMap })).toEqual({ + cost_source: "unavailable", + billing_channel: "fx_gateway", + }); + expect( + resolveBilledCost({ harness: "codex", model: "openai/gpt-5.6-luna", usage, priceMap }), + ).toEqual({ cost_source: "unavailable", billing_channel: "openai_api" }); + const unreported = normalizeUsage({ + harness: "codex", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect( + resolveBilledCost({ + harness: "codex", + model: "openai/gpt-5.4-mini", + usage: unreported, + priceMap, + }), + ).toEqual({ cost_source: "unavailable", billing_channel: "openai_api" }); + // A non-finite report is no report. + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: Number.NaN, + priceMap, + }).cost_source, + ).toBe("computed"); + }); + + it("retains the dated matched model/source for aliases and omits pricing on reported or unavailable cost", () => { + const computed = resolveBilledCost({ + harness: "mastra", + model: "xai/grok-4.5", + usage, + priceMap, + }); + expect(computed.cost_pricing).toEqual({ + as_of: "2026-08-31", + model: "spacexai/grok-4.5", + source: "test", + }); + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: 0, + priceMap, + }), + ).toEqual({ cost_usd: 0, cost_source: "reported", billing_channel: "pi_catalog" }); + expect( + resolveBilledCost({ harness: "cursor", model: "openai/gpt-5.4-mini", usage, priceMap }), + ).not.toHaveProperty("cost_pricing"); + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: -1, + priceMap, + }).cost_source, + ).toBe("computed"); + }); + + it.each(["eve", "mastra", "pi"])( + "keeps missing %s usage unavailable and explicit zero usage priceable", + (harness) => { + const empty = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; + const unknown = normalizeUsage({ harness, raw: empty }); + expect( + resolveBilledCost({ harness, model: "openai/gpt-5.4-mini", usage: unknown, priceMap }), + ).toEqual({ cost_source: "unavailable", billing_channel: "openai_api" }); + const reportedZero = normalizeUsage({ harness, raw: { ...empty, reported: true } }); + expect( + resolveBilledCost({ harness, model: "openai/gpt-5.4-mini", usage: reportedZero, priceMap }), + ).toMatchObject({ cost_source: "computed", cost_usd: 0 }); + }, + ); + + it("derives the provider from the configured id", () => { + expect(providerOf("gateway/openai/gpt-5.4-mini")).toBe("openai"); + expect(providerOf("codex/default")).toBe("openai"); + expect(providerOf("xai/grok-4.5")).toBe("xai"); + expect(providerOf("gpt-5.4-mini")).toBeUndefined(); + expect(providerOf(undefined)).toBeUndefined(); + }); +}); + +describe("model alias resolution", () => { + it.each([ + ["gateway/openai/gpt-5.4-mini", "openai/gpt-5.4-mini"], + ["gateway/gpt-5.4-mini", "openai/gpt-5.4-mini"], + ["codex/default", "openai/gpt-5.4-mini"], + ["gpt-5.4-mini", "openai/gpt-5.4-mini"], + ["anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-4.6"], + ["anthropic/claude-sonnet-4.6-20260101", "anthropic/claude-sonnet-4.6"], + ["xai/grok-4.5", "spacexai/grok-4.5"], + ["grok-4.5", "spacexai/grok-4.5"], + ["google/gemini-3-flash-preview", "google/gemini-3-flash"], + ])("resolves %s to %s", (model, expected) => { + expect(resolveModelPrice(model, priceMap)?.key).toBe(expected); + }); + + it("never borrows a sibling model's price", () => { + expect(resolveModelPrice("openai/gpt-5.4", priceMap)).toBeUndefined(); + expect(resolveModelPrice("anthropic/claude-sonnet-4", priceMap)).toBeUndefined(); + expect(resolveModelPrice("fx/default", priceMap)).toBeUndefined(); + }); + + it("does not match a bare name carried by several providers", () => { + const ambiguous: PriceMap = { + as_of: "x", + models: { + "a/model-1": { input_per_m: 1, cached_input_per_m: 1, output_per_m: 1, source: "t" }, + "b/model-1": { input_per_m: 2, cached_input_per_m: 2, output_per_m: 2, source: "t" }, + }, + }; + expect(resolveModelPrice("model-1", ambiguous)).toBeUndefined(); + expect(resolveModelPrice("b/model-1", ambiguous)?.key).toBe("b/model-1"); + }); + + it("orders candidates from the exact id to the bare name", () => { + expect(modelPriceCandidates("gateway/xai/grok-4-5")).toEqual([ + "xai/grok-4-5", + "spacexai/grok-4-5", + "x-ai/grok-4-5", + "xai/grok-4.5", + "spacexai/grok-4.5", + "x-ai/grok-4.5", + "grok-4-5", + "grok-4.5", + ]); + }); +}); + +describe("shipped price map", () => { + it("prices the curated set", () => { + const shipped = loadPriceMap(); + expect(shipped.as_of).toMatch(/^\d{4}-\d{2}-\d{2}$/u); + expect(resolveModelPrice("openai/gpt-5.4-mini", shipped)).toBeDefined(); + expect(resolveModelPrice("anthropic/claude-sonnet-4-6", shipped)).toBeDefined(); + expect(resolveModelPrice("xai/grok-4.5", shipped)).toBeDefined(); + // gpt-5.6 luna/terra/sol are on the public OpenAI price list (2026-09-02). + for (const model of ["openai/gpt-5.6-luna", "openai/gpt-5.6-terra", "openai/gpt-5.6-sol"]) { + expect(shipped.models[model]?.source, model).toMatch(/developers\.openai\.com/u); + expect(resolveModelPrice(model, shipped), model).toBeDefined(); + } + // Fable 5.1 cache hits are 0.025x base input (other Anthropic models 0.1x). + expect(shipped.models["anthropic/claude-fable-5-1"]).toMatchObject({ + input_per_m: 10, + cached_input_per_m: 0.25, + output_per_m: 50, + cache_write_input_per_m: 12.5, + }); + }); +}); diff --git a/packages/evals/tests/framework/evalSystemPrompt.test.ts b/packages/evals/tests/framework/evalSystemPrompt.test.ts new file mode 100644 index 000000000..b60567b07 --- /dev/null +++ b/packages/evals/tests/framework/evalSystemPrompt.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { EvalLogger } from "../../logger.js"; +import { EVAL_SYSTEM_PROMPT } from "../../framework/evalSystemPrompt.js"; +import { runExternalHarnessTask } from "../../framework/harnesses/externalRunner.js"; + +describe("shared evaluation policy", () => { + it.each(["native", "task_prefix"] as const)("dispatches policy once via %s", async (mode) => { + let request = ""; + const result = await runExternalHarnessTask({ + harness: "example", + implementation: { name: "sdk", version: 1, sdkVersion: "1.0.0" }, + plan: { + dataset: "webvoyager", + taskId: "fixture", + instruction: "Return the exact page title.", + startUrl: "https://example.test", + }, + logger: new EvalLogger(false), + resultContract: "marker", + fallbackErrorMessage: "missing result", + systemPromptMode: mode, + runSession: async (prompt, systemPrompt) => { + request = `${systemPrompt}\n${prompt}`; + expect(systemPrompt).toBe(mode === "native" ? EVAL_SYSTEM_PROMPT : ""); + return { + raw: {}, + resultText: '{"success":true,"finalAnswer":"Example"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }; + }, + toTrajectory: () => { + throw new Error("verifier not configured"); + }, + }); + expect(request.split(EVAL_SYSTEM_PROMPT)).toHaveLength(2); + expect(request).toContain("Return the exact page title."); + expect(result.harnessImplementation).toEqual({ name: "sdk", version: 1, sdkVersion: "1.0.0" }); + }); +}); diff --git a/packages/evals/tests/framework/experimentMetadata.test.ts b/packages/evals/tests/framework/experimentMetadata.test.ts new file mode 100644 index 000000000..fa4ed6ebe --- /dev/null +++ b/packages/evals/tests/framework/experimentMetadata.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { buildExperimentMetadata } from "../../framework/runner.js"; +import type { Testcase } from "../../types/evals.js"; + +function row(meta: Record): Testcase { + return { + input: { name: "agent/hardbenchmark", modelName: "openai/gpt-5.4-mini" as never }, + name: "agent/hardbenchmark", + tags: [], + metadata: { model: "openai/gpt-5.4-mini", test: "t", ...meta } as never, + expected: true, + }; +} + +describe("buildExperimentMetadata", () => { + it("always carries tool surface and model for bench runs, derived from rows", () => { + const meta = buildExperimentMetadata({ + environment: "BROWSERBASE", + tier: "bench", + harness: "mastra", + testcases: [ + row({ toolSurface: "stagehand_facade", provider: "openai", dataset: "hardbenchmark" }), + row({ toolSurface: "stagehand_facade", provider: "openai", dataset: "hardbenchmark" }), + ], + }); + expect(meta).toMatchObject({ + environment: "BROWSERBASE", + tier: "bench", + harness: "mastra", + tool_surface: "stagehand_facade", + model: "openai/gpt-5.4-mini", + provider: "openai", + dataset: "hardbenchmark", + task_count: 2, + }); + }); + + it("lists several distinct surfaces or models instead of dropping them", () => { + const meta = buildExperimentMetadata({ + environment: "LOCAL", + tier: "bench", + testcases: [ + row({ toolSurface: "stagehand_facade", model: "a/x" }), + row({ toolSurface: "stagehand_facade_legacy", model: "b/y" }), + ], + }); + expect(meta.tool_surface).toEqual(["stagehand_facade", "stagehand_facade_legacy"]); + expect(meta.model).toEqual(["a/x", "b/y"]); + }); + + it("prefers explicit core surface / model override and omits core placeholder models", () => { + const meta = buildExperimentMetadata({ + environment: "LOCAL", + tier: "core", + coreToolSurface: "understudy_code", + testcases: [row({ model: "none" })], + }); + expect(meta.tool_surface).toBe("understudy_code"); + expect(meta).not.toHaveProperty("model"); + }); +}); diff --git a/packages/evals/tests/framework/externalRunner.test.ts b/packages/evals/tests/framework/externalRunner.test.ts index 6fbfbda99..a85b0cad5 100644 --- a/packages/evals/tests/framework/externalRunner.test.ts +++ b/packages/evals/tests/framework/externalRunner.test.ts @@ -3,16 +3,29 @@ import { EvalLogger } from "../../logger.js"; import type { ExternalHarnessTaskPlan } from "../../framework/externalHarnessPlan.js"; import { buildExternalHarnessPrompt, + buildFacadeToolCallMetrics, buildNormalizedHarnessMetrics, + buildTimingMetrics, + buildUsageCostMetrics, + deriveTerminationReason, legacyHarnessFieldPrefix, parseEvalResult, + resolveFinalAnswer, runExternalHarnessTask, + stripEmbeddedEvalReports, } from "../../framework/harnesses/externalRunner.js"; +import { buildTrajectory } from "../../framework/harnesses/trajectoryAdapter.js"; +import { EVAL_SYSTEM_PROMPT } from "../../framework/evalSystemPrompt.js"; + +const verifierState = vi.hoisted(() => ({ + result: undefined as Record | undefined, +})); vi.mock("stagehand-v3", async (importOriginal) => { const mod = await importOriginal(); class FakeV3Evaluator { async verify() { + if (verifierState.result) return verifierState.result; throw new Error("fake verifier unavailable"); } } @@ -42,6 +55,119 @@ const plan: ExternalHarnessTaskPlan = { }; describe("external harness runner", () => { + it.each([true, false])( + "keeps execution loss separate from verified completion=%s", + async (outcomeSuccess) => { + verifierState.result = { + outcomeSuccess, + processScore: outcomeSuccess ? 1 : 0, + perCriterion: [{ criterion: "title", maxPoints: 1, earnedPoints: outcomeSuccess ? 1 : 0 }], + }; + try { + const result = await runExternalHarnessTask({ + harness: "example", + plan, + logger: new EvalLogger(false), + resultContract: "marker", + fallbackErrorMessage: "incomplete", + implementation: { name: "sdk", version: 1 }, + toolAdapter: { + browserSessionLoss: () => ({ + cause: "CDP connection closed", + timestamp: new Date().toISOString(), + }), + observedToolMatcher: (name) => name === "stagehand.run", + }, + verifier: { + v3: {} as never, + dataset: "test", + taskSpec: { + id: "fixture", + instruction: "Read title", + precomputedRubric: { + items: [ + { criterion: "title", description: "Read the correct title", maxPoints: 1 }, + ], + }, + }, + }, + runSession: async () => ({ + raw: {}, + resultText: '{"success":true,"finalAnswer":"Example"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }), + toTrajectory: ({ status }, taskSpec) => + buildTrajectory({ + taskSpec, + status, + finalAnswer: "Example", + toolCalls: [ + { + name: "stagehand.run", + args: { code: "return page.title()" }, + result: "Example", + ok: true, + }, + ], + }), + }); + expect(result.verifierError).toBeUndefined(); + expect(result._success).toBe(outcomeSuccess); + expect(result.harnessStatus).toBe("sdk_error"); + expect(result.terminationReason).toBe("browser_session_lost"); + expect(result.harnessImplementation).toEqual({ name: "sdk", version: 1 }); + } finally { + verifierState.result = undefined; + } + }, + ); + + it.each([ + ["marker", "native"], + ["structured_output", "native"], + ["marker", "task_prefix"], + ["structured_output", "task_prefix"], + ] as const)("dispatches %s with eval policy via %s", async (resultContract, systemPromptMode) => { + const instruction = "Find the item and stop before the final purchase."; + let dispatchedPrompt = ""; + let dispatchedSystemPrompt = ""; + await runExternalHarnessTask({ + harness: "test", + plan: { ...plan, instruction }, + logger: new EvalLogger(false), + resultContract, + systemPromptMode, + toolAdapter: { promptInstructions: "Use the mounted browser." }, + fallbackErrorMessage: "missing result", + runSession: async (prompt, systemPrompt) => { + dispatchedPrompt = prompt; + dispatchedSystemPrompt = systemPrompt; + return { + raw: {}, + resultText: '{"success":true}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }; + }, + toTrajectory: () => { + throw new Error("no verifier configured"); + }, + }); + expect(dispatchedSystemPrompt).toBe(systemPromptMode === "native" ? EVAL_SYSTEM_PROMPT : ""); + expect(dispatchedPrompt.split(EVAL_SYSTEM_PROMPT)).toHaveLength( + systemPromptMode === "native" ? 1 : 2, + ); + expect(dispatchedPrompt).toContain(`Instruction:\n${instruction}`); + expect(dispatchedPrompt).toContain(plan.startUrl); + expect(dispatchedPrompt).toContain("Use the mounted browser."); + expect(dispatchedPrompt.match(/At the end,/g)).toHaveLength(1); + }); + it("parses the last result marker", () => { expect( parseEvalResult( @@ -82,6 +208,39 @@ describe("external harness runner", () => { expect(parseEvalResult("not json")).toEqual({ success: false, raw: "not json" }); }); + it("parses the last report-shaped object trailing free-form narration", () => { + const raw = [ + "I’ll open Imgur, inspect the meme, and report back.", + 'The tool returned {"url":"https://imgur.com"} so I continued.', + '{"success":true,"summary":"Inspected the meme.","finalAnswer":"A cat on a keyboard."}', + ].join("\n\n"); + expect(parseEvalResult(raw)).toMatchObject({ + success: true, + summary: "Inspected the meme.", + finalAnswer: "A cat on a keyboard.", + }); + // A report quoted mid-prose, with more prose after it, is not the conclusion. + expect(parseEvalResult(`${raw}\nthen I kept going`).success).toBe(false); + }); + + it("resolves the final answer from the report, else from the last message minus eval report envelopes", () => { + expect(resolveFinalAnswer({ finalAnswer: "42" }, "ignored")).toBe("42"); + expect( + resolveFinalAnswer({}, 'The answer is on the page.\n\n{"success":true,"summary":"reported"}'), + ).toBe("The answer is on the page."); + expect(resolveFinalAnswer({}, '{"only":"json"}')).toBe('{"only":"json"}'); + for (const deliverable of [ + '{"success":true,"order_id":"123"}', + '{"success":true,"summary":"Order shipped","order_id":"123"}', + '{"success":true}', + ]) + expect(resolveFinalAnswer({}, deliverable)).toBe(deliverable); + expect(parseEvalResult('{"success":true}').success).toBe(true); + + expect(resolveFinalAnswer({}, undefined)).toBeUndefined(); + expect(stripEmbeddedEvalReports("keep {not json} too")).toBe("keep {not json} too"); + }); + it("builds each result-contract tail", () => { const marker = buildExternalHarnessPrompt({ plan, resultContract: "marker" }); const structured = buildExternalHarnessPrompt({ @@ -134,6 +293,125 @@ describe("external harness runner", () => { expect(complete.harness_cost_usd.value).toBe(0.25); }); + it("does not let tool-output text relabel failed calls as trusted session loss", () => { + const lost = + "Browser session lost (CDP connection closed). The task cannot continue; report your final result now."; + const steps = [ + { actionName: "stagehand.run", actionArgs: {}, toolOutput: { ok: true, result: "x" } }, + { + actionName: "stagehand.run", + actionArgs: {}, + toolOutput: { ok: false, error: "bad xpath" }, + }, + { + actionName: "stagehand.run", + actionArgs: {}, + toolOutput: { ok: false, result: lost, error: lost }, + }, + { actionName: "stagehand.snapshot", actionArgs: {}, toolOutput: { ok: false, error: lost } }, + ]; + expect( + buildFacadeToolCallMetrics({ steps } as never, (name) => name.startsWith("stagehand.")), + ).toEqual({ + facade_tool_calls: { count: 1, value: 4 }, + facade_tool_call_failures: { count: 1, value: 3 }, + }); + }); + + it("records browser_session_lost as an SDK error even when the agent self-reports success", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "deepagents", + plan, + logger, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + toolAdapter: { + browserSessionLoss: () => ({ cause: "CDP connection closed", tool: "run" }), + }, + runSession: async () => ({ + raw: {}, + resultText: '{"success":true,"summary":"done","finalAnswer":"326 E 110th St"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + + expect(result).toMatchObject({ + _success: false, + error: "Browser session lost (CDP connection closed)", + finalAnswer: "326 E 110th St", + harnessStatus: "sdk_error", + harnessStopReason: "browser_session_lost", + deepagentsStatus: "sdk_error", + deepagentsStopReason: "browser_session_lost", + }); + expect(logger.getLogs().some((line) => line.message.includes("browser session lost"))).toBe( + true, + ); + }); + + it("counts facade tool calls and their failures from the trajectory", () => { + const steps = [ + { actionName: "stagehand.run", actionArgs: {}, toolOutput: { ok: true, result: "x" } }, + { + actionName: "stagehand.run", + actionArgs: {}, + toolOutput: { ok: false, result: "cancelled" }, + }, + { actionName: "node_repl.js", actionArgs: {}, toolOutput: { ok: true, result: "y" } }, + { actionName: "stagehand.snapshot", actionArgs: {} }, + ]; + expect( + buildFacadeToolCallMetrics({ steps } as never, (name) => name.startsWith("stagehand.")), + ).toEqual({ + facade_tool_calls: { count: 1, value: 3 }, + facade_tool_call_failures: { count: 1, value: 1 }, + }); + }); + + it("records effective policy channel, budget units, and requested effort", async () => { + const result = await runExternalHarnessTask({ + harness: "example", + plan, + logger: new EvalLogger(false), + resultContract: "marker", + fallbackErrorMessage: "missing result", + systemPromptMode: "native", + stepBudget: 100, + stepBudgetUnit: "tool_calls", + configuration: { requestedReasoningEffort: "high" }, + runSession: async (_prompt, policy) => { + expect(policy).toBe(EVAL_SYSTEM_PROMPT); + return { + raw: {}, + resultText: 'EVAL_RESULT: {"success":true,"finalAnswer":"done"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + metrics: {}, + }; + }, + toTrajectory: () => { + throw new Error("no verifier"); + }, + }); + expect(result.harnessConfiguration).toEqual({ + evalPolicyVersion: 1, + systemPromptMode: "native", + stepBudget: 100, + stepBudgetUnit: "tool_calls", + requestedReasoningEffort: "high", + }); + expect(result.usageConvention).toBe("unreported"); + expect(result.cost_usd).toBeUndefined(); + }); + it("assembles normalized and deprecated task-result fields", async () => { const result = await runExternalHarnessTask({ harness: "claude_code", @@ -307,6 +585,34 @@ describe("external harness runner", () => { expect(result._success).toBe(true); expect(result.harnessStopReason).toBe("maximum turn budget reached"); + expect(result.terminationReason).toBe("step_budget"); + }); + + it("derives why a run ended from its status and stop reason", () => { + expect(deriveTerminationReason({ status: "completed" })).toBe("completed"); + expect( + deriveTerminationReason({ + status: "max_turns", + stopReason: "tool step budget exhausted (75 steps)", + }), + ).toBe("step_budget"); + expect( + deriveTerminationReason({ status: "sdk_error", stopReason: "browser_session_lost" }), + ).toBe("browser_session_lost"); + expect(deriveTerminationReason({ status: "sdk_error", stopReason: "aborted" })).toBe("aborted"); + expect( + deriveTerminationReason({ + status: "sdk_error", + stopReason: "This operation was aborted", + }), + ).toBe("aborted"); + expect(deriveTerminationReason({ status: "sdk_error", stopReason: "interrupted" })).toBe( + "aborted", + ); + expect(deriveTerminationReason({ status: "sdk_error", stopReason: "ECONNRESET" })).toBe( + "sdk_error", + ); + expect(deriveTerminationReason({ status: "sdk_error" })).toBe("sdk_error"); }); it("bounds hanging evidence capture before verifier fallback", async () => { @@ -351,7 +657,8 @@ describe("external harness runner", () => { toTrajectory: () => ({}) as never, }); - expect(result._success).toBe(true); + expect(result._success).toBe(false); + expect(result.agentReportedSuccess).toBe(true); expect(result.verifierError).toBeDefined(); expect(captureInvocations).toBe(1); expect(drainInvocations).toBe(1); @@ -361,6 +668,64 @@ describe("external harness runner", () => { } }); + it("ships the normalized step trace in the row logs once the trajectory exists", async () => { + const logger = new EvalLogger(false); + logger.log({ category: "codex", message: "tool-call-delta event", level: 2 }); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + logger, + toolAdapter: { observedToolMatcher: (name) => name.startsWith("stagehand.") }, + verifier: { + v3: {} as never, + taskSpec: { id: "wv-1", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: '{"success":true,"summary":"done","finalAnswer":"ok"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + metrics: {}, + }), + toTrajectory: (_input, taskSpec) => + buildTrajectory({ + taskSpec, + toolCalls: [ + { + name: "stagehand.run", + args: { code: "return page.title()" }, + result: "Example", + ok: true, + reasoning: "read the title", + }, + { name: "bash", args: { command: "echo hi" }, result: "hi", ok: true }, + ], + }), + }); + + const messages = (result.logs ?? []).map((line) => line.message); + expect(messages).toEqual([ + expect.stringContaining("cost unavailable for codex on (unknown model)"), + "step 1 · think · read the title", + "step 1 · run · ok · return page.title() → Example", + "step 2 · bash · ok · echo hi → hi", + "summary · done", + "answer · ok", + expect.stringMatching( + /^result · completed · steps=2 · facade_calls=1 · in=10 \(cached 0\) out=5 · agent=\d+\.\ds$/u, + ), + expect.stringContaining("verifier integration failed"), + expect.stringMatching( + /^timing · agent=\d+\.\ds · evidence=\d+\.\ds · verifier=\d+\.\ds · total=\d+\.\ds$/u, + ), + ]); + expect(result.metrics).toMatchObject({ facade_tool_calls: { count: 1, value: 1 } }); + }); + it.each([ ["never resolves", () => new Promise(() => {})], ["rejects", () => Promise.reject(new Error("drain failed"))], @@ -411,7 +776,8 @@ describe("external harness runner", () => { }, }); - expect(result._success).toBe(true); + expect(result._success).toBe(false); + expect(result.verifierError).toBeDefined(); expect(trajectoryInput?.finalObservation).toBe(finalObservation); expect(trajectoryInput?.stepObservations).toBeUndefined(); expect(captureInvocations).toBe(1); @@ -421,4 +787,339 @@ describe("external harness runner", () => { else process.env.EVAL_CAPTURE_EVIDENCE_TIMEOUT_MS = previous; } }); + + it("splits agent, evidence and verifier wall-clock into separate metrics", async () => { + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + logger: new EvalLogger(false), + toolAdapter: { + captureEvidence: async () => { + await delay(30); + return { url: "https://example.com" } as never; + }, + }, + verifier: { + v3: {} as never, + taskSpec: { id: "wv-1", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => { + await delay(50); + return { + raw: {}, + resultText: '{"success":true,"summary":"done","finalAnswer":"ok"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + metrics: {}, + }; + }, + toTrajectory: (_input, taskSpec) => buildTrajectory({ taskSpec, toolCalls: [] }), + }); + const metrics = result.metrics as Record; + + expect(metrics.agent_wall_ms.value).toBeGreaterThanOrEqual(45); + expect(metrics.evidence_ms.value).toBeGreaterThanOrEqual(25); + expect(metrics.verifier_wall_ms.value).toBeGreaterThanOrEqual(0); + expect(metrics.total_wall_ms.value).toBeCloseTo( + metrics.agent_wall_ms.value + metrics.evidence_ms.value + metrics.verifier_wall_ms.value, + 3, + ); + expect(result.agent_wall_ms).toBe(Math.round(metrics.agent_wall_ms.value)); + }); + + it("reports total_wall_ms as the agent time alone when no verifier runs", async () => { + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + logger: new EvalLogger(false), + resultContract: "marker", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: 'EVAL_RESULT: {"success":true}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + const metrics = result.metrics as Record; + expect(metrics.total_wall_ms.value).toBe(metrics.agent_wall_ms.value); + expect(metrics.verifier_wall_ms).toBeUndefined(); + }); + + it("sums the timing split into total_wall_ms", () => { + expect(buildTimingMetrics({ agentWallMs: 1000, evidenceMs: 200, verifierWallMs: 300 })).toEqual( + { + agent_wall_ms: { count: 1, value: 1000 }, + evidence_ms: { count: 1, value: 200 }, + verifier_wall_ms: { count: 1, value: 300 }, + total_wall_ms: { count: 1, value: 1500 }, + }, + ); + }); + + it("emits normalized usage and the harness-reported bill next to the harness-native metrics", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "claude_code", + plan, + model: "anthropic/claude-sonnet-4-6", + logger, + resultContract: "marker", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: 'EVAL_RESULT: {"success":true}', + transcriptText: "", + status: "completed", + usage: { + inputTokens: 1_000_000, + cachedInputTokens: 2_000_000, + cacheCreationInputTokens: 0, + outputTokens: 100_000, + totalTokens: 3_100_000, + }, + costUsd: 4.2, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + const metrics = result.metrics as Record; + + // Anthropic convention: cache reads sit outside input_tokens. + expect(metrics.usage_input_total.value).toBe(3_000_000); + expect(metrics.usage_input_cached.value).toBe(2_000_000); + expect(metrics.usage_output.value).toBe(100_000); + expect(metrics.usage_reasoning.value).toBe(0); + // Legacy metrics stay untouched for existing dashboards. + expect(metrics.harness_input_tokens.value).toBe(1_000_000); + expect(metrics.harness_cost_usd.value).toBe(4.2); + // The reported bill is the cost column. + expect(metrics.cost_usd.value).toBe(4.2); + expect(result).toMatchObject({ + cost_source: "reported", + billing_channel: "anthropic_api", + cost_usd: 4.2, + }); + expect(logger.getLogs().some((line) => line.category === "cost")).toBe(false); + }); + + it("computes a direct-API harness's bill at provider list price when nothing was reported", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + model: "openai/gpt-5.4-mini", + logger, + resultContract: "marker", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: 'EVAL_RESULT: {"success":true}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1_000_000, outputTokens: 0, totalTokens: 1_000_000 }, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + const metrics = result.metrics as Record; + + // gpt-5.4-mini list price $0.75/M uncached input. + expect(metrics.cost_usd.value).toBeCloseTo(0.75, 6); + expect(result).toMatchObject({ + cost_source: "computed", + billing_channel: "openai_api", + cost_pricing: { + as_of: expect.any(String), + model: expect.any(String), + source: expect.any(String), + }, + }); + expect(logger.getLogs().some((line) => line.category === "cost")).toBe(false); + }); + + it("leaves cost absent and names the model and channel when the bill is unavailable", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + model: "openai/gpt-unpriced-fixture", + logger, + resultContract: "marker", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: 'EVAL_RESULT: {"success":true}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 100, cachedInputTokens: 40, outputTokens: 5, totalTokens: 105 }, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + const metrics = result.metrics as Record; + + expect(metrics.usage_input_total.value).toBe(100); + expect(metrics.usage_input_cached.value).toBe(40); + expect(metrics.cost_usd).toBeUndefined(); + expect(result.cost_source).toBe("unavailable"); + expect(result.billing_channel).toBe("openai_api"); + expect(result.cost_usd).toBeUndefined(); + const costLine = logger.getLogs().find((line) => line.category === "cost"); + expect(costLine?.level).toBe(1); + expect(costLine?.message).toContain("openai/gpt-unpriced-fixture"); + expect(costLine?.message).toContain("openai_api"); + }); + + it("shows the billed cost and its source on the timing line", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + model: "openai/gpt-5.4-mini", + logger, + verifier: { + v3: {} as never, + taskSpec: { id: "wv-1", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: '{"success":true,"summary":"done","finalAnswer":"ok"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1_000_000, outputTokens: 0, totalTokens: 1_000_000 }, + metrics: {}, + }), + toTrajectory: (_input, taskSpec) => buildTrajectory({ taskSpec, toolCalls: [] }), + }); + const timing = result.logs?.find((line) => line.message.startsWith("timing")); + expect(timing?.message).toContain("cost=$0.75 (computed)"); + }); + + it("builds usage/cost metrics without inventing a zero-dollar estimate", () => { + const usage = { + input_total: 10, + input_cached: 4, + input_cache_write: 0, + input_uncached: 6, + output: 2, + reasoning: 1, + reasoning_in_output: true, + convention: "openai_cached_subset" as const, + }; + expect( + buildUsageCostMetrics(usage, { cost_source: "unavailable", billing_channel: "openai_api" }), + ).toEqual({ + usage_input_total: { count: 1, value: 10 }, + usage_input_cached: { count: 1, value: 4 }, + usage_output: { count: 1, value: 2 }, + usage_reasoning: { count: 1, value: 1 }, + }); + expect( + buildUsageCostMetrics(usage, { + cost_source: "computed", + cost_usd: 0.5, + billing_channel: "openai_api", + }), + ).toEqual({ + usage_input_total: { count: 1, value: 10 }, + usage_input_cached: { count: 1, value: 4 }, + usage_output: { count: 1, value: 2 }, + usage_reasoning: { count: 1, value: 1 }, + cost_usd: { count: 1, value: 0.5 }, + }); + // Unreported usage carries no usage_* metrics: zeros would read as a free run. + expect( + buildUsageCostMetrics( + { + ...usage, + input_total: 0, + input_cached: 0, + input_uncached: 0, + output: 0, + reasoning: 0, + convention: "unreported", + }, + { cost_source: "unavailable", billing_channel: "subscription" }, + ), + ).toEqual({}); + }); +}); + +it("keeps task JSON as the final answer and passes it to trajectory conversion", async () => { + const answer = JSON.stringify({ products: [{ name: "example", price: 19 }] }); + let trajectoryAnswer: string | undefined; + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + logger: new EvalLogger(false), + verifier: { + v3: {} as never, + taskSpec: { id: "json-task", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: answer, + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }), + toTrajectory: ({ parsed }, taskSpec) => { + trajectoryAnswer = parsed.finalAnswer; + return buildTrajectory({ taskSpec, toolCalls: [], finalAnswer: parsed.finalAnswer }); + }, + }); + expect(result.finalAnswer).toBe(answer); + expect(trajectoryAnswer).toBe(answer); +}); + +it("sanitizes SDK stop reasons before emitting verifier trajectory traces", async () => { + const logger = new EvalLogger(false); + await runExternalHarnessTask({ + harness: "codex", + plan, + logger, + verifier: { + v3: {} as never, + taskSpec: { id: "trace-task", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: "", + transcriptText: "", + status: "sdk_error", + stopReason: "request failed https://provider.example?apiKey=secret-query-value", + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + metrics: {}, + }), + toTrajectory: (_input, taskSpec) => buildTrajectory({ taskSpec, toolCalls: [] }), + }); + expect(logger.getLogs().some((line) => line.category === "trace")).toBe(true); + expect(JSON.stringify(logger.getLogs())).not.toContain("secret-query-value"); }); diff --git a/packages/evals/tests/framework/gradeExternalTrajectory.test.ts b/packages/evals/tests/framework/gradeExternalTrajectory.test.ts index 378ac0d1a..167adcafb 100644 --- a/packages/evals/tests/framework/gradeExternalTrajectory.test.ts +++ b/packages/evals/tests/framework/gradeExternalTrajectory.test.ts @@ -1,7 +1,13 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { Rubric, TaskSpec, Trajectory, TrajectoryStep } from "stagehand-v3"; -import { gradeExternalTrajectory } from "../../framework/verifierAdapter.js"; +import { + buildPersistedEvaluationResult, + gradeExternalTrajectory, +} from "../../framework/verifierAdapter.js"; import { EvalLogger } from "../../logger.js"; const mockState = vi.hoisted(() => ({ @@ -28,6 +34,64 @@ vi.mock("stagehand-v3", async (importOriginal) => { }); describe("gradeExternalTrajectory", () => { + it("counts a missing V3 criterion score in the denominator", async () => { + mockState.evaluationResult = { + outcomeSuccess: true, + processScore: 0.9, + perCriterion: [ + { criterion: "step one", maxPoints: 1, earnedPoints: 1 }, + { criterion: "step two", maxPoints: 2, earnedPoints: null, evidenceInsufficient: true }, + ], + }; + process.env.EVAL_SUCCESS_MODE = "process"; + const result = await grade({ _success: true }); + expect(result.verifierError).toBeUndefined(); + expect(result.processScoreStrict).toBeCloseTo(1 / 3); + expect(result.processScoreLenient).toBe(0.9); + expect(result._success).toBe(false); + }); + + it("persists raw uncertainty and captured evidence without accepting a grade", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "failed-verifier-evidence-")); + process.env.VERIFIER_PERSIST_TRAJECTORIES = "1"; + mockState.evaluationResult = { + outcomeSuccess: false, + processScore: 0, + findings: [{ category: "verifier_uncertainty", summary: "provider unavailable" }], + }; + try { + const result = await gradeExternalTrajectory({ + buildTrajectory: () => trajectory, + verifier: { v3: {} as never, taskSpec, dataset: "test", trajectoryRoot: root }, + baseResult: { _success: true }, + errorMessage: "unverified", + category: "fixture", + logger: new EvalLogger(false), + }); + expect(result._success).toBe(false); + expect(result.agentReportedSuccess).toBe(true); + expect(result.verifierError).toContain("uncertainty"); + const dir = String(result.trajectoryDir); + expect( + JSON.parse(await fs.readFile(path.join(dir, "trajectory.json"), "utf8")).steps, + ).toHaveLength(3); + const persisted = JSON.parse( + await fs.readFile(path.join(dir, "scores", "result.json"), "utf8"), + ); + expect(persisted).toMatchObject({ graded: false, judge: mockState.evaluationResult }); + expect(persisted).not.toHaveProperty("outcomeSuccess"); + expect(persisted).not.toHaveProperty("processScore"); + expect( + JSON.parse(await fs.readFile(path.join(dir, "task_data.json"), "utf8")).result, + ).toEqual(persisted); + expect( + JSON.parse(await fs.readFile(path.join(dir, "scores", "verifier-error.json"), "utf8")) + .graded, + ).toBe(false); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); const rubric: Rubric = { items: [ { criterion: "step one", description: "does step one", maxPoints: 1 }, @@ -43,7 +107,14 @@ describe("gradeExternalTrajectory", () => { const trajectory = { task: taskSpec, - steps: [{}, {}, {}] as TrajectoryStep[], + steps: Array.from({ length: 3 }, () => ({ + actionName: "stagehand__run", + actionArgs: {}, + reasoning: "", + agentEvidence: { modalities: [{ type: "text", content: "captured page evidence" }] }, + probeEvidence: {}, + toolOutput: { ok: true, result: "captured page evidence" }, + })) as TrajectoryStep[], status: "complete", finalAnswer: "done", usage: {}, @@ -64,6 +135,7 @@ describe("gradeExternalTrajectory", () => { let savedPersist: string | undefined; beforeEach(() => { + mockState.evaluationResult = { outcomeSuccess: true, processScore: 0.92 }; savedSuccessMode = process.env.EVAL_SUCCESS_MODE; savedPersist = process.env.VERIFIER_PERSIST_TRAJECTORIES; delete process.env.EVAL_SUCCESS_MODE; @@ -113,4 +185,175 @@ describe("gradeExternalTrajectory", () => { expect(result._success).toBe(true); expect(result.processScore).toBe(0.95); }); + + it("surfaces the judge verdict, gate list and gate metrics on the result", async () => { + const result = await grade({ metrics: { harness_total_tokens: { count: 1, value: 10 } } }); + + expect(result.judgeOutcomeSuccess).toBe(true); + expect(result.outcomeGates).toEqual([]); + expect(result.processScoreLenient).toBe(0.92); + expect(result.processScoreStrict).toBe(0.92); + expect(result.scoringIncomplete).toBe(true); + const metrics = result.metrics as Record; + expect(metrics.harness_total_tokens).toEqual({ count: 1, value: 10 }); + expect(metrics.outcome_gated).toEqual({ count: 1, value: 0 }); + expect(metrics.scoring_incomplete).toEqual({ count: 1, value: 1 }); + expect(metrics.answer_grounded).toBeUndefined(); + }); + + it("gates a judge pass that never touched the browser and fails _success", async () => { + const result = await gradeExternalTrajectory({ + buildTrajectory: () => + ({ + ...trajectory, + steps: [ + { actionName: "web_fetch", actionArgs: {}, toolOutput: { ok: true, result: "" } }, + ], + }) as unknown as Trajectory, + verifier: { v3: {} as never, taskSpec, dataset: "test" }, + baseResult: { _success: true }, + errorMessage: "agent reported failure", + category: "fx", + logger: new EvalLogger(false), + isFacadeTool: (name) => name.startsWith("stagehand"), + }); + + expect(result.judgeOutcomeSuccess).toBe(true); + expect(result.outcomeSuccess).toBe(false); + expect(result.outcomeGates).toEqual(["no_browser_use"]); + expect(result._success).toBe(false); + // A gate that overrides a judge pass names itself in the row error so the + // reason is visible where the row is read, with the agent's claim attached. + expect(result.error).toMatch( + /^gated: no_browser_use — no browser tool calls \(judge passed; agent said: agent reported failure\)$/, + ); + expect((result.metrics as Record).outcome_gated.value).toBe(1); + }); + + it("persists the gated verdict at the top level of scores/result.json", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "gated-result-json-")); + process.env.VERIFIER_PERSIST_TRAJECTORIES = "1"; + try { + const result = await gradeExternalTrajectory({ + buildTrajectory: () => + ({ + ...trajectory, + finalAnswer: "", + status: "error", + steps: [ + { + actionName: "stagehand__run", + actionArgs: {}, + reasoning: "", + agentEvidence: { modalities: [] }, + probeEvidence: {}, + toolOutput: { ok: true, result: "" }, + }, + ], + }) as Trajectory, + verifier: { v3: {} as never, taskSpec, dataset: "test", trajectoryRoot: root }, + baseResult: { _success: true }, + errorMessage: "agent reported failure", + category: "eve", + logger: new EvalLogger(false), + }); + expect(result.verifierError).toBeUndefined(); + expect(result.outcomeGates).toEqual(["no_final_answer"]); + + const dir = result.trajectoryDir as string; + const persisted = JSON.parse( + await fs.readFile(path.join(dir, "scores", "result.json"), "utf8"), + ); + expect(persisted).toMatchObject({ + outcomeSuccess: false, + judgeOutcomeSuccess: true, + outcomeGates: ["no_final_answer"], + processScore: 0.92, + processScoreStrict: 0.92, + processScoreLenient: 0.92, + judge: { outcomeSuccess: true, processScore: 0.92 }, + }); + const taskData = JSON.parse(await fs.readFile(path.join(dir, "task_data.json"), "utf8")); + expect(taskData.result.outcomeSuccess).toBe(false); + // The sidecar audit file stays. + const gates = JSON.parse(await fs.readFile(path.join(dir, "scores", "gates.json"), "utf8")); + expect(gates.outcomeSuccess).toBe(false); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it("builds the persisted result from the judge verdict and the gates", () => { + const persisted = buildPersistedEvaluationResult( + { outcomeSuccess: true, processScore: 0.8, perCriterion: [], evidenceInsufficient: ["x"] }, + { + outcomeSuccess: false, + judgeOutcomeSuccess: true, + outcomeGates: ["no_browser_use"], + processScore: 0.5, + processScoreStrict: 0.5, + processScoreLenient: 0.8, + perCriterion: [ + { criterion: "a", maxPoints: 1, earnedPoints: 0, explanation: "", blocked: true }, + ], + blockedCriteria: 1, + scoringIncomplete: false, + }, + ); + expect(persisted.outcomeSuccess).toBe(false); + expect(persisted.judgeOutcomeSuccess).toBe(true); + expect(persisted.processScore).toBe(0.5); + expect(persisted.processScoreLenient).toBe(0.8); + expect(persisted.perCriterion).toEqual([expect.objectContaining({ blocked: true })]); + expect(persisted.evidenceInsufficient).toEqual(["x"]); + expect(persisted.judge).toEqual({ + outcomeSuccess: true, + processScore: 0.8, + perCriterion: [], + evidenceInsufficient: ["x"], + }); + }); + + it("records grounding as advisory by default and gates only when opted in", async () => { + const searchOnly = { + ...trajectory, + finalAnswer: "The seat costs SGD 5.", + steps: [ + { + actionName: "stagehand__run", + actionArgs: { code: "await page.goto('https://www.google.com/search?q=seat')" }, + probeEvidence: { url: "https://www.google.com/search?q=seat" }, + toolOutput: { ok: true, result: "AirAsia seat SGD 5" }, + }, + ], + } as unknown as Trajectory; + const run = (dataset: string) => + gradeExternalTrajectory({ + buildTrajectory: () => searchOnly, + verifier: { v3: {} as never, taskSpec, dataset }, + baseResult: { _success: true }, + errorMessage: "agent reported failure", + category: "eve", + logger: new EvalLogger(false), + }); + + // Default: a correct answer sourced from a search snippet still passes; + // the grounding result is recorded so snippet-sourced passes stay filterable. + const advisory = await run("hardbenchmark"); + expect(advisory.outcomeGates).toEqual([]); + expect(advisory._success).toBe(true); + expect((advisory.metrics as Record).answer_grounded.value).toBe(0); + expect( + (advisory.grounding as { ungrounded: Array<{ text: string }> }).ungrounded[0]?.text, + ).toBe("SGD 5"); + + process.env.EVAL_REQUIRE_GROUNDING = "1"; + try { + const gated = await run("hardbenchmark"); + expect(gated.outcomeGates).toEqual(["ungrounded_answer"]); + expect(gated._success).toBe(false); + } finally { + delete process.env.EVAL_REQUIRE_GROUNDING; + } + }); }); diff --git a/packages/evals/tests/framework/harnessObservations.test.ts b/packages/evals/tests/framework/harnessObservations.test.ts index 6c9b65c79..39a3fb804 100644 --- a/packages/evals/tests/framework/harnessObservations.test.ts +++ b/packages/evals/tests/framework/harnessObservations.test.ts @@ -9,6 +9,7 @@ import { } from "../../framework/observationRecorder.js"; import { armsOverLimit, + armsWithPassesWithoutBrowserUse, armsWithUngradedRuns, resolveUnverifiableCriteriaLimit, summarizeArmVerifiability, @@ -238,6 +239,7 @@ describe("verifiability gate", () => { ungradedRuns: 0, unverifiableCriteria: 1, totalCriteria: 7, + passesWithoutBrowserUse: 0, }, { arm: "claude_code × playwright_code × model-a", @@ -245,6 +247,7 @@ describe("verifiability gate", () => { ungradedRuns: 0, unverifiableCriteria: 2, totalCriteria: 5, + passesWithoutBrowserUse: 0, }, ]); }); @@ -254,8 +257,22 @@ describe("verifiability gate", () => { process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA = "1"; expect(resolveUnverifiableCriteriaLimit()).toBe(1); const arms = [ - { arm: "a", gradedRuns: 1, ungradedRuns: 0, unverifiableCriteria: 1, totalCriteria: 4 }, - { arm: "b", gradedRuns: 1, ungradedRuns: 0, unverifiableCriteria: 2, totalCriteria: 4 }, + { + arm: "a", + gradedRuns: 1, + ungradedRuns: 0, + unverifiableCriteria: 1, + totalCriteria: 4, + passesWithoutBrowserUse: 0, + }, + { + arm: "b", + gradedRuns: 1, + ungradedRuns: 0, + unverifiableCriteria: 2, + totalCriteria: 4, + passesWithoutBrowserUse: 0, + }, ]; expect(armsOverLimit(arms, 1).map((a) => a.arm)).toEqual(["b"]); }); @@ -280,6 +297,42 @@ describe("verifiability gate", () => { expect(armsWithUngradedRuns(arms).map((a) => a.arm)).toEqual([arms[0].arm]); }); + it("counts passes that never called the mounted browser surface", () => { + const arms = summarizeArmVerifiability( + [ + row("m", "stagehand_facade", { + criterionCount: 3, + evidenceInsufficient: [], + _success: true, + metrics: { facade_tool_calls: { count: 1, value: 0 } }, + }), + row("m", "stagehand_facade", { + criterionCount: 3, + evidenceInsufficient: [], + _success: true, + metrics: { facade_tool_calls: { count: 1, value: 4 } }, + }), + row("m", "stagehand_facade", { + criterionCount: 3, + evidenceInsufficient: [], + _success: false, + metrics: { facade_tool_calls: { count: 1, value: 0 } }, + }), + // No facade metric at all (older rows): not counted either way. + row("m", "stagehand_facade", { + criterionCount: 3, + evidenceInsufficient: [], + _success: true, + }), + ], + "codex", + ); + expect(arms).toHaveLength(1); + expect(arms[0].gradedRuns).toBe(4); + expect(arms[0].passesWithoutBrowserUse).toBe(1); + expect(armsWithPassesWithoutBrowserUse(arms).map((a) => a.arm)).toEqual([arms[0].arm]); + }); + it("treats malformed limit values as report-only", () => { for (const raw of ["1.5", "10foo", "-2", "", " "]) { process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA = raw; diff --git a/packages/evals/tests/framework/persistTrajectory.test.ts b/packages/evals/tests/framework/persistTrajectory.test.ts index 2384cfb13..6a834c979 100644 --- a/packages/evals/tests/framework/persistTrajectory.test.ts +++ b/packages/evals/tests/framework/persistTrajectory.test.ts @@ -82,6 +82,32 @@ describe("persistAdapterTrajectory", () => { await fs.rm(tmpRoot, { recursive: true, force: true }); } }); + + it("records terminationReason in trajectory.json and metadata.json", async () => { + const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "persist-adapter-termination-")); + try { + const taskSpec: TaskSpec = { id: "budget-task", instruction: "Test task" }; + const { directory } = await persistAdapterTrajectory({ + trajectory: { + ...makeTrajectory(taskSpec), + status: "error", + terminationReason: "step_budget", + }, + taskSpec, + outputRoot: tmpRoot, + runId: "budget-run", + persist: true, + }); + const trajectory = JSON.parse( + await fs.readFile(path.join(directory, "trajectory.json"), "utf8"), + ); + expect(trajectory).toMatchObject({ status: "error", terminationReason: "step_budget" }); + const metadata = JSON.parse(await fs.readFile(path.join(directory, "metadata.json"), "utf8")); + expect(metadata).toMatchObject({ status: "error", terminationReason: "step_budget" }); + } finally { + await fs.rm(tmpRoot, { recursive: true, force: true }); + } + }); }); function makeTrajectory(task: TaskSpec): Trajectory { diff --git a/packages/evals/tests/framework/piScreenshotPipeline.test.ts b/packages/evals/tests/framework/piScreenshotPipeline.test.ts new file mode 100644 index 000000000..b61065fac --- /dev/null +++ b/packages/evals/tests/framework/piScreenshotPipeline.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { compactPiEvent } from "@browserbasehq/stagehand-integrations-pi-sdk"; +import { piAdapter } from "../../framework/harnesses/piAdapter.js"; + +describe("Pi screenshot evidence pipeline", () => { + it("keeps explicit omission evidence without re-decoding an over-budget screenshot", () => { + const event = compactPiEvent( + { + type: "tool_execution_end", + toolCallId: "oversize", + toolName: "screenshot", + result: { content: [{ type: "image", data: "AAAA", mimeType: "image/png" }] }, + }, + { remainingBytes: 0 }, + ); + const trajectory = piAdapter.fromHarnessResult( + { events: [event], finalAnswer: "done" }, + { id: "omitted-image", instruction: "Capture a screenshot." }, + ); + const modalities = trajectory.steps[0].agentEvidence.modalities; + expect(modalities.filter((modality) => modality.type === "image")).toEqual([]); + expect(JSON.stringify(modalities)).toContain("Screenshot omitted"); + expect(JSON.stringify(event)).not.toContain("AAAA"); + }); + it.each([false, true])("retains the screenshot after SDK compaction=%s", (compact) => { + const png = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aZu0AAAAASUVORK5CYII=", + "base64", + ); + const event = { + type: "tool_execution_end", + toolCallId: "screenshot-1", + toolName: "mcp__stagehand__screenshot", + result: { + content: [ + { type: "text", text: "Screenshot captured." }, + { type: "image", data: png.toString("base64"), mimeType: "image/png" }, + ], + }, + isError: false, + }; + + const trajectory = piAdapter.fromHarnessResult( + { events: [compact ? compactPiEvent(event) : event], finalAnswer: "done" }, + { + id: "pi-screenshot-pipeline", + instruction: "Capture a screenshot.", + initUrl: "https://example.invalid", + }, + ); + + expect(trajectory.steps).toHaveLength(1); + const images = trajectory.steps[0].agentEvidence.modalities.filter( + (modality) => modality.type === "image", + ); + expect(images).toHaveLength(1); + expect(images[0]).toMatchObject({ bytes: png, mediaType: "image/png" }); + expect(event.result.content[1].data).toBe(png.toString("base64")); + }); +}); diff --git a/packages/evals/tests/framework/reasoningSummary.test.ts b/packages/evals/tests/framework/reasoningSummary.test.ts new file mode 100644 index 000000000..4cf9e2342 --- /dev/null +++ b/packages/evals/tests/framework/reasoningSummary.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_REASONING_SUMMARY, + isOpenAiModel, + openAiReasoningProviderOptions, + readReasoningSummary, +} from "../../framework/reasoningSummary.js"; + +describe("reasoning summary switch", () => { + it("is on by default and honours every documented spelling of off", () => { + expect(readReasoningSummary({})).toBe(DEFAULT_REASONING_SUMMARY); + expect(readReasoningSummary({ EVAL_REASONING_SUMMARY: "auto" })).toBe("auto"); + expect(readReasoningSummary({ EVAL_REASONING_SUMMARY: "Concise" })).toBe("concise"); + for (const off of ["off", "none", "false", "0"]) { + expect(readReasoningSummary({ EVAL_REASONING_SUMMARY: off })).toBeUndefined(); + } + expect(readReasoningSummary({ EVAL_REASONING_SUMMARY: "verbose" })).toBe( + DEFAULT_REASONING_SUMMARY, + ); + }); + + it.each(["openai/gpt-4.1-mini", "gpt-4o", "text-embedding-3-small", "openai/gpt-5-chat-latest"])( + "does not request reasoning on %s", + (model) => expect(openAiReasoningProviderOptions(model, {})).toBeUndefined(), + ); + + it("only asks OpenAI models for summaries", () => { + expect(isOpenAiModel("openai/gpt-5.6-luna")).toBe(true); + expect(isOpenAiModel("gpt-5.4-mini")).toBe(true); + expect(isOpenAiModel("anthropic/claude-sonnet-4-6")).toBe(false); + expect(openAiReasoningProviderOptions("openai/gpt-5.6-luna", {})).toEqual({ + openai: { reasoningSummary: "detailed" }, + }); + expect(openAiReasoningProviderOptions("anthropic/claude-sonnet-4-6", {})).toBeUndefined(); + expect( + openAiReasoningProviderOptions("openai/gpt-5.6-luna", { EVAL_REASONING_SUMMARY: "off" }), + ).toBeUndefined(); + }); +}); diff --git a/packages/evals/tests/framework/stepBudget.test.ts b/packages/evals/tests/framework/stepBudget.test.ts new file mode 100644 index 000000000..6def51a7a --- /dev/null +++ b/packages/evals/tests/framework/stepBudget.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { DATASET_STEP_BUDGETS, resolveStepBudget } from "../../framework/stepBudget.js"; + +describe("resolveStepBudget", () => { + it("prefers the harness-specific env key over everything else", () => { + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_CODEX_MAX_STEPS", + dataset: "hardbenchmark", + harnessDefault: 100, + env: { EVAL_CODEX_MAX_STEPS: "12", AGENT_EVAL_MAX_STEPS: "34" }, + }), + ).toBe(12); + }); + + it("falls back to AGENT_EVAL_MAX_STEPS before the dataset budget", () => { + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_CODEX_MAX_STEPS", + dataset: "hardbenchmark", + harnessDefault: 100, + env: { AGENT_EVAL_MAX_STEPS: "34" }, + }), + ).toBe(34); + }); + + it("applies the dataset budget when no env override is set", () => { + expect(DATASET_STEP_BUDGETS.hardbenchmark).toBe(100); + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_EVE_MAX_STEPS", + dataset: "hardbenchmark", + harnessDefault: 50, + env: {}, + }), + ).toBe(100); + }); + + it("keeps the harness default for datasets without a budget", () => { + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_EVE_MAX_STEPS", + dataset: "webvoyager", + harnessDefault: 50, + env: {}, + }), + ).toBe(50); + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_CLAUDE_CODE_MAX_TURNS", + dataset: undefined, + harnessDefault: 50, + env: {}, + }), + ).toBe(50); + }); + + it("ignores non-positive and non-numeric env values", () => { + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_FX_MAX_STEPS", + dataset: "hardbenchmark", + harnessDefault: 60, + env: { EVAL_FX_MAX_STEPS: "0", AGENT_EVAL_MAX_STEPS: "lots" }, + }), + ).toBe(100); + }); +}); diff --git a/packages/evals/tests/framework/toSummaryResult.test.ts b/packages/evals/tests/framework/toSummaryResult.test.ts new file mode 100644 index 000000000..44ce0eccf --- /dev/null +++ b/packages/evals/tests/framework/toSummaryResult.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { toSummaryResult } from "../../framework/runner.js"; +import type { EvalInput } from "../../types/evals.js"; + +const input = { name: "agent/hardbenchmark", modelName: "google/gemini-3.5-flash" } as EvalInput; + +describe("toSummaryResult", () => { + it("keeps object outputs and derives the score", () => { + const row = toSummaryResult({ + input, + output: { _success: true, steps: 4 }, + metadata: { categories: ["navigation", 3] }, + }); + expect(row).toEqual({ + input, + output: { _success: true, steps: 4 }, + name: "agent/hardbenchmark", + score: 1, + categories: ["navigation"], + }); + }); + + it("wraps boolean outputs", () => { + expect(toSummaryResult({ input, output: false }).output).toEqual({ _success: false }); + }); + + it("treats a Braintrust row without output as a failed row carrying the error", () => { + const row = toSummaryResult({ input, output: undefined, error: new Error("span failed") }); + expect(row.score).toBe(0); + expect(row.output).toEqual({ _success: false, error: "span failed" }); + expect(row.name).toBe("agent/hardbenchmark"); + }); + + it("explains a missing output when Braintrust reports no error either", () => { + expect(toSummaryResult({ input }).output).toEqual({ + _success: false, + error: "Braintrust reported no output for this task", + }); + expect(toSummaryResult({ input, output: null, error: "boom" }).output).toEqual({ + _success: false, + error: "boom", + }); + }); +}); diff --git a/packages/evals/tests/framework/toolSurfaceResolution.test.ts b/packages/evals/tests/framework/toolSurfaceResolution.test.ts index 385ecd4b1..e47084213 100644 --- a/packages/evals/tests/framework/toolSurfaceResolution.test.ts +++ b/packages/evals/tests/framework/toolSurfaceResolution.test.ts @@ -12,8 +12,8 @@ describe("tool surface resolution", () => { expect(resolveToolSurface(stagehandHarness, "understudy_code")).toBe("understudy_code"); }); - it("defaults to the first supported surface and accepts supported requests", () => { - expect(resolveToolSurface(claudeCodeHarness)).toBe("browse_cli"); + it("defaults to the facade when supported and accepts supported requests", () => { + expect(resolveToolSurface(claudeCodeHarness)).toBe("stagehand_facade"); expect(resolveToolSurface(claudeCodeHarness, "cdp_code")).toBe("cdp_code"); }); diff --git a/packages/evals/tests/framework/traceLog.test.ts b/packages/evals/tests/framework/traceLog.test.ts new file mode 100644 index 000000000..9b57e75d4 --- /dev/null +++ b/packages/evals/tests/framework/traceLog.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, it } from "vitest"; +import type { Trajectory } from "stagehand-v3"; +import { + buildTrajectoryTraceLines, + emitTrajectoryTrace, + shortToolName, + TRACE_AUXILIARY_MAX_CHARS, + type TrajectoryTraceInput, +} from "../../framework/harnesses/traceLog.js"; +import { buildTrajectory } from "../../framework/harnesses/trajectoryAdapter.js"; + +const taskSpec = { id: "wv-1", instruction: "Find the title", initUrl: "https://example.com" }; + +function threeStepTrajectory(): Trajectory { + return buildTrajectory({ + taskSpec, + toolCalls: [ + { + name: "mcp__stagehand__run", + args: { code: "await page.goto('https://example.com');\n return page.title();" }, + result: "Example Domain", + ok: true, + reasoning: "I should open the start URL first\nand read the title.", + }, + { + name: "stagehand.screenshot", + args: {}, + result: "Screenshot captured.", + ok: true, + images: [{ bytes: Buffer.alloc(42 * 1024), mediaType: "image/png" }], + }, + { + name: "stagehand_run", + args: { code: "await page.click('#nope')" }, + result: undefined, + ok: false, + error: "Timeout 30000ms exceeded waiting for #nope", + }, + ], + usage: { input_tokens: 100, output_tokens: 20 }, + }); +} + +const outcome: TrajectoryTraceInput["outcome"] = { + status: "completed", + stopReason: undefined, + usage: { inputTokens: 12345, outputTokens: 678, cachedInputTokens: 9000, totalTokens: 13023 }, +}; + +describe("trajectory trace log", () => { + it("normalizes FX MCP facade tool names", () => { + expect(shortToolName("mcp_stagehand_run")).toBe("run"); + expect(shortToolName("mcp_stagehand_browser_snapshot")).toBe("snapshot"); + }); + + it("emits one identical-shape line per step plus a result line", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome, + isFacadeTool: (name) => /run|screenshot/u.test(name), + }); + + expect(lines.map((line) => line.message)).toEqual([ + "step 1 · think · I should open the start URL first and read the title.", + "step 1 · run · ok · await page.goto('https://example.com'); return page.title(); → Example Domain", + "step 2 · screenshot · ok → [image 42 KB]", + "step 3 · run · ERR · await page.click('#nope') → Timeout 30000ms exceeded waiting for #nope", + "answer · (none — agent reported none)", + "result · completed · steps=3 · facade_calls=3 · in=12345 out=678 cached=9000", + ]); + expect(lines.every((line) => line.category === "trace")).toBe(true); + expect(lines.map((line) => line.level)).toEqual([1, 1, 1, 0, 1, 1]); + }); + + it("traces the agent's summary and final answer before the result line", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome, + report: { + success: true, + summary: "Opened the site and read the title.", + finalAnswer: "Example Domain\nsecond line " + "x".repeat(300), + }, + }); + const tail = lines.slice(-3).map((line) => line.message); + expect(tail[0]).toBe("summary · Opened the site and read the title."); + expect(tail[1].startsWith("answer · Example Domain second line xxx")).toBe(true); + expect(tail[1].length).toBeLessThanOrEqual("answer · ".length + 200 + 1); + expect(lines.at(-2)?.auxiliary?.answer?.value).toContain("x".repeat(300)); + expect(tail[2].startsWith("result · completed")).toBe(true); + }); + + it("states a missing answer with the stop status, as an error line", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome: { ...outcome, status: "max_turns", stopReason: "turn budget exhausted" }, + report: { success: false, summary: "", finalAnswer: "" }, + }); + const answer = lines.at(-2)!; + expect(answer.message).toBe("answer · (none — max_turns)"); + expect(answer.level).toBe(0); + }); + + it("keeps the full code and result in auxiliary", () => { + const [, run, , failed, , result] = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome, + }); + expect(run.auxiliary).toEqual({ + tool: { value: "mcp__stagehand__run", type: "string" }, + code: { + value: "await page.goto('https://example.com'); return page.title();", + type: "string", + }, + result: { value: "Example Domain", type: "string" }, + }); + expect(failed.auxiliary?.error).toEqual({ + value: "Timeout 30000ms exceeded waiting for #nope", + type: "string", + }); + expect(failed.auxiliary?.result).toBeUndefined(); + expect(result.auxiliary?.usage).toEqual({ + value: JSON.stringify(outcome.usage), + type: "object", + }); + expect(result.message).not.toContain("facade_calls"); + }); + + it("clips long code and results to a single line while capping auxiliary", () => { + const longCode = "await page.locator('x').click();\n".repeat(40); + const hugeResult = "y".repeat(TRACE_AUXILIARY_MAX_CHARS + 500); + const [line] = buildTrajectoryTraceLines({ + trajectory: buildTrajectory({ + taskSpec, + toolCalls: [{ name: "run", args: { code: longCode }, result: hugeResult, ok: true }], + }), + outcome, + }); + expect(line.message).not.toContain("\n"); + expect(line.message.length).toBeLessThan(450); + expect(line.message).toMatch(/… → y+…$/u); + expect(line.auxiliary?.result?.value).toHaveLength( + TRACE_AUXILIARY_MAX_CHARS + "…[truncated 500 chars]".length, + ); + }); + + it("summarizes snapshots by node count, object results as JSON, and stop reasons", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: buildTrajectory({ + taskSpec, + toolCalls: [ + { + name: "stagehand.snapshot", + args: { includeIframes: true }, + result: "[1-1] RootWebArea: Example\n [1-2] link: More\n [1-3] button: Go", + ok: true, + }, + { + name: "mcp__stagehand__run", + args: { code: "return {a: 1}" }, + result: { a: 1 }, + ok: true, + }, + { name: "bash", args: { command: "ls -la" }, result: "total 0", ok: true }, + ], + }), + outcome: { ...outcome, status: "sdk_error", stopReason: "max turns\nreached" }, + }); + expect(lines.map((line) => line.message)).toEqual([ + 'step 1 · snapshot · ok · {"includeIframes":true} → [snapshot 3 nodes] [1-1] RootWebArea: Example', + 'step 2 · run · ok · return {a: 1} → {"a":1}', + "step 3 · bash · ok · ls -la → total 0", + "answer · (none — sdk_error)", + "result · sdk_error · max turns reached · steps=3 · in=12345 out=678 cached=9000", + ]); + }); + + it("prefers the normalized usage and agent wall-clock on the result line", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome, + usage: { + input_total: 21345, + input_cached: 9000, + input_cache_write: 0, + input_uncached: 12345, + output: 678, + reasoning: 0, + reasoning_in_output: true, + convention: "anthropic_cache_separate", + }, + agentWallMs: 42_049, + }); + const result = lines.at(-1)!; + expect(result.message).toBe( + "result · completed · steps=3 · in=21345 (cached 9000) out=678 · agent=42.0s", + ); + expect(JSON.parse(result.auxiliary!.normalized_usage!.value as string)).toMatchObject({ + convention: "anthropic_cache_separate", + }); + }); + + it("emits through an EvalLogger-compatible sink", () => { + const logged: string[] = []; + emitTrajectoryTrace( + { log: (line) => void logged.push(line.message) }, + { trajectory: threeStepTrajectory(), outcome }, + ); + expect(logged).toHaveLength(6); + expect(logged.at(-1)).toMatch(/^result · completed/u); + }); + + it("collapses harness-specific tool names onto the surface tool", () => { + expect(shortToolName("mcp__stagehand__run")).toBe("run"); + expect(shortToolName("mcp__stagehand_browser__run")).toBe("run"); + expect(shortToolName("stagehand.snapshot")).toBe("snapshot"); + expect(shortToolName("stagehand_screenshot")).toBe("screenshot"); + expect(shortToolName("stagehand_run")).toBe("run"); + expect(shortToolName("Bash")).toBe("Bash"); + expect(shortToolName("web_search")).toBe("web_search"); + expect(shortToolName("")).toBe("tool"); + }); +}); diff --git a/packages/evals/tests/framework/usageNormalization.test.ts b/packages/evals/tests/framework/usageNormalization.test.ts new file mode 100644 index 000000000..2b0b9092c --- /dev/null +++ b/packages/evals/tests/framework/usageNormalization.test.ts @@ -0,0 +1,279 @@ +import { describe, expect, it } from "vitest"; +import { + formatNormalizedUsage, + normalizeUsage, + usageConventionFor, +} from "../../framework/usageNormalization.js"; + +describe("normalizeUsage", () => { + it.each(["constructor", "__proto__", "toString"])( + "defaults safely for unknown harness %s", + (harness) => { + expect(usageConventionFor(harness)).toBe("openai_cached_subset"); + expect( + normalizeUsage({ harness, raw: { inputTokens: 10, outputTokens: 5, totalTokens: 15 } }), + ).toMatchObject({ input_total: 10, output: 5, convention: "openai_cached_subset" }); + }, + ); + + it("keeps FX reasoning outside output without duplicating cached input", () => { + const usage = normalizeUsage({ + harness: "fx", + raw: { + inputTokens: 1000, + cachedInputTokens: 600, + outputTokens: 200, + reasoningOutputTokens: 50, + totalTokens: 1250, + }, + }); + expect(usage).toMatchObject({ + input_total: 1000, + input_uncached: 400, + output: 200, + reasoning: 50, + reasoning_in_output: false, + }); + expect(usage.input_total + usage.output + usage.reasoning).toBe(1250); + }); + + it.each(["eve", "mastra", "pi", "codex", "cursor", "cursor_sdk"])( + "treats zero-filled %s telemetry with no presence flag as unknown", + (harness) => { + const raw = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; + expect(normalizeUsage({ harness, raw }).convention).toBe("unreported"); + expect(normalizeUsage({ harness, raw: { ...raw, reported: true } }).convention).toBe( + usageConventionFor(harness), + ); + }, + ); + + it("keeps explicit missing Cursor telemetry unknown even when stale counts are nonzero", () => { + const raw = { + inputTokens: 10, + cachedInputTokens: 100, + outputTokens: 20, + totalTokens: 130, + reported: false, + }; + expect(normalizeUsage({ harness: "cursor", raw }).convention).toBe("unreported"); + expect(normalizeUsage({ harness: "cursor", raw: { ...raw, reported: true } })).toMatchObject({ + input_total: 110, + convention: "uncached_only", + }); + }); + + it("uses cache-only counts only for conventions where cache is a separate billable bucket", () => { + const raw = { inputTokens: 0, outputTokens: 0, cachedInputTokens: 100, totalTokens: 100 }; + expect(normalizeUsage({ harness: "pi", raw })).toMatchObject({ + convention: "uncached_only", + input_total: 100, + }); + expect(normalizeUsage({ harness: "codex", raw }).convention).toBe("unreported"); + expect( + normalizeUsage({ + harness: "codex", + raw: { ...raw, cachedInputTokens: 0, reasoningOutputTokens: 100 }, + }).convention, + ).toBe("unreported"); + }); + + it("does not infer priced buckets from a total-only or invalid count", () => { + for (const inputTokens of [0, -1, NaN, Infinity]) { + expect( + normalizeUsage({ + harness: "mastra", + raw: { inputTokens, outputTokens: 0, totalTokens: 500 }, + }).convention, + ).toBe("unreported"); + } + }); + + it("treats cursor_sdk usage as unreported when no token buckets are exposed", () => { + expect(usageConventionFor("cursor_sdk")).toBe("uncached_only"); + expect( + normalizeUsage({ + harness: "cursor_sdk", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }).convention, + ).toBe("unreported"); + }); + + it("adds Cursor SDK cache reads and writes to its uncached input bucket", () => { + // @cursor/sdk 1.0.31 toTokenUsage computes total as the sum of these four + // buckets; reasoning is already part of output and must not be added again. + const raw = { + inputTokens: 40, + cachedInputTokens: 127_000, + cacheCreationInputTokens: 3_000, + outputTokens: 900, + reasoningOutputTokens: 120, + totalTokens: 130_940, + reported: true, + }; + const usage = normalizeUsage({ harness: "cursor_sdk", raw }); + expect(usage).toEqual({ + input_total: 130_040, + input_cached: 127_000, + input_cache_write: 3_000, + input_uncached: 40, + output: 900, + reasoning: 120, + reasoning_in_output: true, + convention: "uncached_only", + }); + expect(usage.input_total + usage.output).toBe(raw.totalTokens); + }); + + it("treats OpenAI-style cached tokens as a subset of input (codex, mastra, eve, deepagents, fx)", () => { + for (const harness of ["codex", "mastra", "eve", "deepagents"]) { + const usage = normalizeUsage({ + harness, + raw: { + inputTokens: 1000, + cachedInputTokens: 600, + outputTokens: 200, + reasoningOutputTokens: 50, + totalTokens: 1200, + }, + }); + expect(usage, harness).toEqual({ + input_total: 1000, + input_cached: 600, + input_cache_write: 0, + input_uncached: 400, + output: 200, + reasoning: 50, + reasoning_in_output: true, + convention: "openai_cached_subset", + }); + } + }); + + it("keeps cache writes inside the total for the subset convention (AI SDK anthropic via eve)", () => { + const usage = normalizeUsage({ + harness: "eve", + raw: { + inputTokens: 1000, + cachedInputTokens: 600, + cacheCreationInputTokens: 100, + outputTokens: 10, + totalTokens: 1010, + }, + }); + expect(usage).toMatchObject({ + input_total: 1000, + input_cached: 600, + input_cache_write: 100, + input_uncached: 300, + }); + }); + + it("never lets a cached count larger than input go negative", () => { + const usage = normalizeUsage({ + harness: "codex", + raw: { inputTokens: 100, cachedInputTokens: 150, outputTokens: 1, totalTokens: 101 }, + }); + expect(usage).toMatchObject({ input_total: 100, input_cached: 100, input_uncached: 0 }); + }); + + it("adds Anthropic cache reads and writes on top of input_tokens (claude_code)", () => { + const usage = normalizeUsage({ + harness: "claude_code", + raw: { + inputTokens: 40, + cachedInputTokens: 127_000, + cacheCreationInputTokens: 3_000, + outputTokens: 900, + totalTokens: 130_940, + }, + }); + expect(usage).toEqual({ + input_total: 130_040, + input_cached: 127_000, + input_cache_write: 3_000, + input_uncached: 40, + output: 900, + reasoning: 0, + reasoning_in_output: true, + convention: "anthropic_cache_separate", + }); + }); + + it("keeps claude_cua cache reads and writes separate until normalization", () => { + const usage = normalizeUsage({ + harness: "claude_cua", + raw: { + inputTokens: 12, + cachedInputTokens: 30, + cacheCreationInputTokens: 8, + outputTokens: 5, + totalTokens: 55, + }, + }); + expect(usage).toMatchObject({ + convention: "anthropic_cache_separate", + input_total: 50, + input_uncached: 12, + input_cached: 30, + input_cache_write: 8, + }); + }); + + it("treats pi input as the uncached remainder (input_tokens=40 with 127k cached observed)", () => { + const usage = normalizeUsage({ + harness: "pi", + raw: { + inputTokens: 40, + cachedInputTokens: 127_000, + cacheCreationInputTokens: 0, + outputTokens: 500, + reasoningOutputTokens: 120, + totalTokens: 127_540, + }, + }); + expect(usage).toMatchObject({ + input_total: 127_040, + input_cached: 127_000, + input_uncached: 40, + output: 500, + reasoning: 120, + convention: "uncached_only", + }); + }); + + it("marks cursor usage as unreported instead of zero", () => { + const usage = normalizeUsage({ + harness: "cursor", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect(usage.convention).toBe("unreported"); + expect(formatNormalizedUsage(usage)).toBe("in=? out=? (usage unreported)"); + }); + + it("treats reported:false as unreported even for a harness that normally reports usage", () => { + const usage = normalizeUsage({ + harness: "codex", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect(usage.convention).toBe("unreported"); + expect( + normalizeUsage({ + harness: "codex", + raw: { inputTokens: 5, outputTokens: 1, totalTokens: 6, reported: true }, + }).convention, + ).toBe("openai_cached_subset"); + }); + + it("falls back to the subset convention for unregistered harnesses", () => { + expect(usageConventionFor("brand_new")).toBe("openai_cached_subset"); + }); + + it("formats the trace token summary from the normalized buckets", () => { + const usage = normalizeUsage({ + harness: "claude_code", + raw: { inputTokens: 10, cachedInputTokens: 90, outputTokens: 5, totalTokens: 105 }, + }); + expect(formatNormalizedUsage(usage)).toBe("in=100 (cached 90) out=5"); + }); +}); diff --git a/packages/evals/tests/framework/verifierGates.test.ts b/packages/evals/tests/framework/verifierGates.test.ts new file mode 100644 index 000000000..093e8a1b0 --- /dev/null +++ b/packages/evals/tests/framework/verifierGates.test.ts @@ -0,0 +1,705 @@ +import { describe, expect, it } from "vitest"; +import type { CriterionScore, EvaluationResult, TrajectoryStep } from "stagehand-v3"; + +import { + applyVerdictGates, + checkAnswerGrounding, + extractGroundingDatums, + isSearchEngineUrl, + resolveRequireGrounding, + strictProcessScore, +} from "../../framework/verifierGates.js"; + +type StepInit = { + action?: string; + /** probeEvidence.url after the step. */ + url?: string; + /** actionArgs.code (a facade `run` call); URLs inside are used as hints. */ + code?: string; + output?: unknown; + ok?: boolean; +}; + +function step({ + action = "stagehand__run", + url, + code, + output = "", + ok = true, +}: StepInit): TrajectoryStep { + return { + actionName: action, + actionArgs: code ? { code } : {}, + reasoning: "", + agentEvidence: { modalities: [] }, + probeEvidence: url ? { url } : {}, + toolOutput: { ok, result: output }, + }; +} + +function criterion( + name: string, + earned: number | null, + max: number, + explanation = "", + extra: Partial = {}, +): CriterionScore { + return { criterion: name, maxPoints: max, earnedPoints: earned, explanation, ...extra }; +} + +const passingJudge: EvaluationResult = { + outcomeSuccess: true, + processScore: 1, + perCriterion: [ + criterion("find it", 2, 2, "Found on the page."), + criterion("report", 1, 1, "Reported."), + ], + evidenceInsufficient: [], +}; + +const isFacadeTool = (name: string) => name.startsWith("stagehand__"); + +describe("applyVerdictGates — outcome gates", () => { + const goodSteps = [step({ url: "https://www.example.com/", output: "Total: $18.95" })]; + + it("leaves a clean judge pass alone", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { steps: goodSteps, status: "complete", finalAnswer: "It costs $18.95." }, + isFacadeTool, + requireGrounding: true, + }); + expect(gates.outcomeSuccess).toBe(true); + expect(gates.judgeOutcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual([]); + }); + + it("gates a pass with an empty final answer", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { steps: goodSteps, status: "complete", finalAnswer: " " }, + requireGrounding: false, + }); + expect(gates.outcomeSuccess).toBe(false); + expect(gates.judgeOutcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual(["no_final_answer"]); + }); + + it("preserves a supported completion when execution later ended in error", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { steps: goodSteps, status: "error", finalAnswer: "done" }, + requireGrounding: false, + }); + expect(gates.outcomeGates).toEqual([]); + expect(gates.outcomeSuccess).toBe(true); + }); + + it("gates a pass with zero facade tool calls only when a matcher is supplied", () => { + const trajectory = { + steps: [step({ action: "web_fetch", output: "price $18.95" })], + status: "complete" as const, + finalAnswer: "It costs $18.95.", + }; + const withMatcher = applyVerdictGates({ + evaluation: passingJudge, + trajectory, + isFacadeTool, + requireGrounding: false, + }); + expect(withMatcher.outcomeGates).toEqual(["no_browser_use"]); + + const withoutMatcher = applyVerdictGates({ + evaluation: passingJudge, + trajectory, + requireGrounding: false, + }); + expect(withoutMatcher.outcomeGates).toEqual([]); + expect(withoutMatcher.outcomeSuccess).toBe(true); + }); + + it("gates an answer whose only numbers came from search-engine pages", () => { + const trajectory = { + steps: [ + step({ + url: "https://www.google.com/search?q=seat+fee", + output: "AirAsia standard seat SGD 5 per sector", + }), + step({ url: "https://www.airasia.com/flights/", output: "Book flights" }), + ], + status: "complete" as const, + finalAnswer: "A window seat costs SGD 5 per sector.", + }; + const strict = applyVerdictGates({ + evaluation: passingJudge, + trajectory, + requireGrounding: true, + }); + expect(strict.outcomeGates).toEqual(["ungrounded_answer"]); + expect(strict.grounding?.ungrounded.map((d) => d.text)).toEqual(["SGD 5"]); + expect(strict.grounding?.ungrounded[0]?.onlyInSearchEngine).toBe(true); + + const lenient = applyVerdictGates({ + evaluation: passingJudge, + trajectory, + requireGrounding: false, + }); + expect(lenient.outcomeGates).toEqual([]); + expect(lenient.grounding?.gatesOutcome).toBe(true); + }); + + it("never flips a judge fail and reports no gates for it", () => { + const gates = applyVerdictGates({ + evaluation: { ...passingJudge, outcomeSuccess: false }, + trajectory: { steps: [], status: "error", finalAnswer: "" }, + isFacadeTool, + requireGrounding: true, + }); + expect(gates.outcomeSuccess).toBe(false); + expect(gates.judgeOutcomeSuccess).toBe(false); + expect(gates.outcomeGates).toEqual([]); + }); + + it("reports only the missing-answer gate without a facade matcher", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { steps: goodSteps, status: "error", finalAnswer: "" }, + requireGrounding: true, + }); + expect(gates.outcomeGates).toEqual(["no_final_answer"]); + }); +}); + +describe("strictProcessScore", () => { + it("reports blocker wording without inferring missing evidence", () => { + const { perCriterion, strict, blockedCriteria } = strictProcessScore({ + perCriterion: [ + criterion("search", 4, 4, "Configured the search with all constraints."), + criterion( + "seat cost", + 5, + 5, + "Due to an uncontrollable platform blocker the seat map was unreachable. Full credit awarded.", + ), + criterion("report", 3, 3, "Reported correctly."), + ], + evidenceInsufficient: [], + }); + expect(strict).toBe(1); + expect(blockedCriteria).toBe(0); + expect(perCriterion?.[1]?.blockerMentioned).toBe(true); + expect(perCriterion?.[0]?.blocked).toBeUndefined(); + }); + + it("does not mark a partially-credited criterion as blocked even if it mentions a blocker", () => { + const { perCriterion, strict } = strictProcessScore({ + perCriterion: [ + criterion("seat cost", 3, 5, "Blocked midway; partial credit for the attempt."), + ], + }); + expect(perCriterion?.[0]?.blocked).toBeUndefined(); + expect(strict).toBeCloseTo(0.6); + }); + + it("zeroes evidenceInsufficient criteria via the flag or the top-level list", () => { + const { strict, blockedCriteria } = strictProcessScore({ + perCriterion: [ + criterion("a", 2, 2, "ok", { evidenceInsufficient: true }), + criterion("b", 2, 2, "ok"), + criterion("c", 2, 2, "ok"), + ], + evidenceInsufficient: ["c"], + }); + expect(blockedCriteria).toBe(2); + expect(strict).toBeCloseTo(2 / 6); + }); + + it("excludes not-applicable criteria and zeros explicitly unsupported work", () => { + const blocked = "Access Denied wall at step 0. Full credit due to uncontrollable blocker."; + const { strict, blockedCriteria } = strictProcessScore({ + perCriterion: [ + criterion("locate", 2, 2, blocked, { evidenceInsufficient: true }), + criterion("add to cart", 3, 3, blocked, { evidenceInsufficient: true }), + criterion("conditional", null, 2, "Not applicable."), + ], + }); + expect(blockedCriteria).toBe(2); + expect(strict).toBe(0); + }); + + it("returns undefined without perCriterion", () => { + expect(strictProcessScore({})).toEqual({ + perCriterion: undefined, + strict: undefined, + blockedCriteria: 0, + }); + }); +}); + +describe("grounding — datum extraction", () => { + it("extracts currency, percent, time, decimal and long-integer datums as gating", () => { + const datums = extractGroundingDatums( + "Costs SGD 5 or $18.95 (12% off), zip 11222, 2.4 miles, lap 1:27:02.624, 3 stops, in 2023.", + ); + const byText = Object.fromEntries(datums.map((d) => [d.text, d])); + expect(byText["SGD 5"]).toMatchObject({ kind: "currency", gates: true }); + expect(byText["$18.95"]).toMatchObject({ kind: "currency", gates: true }); + expect(byText["12%"]).toMatchObject({ kind: "percent", gates: true }); + expect(byText["1:27:02.624"]).toMatchObject({ kind: "time", gates: true }); + expect(byText["2.4"]).toMatchObject({ kind: "decimal", gates: true }); + expect(byText["11222"]).toMatchObject({ kind: "integer", gates: true }); + expect(byText["3"]).toMatchObject({ kind: "integer", gates: false }); + expect(byText["2023"]).toMatchObject({ kind: "integer", gates: false }); + }); + + it("treats capitalised multi-word entities as advisory", () => { + const datums = extractGroundingDatums("Max Verstappen won the Abu Dhabi Grand Prix."); + const entities = datums.filter((d) => d.kind === "entity").map((d) => d.text); + expect(entities).toContain("Max Verstappen"); + expect(entities).toContain("Abu Dhabi Grand Prix"); + expect(datums.every((d) => d.kind !== "entity" || d.gates === false)).toBe(true); + }); + + it("never gates on a datum echoed from the task instruction", () => { + const datums = extractGroundingDatums( + "Under $15: CVS gummies at $8.99.", + "Find CVS multivitamins under $15", + ); + const byText = Object.fromEntries(datums.map((d) => [d.text, d])); + expect(byText["$15"]).toMatchObject({ gates: false, fromInstruction: true }); + expect(byText["$8.99"]).toMatchObject({ gates: true }); + }); +}); + +describe("grounding — matching", () => { + it("uses captured step probe text as grounding evidence", () => { + const captured = step({ url: "https://shop.example.com", output: "" }); + captured.probeEvidence.ariaTree = "Price: $18.95"; + expect(checkAnswerGrounding("The price is $18.95", [captured])?.gatesOutcome).toBe(false); + }); + + it("uses the terminal observation and reports its source separately", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { + steps: [], + status: "complete", + finalAnswer: "$18.95", + finalObservation: { url: "https://shop.example.com", ariaTree: "Price: $18.95" }, + }, + requireGrounding: true, + }); + expect(gates.outcomeSuccess).toBe(true); + expect(gates.grounding?.checked[0]).toMatchObject({ groundedAtFinalObservation: true }); + expect(gates.grounding?.checked[0]).not.toHaveProperty("groundedAtStep"); + }); + + it("does not trust matching text without a known page URL", () => { + const result = checkAnswerGrounding("The price is $18.95", [step({ output: "Price: $18.95" })]); + expect(result?.gatesOutcome).toBe(true); + expect(result?.checked[0]).toMatchObject({ seenOnUnknownPage: true }); + }); + + it("does not promote search or unknown terminal observations to page evidence", () => { + for (const url of [undefined, "https://www.google.com/search?q=price"]) { + const result = checkAnswerGrounding("The price is $18.95", [], "", { + url, + ariaTree: "Price: $18.95", + }); + expect(result?.gatesOutcome).toBe(true); + } + }); + it("matches currency with alias, spacing and trailing zeros", () => { + const steps = [step({ url: "https://shop.example.com", output: "Seat fee: S$ 5.00 each" })]; + const result = checkAnswerGrounding("SGD 5 per sector", steps); + expect(result?.ungrounded).toEqual([]); + expect(result?.checked[0]?.groundedAtStep).toBe(0); + }); + + it("matches numbers regardless of thousands separators and case", () => { + const steps = [step({ url: "https://data.example.com", output: "POPULATION: 8336817 (est.)" })]; + expect(checkAnswerGrounding("about 8,336,817 people", steps)?.gatesOutcome).toBe(false); + }); + + it("does not let a currency amount match inside a longer number", () => { + const steps = [step({ url: "https://shop.example.com", output: "Total $118.95" })]; + expect(checkAnswerGrounding("costs $18.95", steps)?.gatesOutcome).toBe(true); + }); + + it("carries the page URL forward to steps without a hint", () => { + const steps = [ + step({ code: "await page.goto('https://www.google.com/search?q=x')", output: "results" }), + step({ action: "stagehand__snapshot", output: "AI Overview: the fee is $42.50" }), + ]; + const result = checkAnswerGrounding("fee is $42.50", steps); + expect(result?.gatesOutcome).toBe(true); + expect(result?.ungrounded[0]?.onlyInSearchEngine).toBe(true); + }); + + it("prefers the probe URL over URLs mentioned in the code or output", () => { + const steps = [ + step({ + url: "https://www.target-site.com/results", + code: "await page.goto('https://www.google.com/search?q=x'); await page.click('a')", + output: "target-site result: $42.50", + }), + ]; + expect(checkAnswerGrounding("fee is $42.50", steps)?.gatesOutcome).toBe(false); + }); + + it("does not gate when the headline datum is grounded but a secondary one is not", () => { + const steps = [step({ url: "https://www.ups.com/rates", output: "Medium box from $18.95" })]; + const result = checkAnswerGrounding("UPS medium: $18.95. FedEx is around $24.35.", steps); + expect(result?.groundedNumeric).toBe(1); + expect(result?.ungroundedNumeric).toBe(1); + expect(result?.gatesOutcome).toBe(false); + }); + + it("returns undefined when the answer has nothing to check", () => { + expect(checkAnswerGrounding("done", [])).toBeUndefined(); + }); + + it("does not gate an answer with only entity datums", () => { + const result = checkAnswerGrounding("Canyonlands National Park", []); + expect(result?.gatesOutcome).toBe(false); + expect(result?.ungrounded).toHaveLength(1); + }); + + it("classifies search-engine hosts by domain, including subdomains", () => { + expect(isSearchEngineUrl("https://www.google.com/search?q=a")).toBe(true); + expect(isSearchEngineUrl("https://html.duckduckgo.com/html/?q=a")).toBe(true); + expect(isSearchEngineUrl("https://www.scribd.com/doc/1")).toBe(true); + expect(isSearchEngineUrl("https://old.reddit.com/r/x")).toBe(true); + expect(isSearchEngineUrl("https://www.googleapis-mirror.example.com/")).toBe(false); + expect(isSearchEngineUrl("https://www.airasia.com/")).toBe(false); + expect(isSearchEngineUrl(undefined)).toBe(false); + }); +}); + +describe("scoringIncomplete", () => { + it("flags a rubric with more items than judged criteria without touching the outcome", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { + steps: [step({ url: "https://www.example.com", output: "$18.95" })], + status: "complete", + finalAnswer: "$18.95", + }, + requireGrounding: true, + rubricItemCount: 3, + }); + expect(gates.scoringIncomplete).toBe(true); + expect(gates.outcomeSuccess).toBe(true); + }); + + it("is false when counts match or no rubric count is known", () => { + const base = { + evaluation: passingJudge, + trajectory: { steps: [] as TrajectoryStep[], status: "complete" as const, finalAnswer: "ok" }, + requireGrounding: false, + }; + expect(applyVerdictGates({ ...base, rubricItemCount: 2 }).scoringIncomplete).toBe(false); + expect(applyVerdictGates(base).scoringIncomplete).toBe(false); + }); +}); + +describe("resolveRequireGrounding", () => { + it("honours the env override in both directions", () => { + expect(resolveRequireGrounding("hardbenchmark", true, { EVAL_REQUIRE_GROUNDING: "0" })).toBe( + false, + ); + expect(resolveRequireGrounding("custom", false, { EVAL_REQUIRE_GROUNDING: "1" })).toBe(true); + }); + + it("is advisory by default and gates only with EVAL_REQUIRE_GROUNDING=1", () => { + expect(resolveRequireGrounding("hardbenchmark", false, {})).toBe(false); + expect(resolveRequireGrounding("custom", true, {})).toBe(false); + expect(resolveRequireGrounding("hardbenchmark", true, { EVAL_REQUIRE_GROUNDING: "1" })).toBe( + true, + ); + expect(resolveRequireGrounding("hardbenchmark", true, { EVAL_REQUIRE_GROUNDING: "0" })).toBe( + false, + ); + }); +}); + +/** + * Minimal excerpts of the audited HardBench rows (2026-08-31, gpt-5.6-luna). + * Each keeps only the fields the gates read; toolOutput text is trimmed to the + * fragment that carried the datum. + */ +describe("evidence rows from the 2026-08-31 audit", () => { + it("deepagents 7e6993f2 (imgur meme): pass with status=error and no answer is gated", () => { + const evaluation: EvaluationResult = { + outcomeSuccess: true, + processScore: 1, + perCriterion: [ + criterion("Include a frog as the background image", 3, 3, "Frog head as background."), + criterion('Add the exact text "Enjoy your life"', 3, 3, "Text added, centered."), + criterion('Ensure "Enjoy your life" is the only text on the meme', 2, 2, "No other text."), + criterion("Present the created meme", 2, 2, "Presented on the editor canvas."), + ], + evidenceInsufficient: [], + }; + const steps = Array.from({ length: 51 }, () => + step({ + action: "stagehand.snapshot", + url: "https://imgur.com/", + output: "[3-5] RootWebArea: Imgur: The magic of the Internet", + }), + ); + const gates = applyVerdictGates({ + evaluation, + trajectory: { steps, status: "error", finalAnswer: "" }, + isFacadeTool: (name) => name.startsWith("stagehand."), + requireGrounding: true, + rubricItemCount: 4, + }); + expect(gates.judgeOutcomeSuccess).toBe(true); + expect(gates.outcomeSuccess).toBe(false); + expect(gates.outcomeGates).toEqual(["no_final_answer"]); + expect(gates.processScoreStrict).toBe(1); + expect(gates.grounding).toBeUndefined(); + }); + + it("eve airasia_88: SGD 5 lifted from Google snippets is gated; blocker credit zeroed", () => { + const evaluation: EvaluationResult = { + outcomeSuccess: true, + processScore: 11 / 12, + perCriterion: [ + criterion( + "Search for AirAsia flights with the correct constraints", + 4, + 4, + "Configured the search on AirAsia with all required constraints.", + ), + criterion( + "Determine window-seat selection cost for the matching itinerary", + 4, + 5, + "Due to an uncontrollable blocker on the results page the agent could not reach the seat map. However, it resolved the fee via search.", + ), + criterion( + "Report unavailability if no matching direct AirAsia flights", + 3, + 3, + "Not applicable because direct flights exist; full points awarded per instructions.", + ), + ], + evidenceInsufficient: [], + }; + const steps = [ + step({ + code: "await page.goto('https://www.google.com/search?q=AirAsia+Singapore+to+Langkawi+booking')", + url: "https://www.google.com/search?q=AirAsia+Singapore+to+Langkawi+booking", + output: + "airasia.com › flights AirAsia standard seat from SGD 5 ... https://www.airasia.com", + }), + step({ + code: "await page.goto('https://www.airasia.com/flights/')", + url: "https://www.airasia.com/flights/", + output: "One-way Round-trip Singapore Changi Airport Langkawi 24/11/2026 27/11/2026", + }), + step({ + url: "https://www.airasia.com/v2/flights/search/?origin=SIN&destination=LGK", + output: "Loading flights ... (skeleton)", + }), + step({ + code: "await page.goto('https://www.google.com/search?q=AirAsia+seat+selection+window+seat+fee+Singapore')", + url: "https://www.google.com/search?q=AirAsia+seat+selection+window+seat+fee+Singapore", + output: "Standard seat (window/aisle) SGD 5 – SGD 10 per sector · support.airasia.com", + }), + step({ + url: "https://www.google.com/search?q=%22International+Route+-+Seat+(AK)%22", + output: "scribd.com › document AirAsia fee schedule: Standard Seat SGD 5 ...", + }), + ]; + const gates = applyVerdictGates({ + evaluation, + trajectory: { + task: { + instruction: + "How much does it cost to select a window seat on a direct AirAsia flight from Singapore to Langkawi from November 24 to November 27? If there are no available flights for those dates, please indicate that in your answer", + }, + steps, + status: "complete", + finalAnswer: + "A standard window seat costs SGD 5 per passenger per sector. For the round trip from Singapore to Langkawi (Nov 24–27, 2026), selecting a window seat for both flights costs SGD 10 total. Direct flights are available.", + }, + isFacadeTool, + requireGrounding: true, + rubricItemCount: 3, + }); + expect(gates.judgeOutcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual(["ungrounded_answer"]); + expect(gates.outcomeSuccess).toBe(false); + const ungrounded = gates.grounding?.ungrounded.filter((d) => d.kind === "currency"); + expect(ungrounded?.map((d) => d.text)).toEqual(["SGD 5", "SGD 10"]); + expect(ungrounded?.every((d) => d.onlyInSearchEngine)).toBe(true); + // "2026" and "24"/"27" are grounded on airasia.com but must not rescue + // the row: years and short integers never count as grounded numerics. + expect(gates.grounding?.groundedNumeric).toBe(0); + // 4/5 on the blocker criterion is partial credit, not the full-credit + // blocker rule, so the strict score matches the judge here. + expect(gates.processScoreStrict).toBeCloseTo(11 / 12); + expect(gates.processScoreLenient).toBeCloseTo(11 / 12); + }); + + it("eve afcebfed (CVS gluten-free): grounded on cvs.com, not gated", () => { + const evaluation: EvaluationResult = { + outcomeSuccess: true, + processScore: 1, + perCriterion: [ + criterion("Identify multivitamins", 2, 2, "Searched 'gluten free multivitamin'."), + criterion("Apply 'gluten-free' filter or identify attribute", 2, 2, "Search term applied."), + criterion("Apply 'CVS Health Brand' filter or identify brand", 2, 2, "Checked CVS brand."), + criterion("Apply price filter 'under $15'", 2, 2, "Checked $5-$10 and $10-$15."), + criterion("Sort or identify by 'most reviewed'", 3, 3, "Sorted by Most Reviewed."), + criterion("Report the identified product", 3, 3, "Reported the top product."), + ], + evidenceInsufficient: [], + }; + const steps = [ + step({ + code: "await page.goto('https://www.cvs.com/')", + url: "https://www.cvs.com/", + output: "CVS", + }), + step({ + url: "https://www.cvs.com/search?searchTerm=gluten%20free%20multivitamin", + output: + "CVS Women's Daily Multivitamin Gummies, 150 CT $8.99 (280) · CVS Men 50+ Advanced Multivitamin Tablets, 65 CT $9.39 (246) · CVS Women's Multivitamin Tablets, 120 CT $13.49 (243)", + }), + ]; + const gates = applyVerdictGates({ + evaluation, + trajectory: { + task: { + instruction: + "Find the most reviewed gluten-free CVS Health Brand multivitamin under $15 on cvs.com", + }, + steps, + status: "complete", + finalAnswer: + "Most-reviewed matches: 1) CVS Women’s Daily Multivitamin Gummies, 150 CT — $8.99, 280 reviews; 2) CVS Men 50+ Advanced Multivitamin Tablets, 65 CT — $9.39, 246 reviews; 3) CVS Women’s Multivitamin Tablets, 120 CT — $13.49, 243 reviews.", + }, + isFacadeTool, + requireGrounding: true, + rubricItemCount: 6, + }); + // Constraint leaks (gluten-free never verified) are the judge's job; the + // deterministic gates have nothing to say about this row. + expect(gates.outcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual([]); + expect(gates.grounding?.groundedNumeric).toBeGreaterThan(0); + expect(gates.grounding?.ungroundedNumeric).toBe(0); + expect(gates.processScoreStrict).toBe(1); + expect(gates.scoringIncomplete).toBe(false); + }); + + it("blocker wording without evidenceInsufficient leaves the strict score unchanged", () => { + const blocked = (what: string) => + `${what} was impossible due to the uncontrollable platform blocker; full process credit is awarded.`; + const evaluation: EvaluationResult = { + outcomeSuccess: false, + processScore: 1, + perCriterion: [ + criterion( + "Locate a men's T-shirt", + 2, + 2, + "Blocked immediately by Macy's anti-bot ('Access Denied').", + ), + criterion("Select 'large' size for the T-shirt", 2, 2, blocked("Selecting the size")), + criterion( + "Apply 'stripe pattern' filter for the T-shirt", + 2, + 2, + blocked("Applying the filter"), + ), + criterion( + "Apply 'short sleeve' filter for the T-shirt", + 2, + 2, + blocked("Applying the filter"), + ), + criterion("Select a T-shirt from the 'Best Sellers' group", 2, 2, blocked("Selecting")), + criterion("Add the selected T-shirt to the cart", 3, 3, blocked("Adding to cart")), + criterion( + "Respect Critical Point boundaries", + 2, + 2, + "Did not cross any transactional boundary as it was blocked from entering the website.", + ), + ], + evidenceInsufficient: [], + }; + const gates = applyVerdictGates({ + evaluation, + trajectory: { + steps: [step({ url: "https://www.macys.com/", output: "Access Denied" })], + status: "complete", + finalAnswer: "Unable to complete the task because Macy’s blocked access.", + }, + isFacadeTool, + requireGrounding: true, + }); + expect(gates.processScoreLenient).toBe(1); + expect(gates.processScoreStrict).toBe(1); + expect(gates.blockedCriteria).toBe(0); + expect(gates.perCriterion?.every((c) => c.blockerMentioned)).toBe(true); + expect(gates.outcomeSuccess).toBe(false); + }); + + it("mastra 864244b6 (F1): race time seen on espn.com after search detours is grounded", () => { + const steps = [ + step({ + action: "stagehand_run", + code: "await page.goto('https://www.bing.com/search?q=2023+Abu+Dhabi+Grand+Prix+race+time')", + url: "https://www.bing.com/search?q=2023+Abu+Dhabi+Grand+Prix+race+time", + output: '{"title": "2023 Abu Dhabi Grand Prix race time 1:27:02.624 - Search"}', + }), + step({ + action: "stagehand_run", + code: "await page.goto('https://www.espn.com/f1/results/_/id/600026790')", + url: "https://www.espn.com/f1/results/_/id/600026790", + output: + "Etihad Airways Abu Dhabi GP November 24 - November 26, 2023 Yas Marina Circuit RACE WINNER Max Verstappen 1:27:02.624 2 Pits", + }), + ]; + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { + steps, + status: "complete", + finalAnswer: "Max Verstappen won first place with a race time of 1:27:02.624.", + }, + isFacadeTool: (name) => name === "stagehand_run", + requireGrounding: true, + }); + expect(gates.outcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual([]); + expect(gates.grounding?.checked.find((d) => d.kind === "time")?.groundedAtStep).toBe(1); + }); + + it("the same answer with the time only in a Google snippet is gated", () => { + const steps = [ + step({ + url: "https://www.google.com/search?q=ESPN+2023+Abu+Dhabi+Grand+Prix+results", + output: + "ESPN https://www.espn.com › race 2023 Etihad Airways Abu Dhabi Grand Prix F1, won by Max Verstappen. Winner VER 1:27:02.624", + }), + step({ url: "https://www.espn.com/f1/", output: "F1 home — schedule, standings" }), + ]; + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { + steps, + status: "complete", + finalAnswer: "Max Verstappen won the 2023 Abu Dhabi Grand Prix in 1:27:02.624.", + }, + requireGrounding: true, + }); + expect(gates.outcomeGates).toEqual(["ungrounded_answer"]); + }); +}); diff --git a/packages/evals/tests/framework/verifierSpan.test.ts b/packages/evals/tests/framework/verifierSpan.test.ts new file mode 100644 index 000000000..8ac2b9877 --- /dev/null +++ b/packages/evals/tests/framework/verifierSpan.test.ts @@ -0,0 +1,42 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { EvaluationResult, Trajectory } from "stagehand-v3"; +import { verifyTraced } from "../../framework/verifierAdapter.js"; + +const log = vi.hoisted(() => vi.fn()); +vi.mock("../../framework/braintrust.js", () => ({ + tracedSpan: async (fn: (span: { log: typeof log }) => Promise) => fn({ log }), +})); + +describe("verifier child-span grading", () => { + beforeEach(() => log.mockClear()); + + it("retains raw uncertainty with ungraded metadata and no synthetic scores", async () => { + const result = { + outcomeSuccess: false, + processScore: 0, + findings: [{ category: "verifier_uncertainty", description: "provider unavailable" }], + } as EvaluationResult; + expect(await verify(result)).toBe(result); + expect(log).toHaveBeenCalledOnce(); + expect(log.mock.calls[0][0]).toMatchObject({ output: result, metadata: { graded: false } }); + expect(log.mock.calls[0][0].metadata.verifierError).toContain("uncertainty"); + expect(log.mock.calls[0][0]).not.toHaveProperty("scores"); + }); + + it.each([false, true])("records trustworthy outcome=%s scores", async (outcomeSuccess) => { + const result = { outcomeSuccess, processScore: 0.5 } as EvaluationResult; + await verify(result); + expect(log.mock.calls[0][0]).toMatchObject({ + output: result, + scores: { outcome: outcomeSuccess ? 1 : 0, process: 0.5 }, + metadata: { graded: true }, + }); + }); +}); + +function verify(result: EvaluationResult) { + return verifyTraced({ verify: async () => result }, { steps: [] } as unknown as Trajectory, { + taskId: "fixture", + dataset: "fixture", + }); +} diff --git a/packages/evals/tests/framework/verifierTrace.test.ts b/packages/evals/tests/framework/verifierTrace.test.ts new file mode 100644 index 000000000..47023e751 --- /dev/null +++ b/packages/evals/tests/framework/verifierTrace.test.ts @@ -0,0 +1,47 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { writeVerifierTrace } from "../../framework/verifierTrace.js"; + +describe("verifier trace persistence", () => { + let dir: string; + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "verifier-trace-")); + }); + afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(dir, { recursive: true, force: true }); + }); + const lines = [{ category: "verifier", message: "fixture", level: 1 as const }]; + + it.each(["../../escape", "..\\escape", "bad\u0000label"])( + "rejects label %s without writing", + async (label) => { + await expect(writeVerifierTrace(dir, lines, label)).rejects.toThrow("label"); + expect(await fs.readdir(dir)).toEqual([]); + }, + ); + + it("writes a normal label as one filename component", async () => { + const file = await writeVerifierTrace(dir, lines, "fixture-pass"); + expect(file).toBe(path.join(dir, "scores/verifier-trace_fixture-pass.jsonl")); + expect(JSON.parse((await fs.readFile(file!, "utf8")).trim())).toEqual(lines[0]); + }); + + it("warns with the target path when trace persistence fails", async () => { + await fs.writeFile(path.join(dir, "scores"), "not a directory"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(await writeVerifierTrace(dir, lines)).toBeUndefined(); + expect(warn).toHaveBeenCalledOnce(); + expect(String(warn.mock.calls[0][0])).toContain(path.join(dir, "scores/verifier-trace.jsonl")); + }); + + it("redacts credentials from trace write errors", async () => { + vi.spyOn(fs, "writeFile").mockRejectedValue(new Error("storage rejected sk-secret1234567890")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(await writeVerifierTrace(dir, lines)).toBeUndefined(); + expect(String(warn.mock.calls[0][0])).toContain("storage rejected"); + expect(String(warn.mock.calls[0][0])).not.toContain("secret1234567890"); + }); +}); diff --git a/packages/evals/tests/initStagehand.test.ts b/packages/evals/tests/initStagehand.test.ts new file mode 100644 index 000000000..a6f291706 --- /dev/null +++ b/packages/evals/tests/initStagehand.test.ts @@ -0,0 +1,127 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { cleanupActiveRunResources } from "../framework/activeRunCleanup.js"; +import { EVAL_SYSTEM_PROMPT } from "../framework/evalSystemPrompt.js"; +import { initStagehand } from "../initStagehand.js"; +import type { EvalLogger } from "../logger.js"; + +const mocks = vi.hoisted(() => ({ + create: vi.fn(), + connectBrowserbase: vi.fn(), + launchLocal: vi.fn(), + launchRemote: vi.fn(), + resolveKey: vi.fn(), +})); + +vi.mock("@browserbasehq/stagehand", () => ({ + Stagehand: { create: mocks.create }, + browserbase: { connect: mocks.connectBrowserbase }, + localBrowser: { launch: mocks.launchLocal }, +})); +vi.mock("../core/targets/browserbase.js", () => ({ + launchRunnerProvidedBrowserbaseChrome: mocks.launchRemote, +})); +vi.mock("../tui/welcomeStatus.js", () => ({ resolveKey: mocks.resolveKey })); + +describe("native Stagehand evaluation initialization", () => { + const page = { id: "test-page" }; + const browser = { close: vi.fn() }; + const stagehand = { + close: vi.fn(), + browser: { context: { activePage: vi.fn() } }, + }; + const releaseSession = vi.fn(); + const logger = { log: vi.fn() } as unknown as EvalLogger; + + beforeEach(() => { + vi.resetAllMocks(); + mocks.resolveKey.mockImplementation((name: string) => ({ + value: + name === "OPENAI_API_KEY" + ? "test-model-key" + : name === "BROWSERBASE_API_KEY" + ? "test-browser-key" + : "", + source: "process-env", + })); + mocks.launchLocal.mockResolvedValue(browser); + mocks.connectBrowserbase.mockResolvedValue(browser); + mocks.launchRemote.mockResolvedValue({ + sessionId: "test-session", + sessionUrl: "https://browserbase.test/sessions/test-session", + debugUrl: "https://browserbase.test/debug/test-session", + cleanup: releaseSession, + }); + mocks.create.mockResolvedValue(stagehand); + stagehand.browser.context.activePage.mockResolvedValue(page); + stagehand.close.mockResolvedValue(undefined); + browser.close.mockResolvedValue(undefined); + releaseSession.mockResolvedValue(undefined); + }); + + afterEach(async () => { + await cleanupActiveRunResources(); + }); + + it.each(["LOCAL", "BROWSERBASE"] as const)( + "passes the shared system prompt through %s initialization and preserves cleanup", + async (environment) => { + const result = await initStagehand({ + logger, + modelName: "openai/gpt-6-astra", + environment, + }); + + expect(mocks.create).toHaveBeenCalledExactlyOnceWith({ + browser, + selfHeal: true, + model: { modelName: "openai/gpt-6-astra", apiKey: "test-model-key" }, + systemPrompt: EVAL_SYSTEM_PROMPT, + logging: { onLog: expect.any(Function) }, + }); + expect(result.stagehand).toBe(stagehand); + expect(result.page).toBe(page); + expect(stagehand.browser.context.activePage).toHaveBeenCalledOnce(); + if (environment === "LOCAL") { + expect(mocks.launchLocal).toHaveBeenCalledExactlyOnceWith({ headless: false }); + expect(mocks.launchRemote).not.toHaveBeenCalled(); + expect(mocks.connectBrowserbase).not.toHaveBeenCalled(); + expect(result.sessionUrl).toBe(""); + expect(result.debugUrl).toBe(""); + } else { + expect(mocks.launchLocal).not.toHaveBeenCalled(); + expect(mocks.launchRemote).toHaveBeenCalledOnce(); + expect(mocks.connectBrowserbase).toHaveBeenCalledExactlyOnceWith({ + apiKey: "test-browser-key", + sessionId: "test-session", + }); + expect(result.sessionUrl).toBe("https://browserbase.test/sessions/test-session"); + expect(result.debugUrl).toBe("https://browserbase.test/debug/test-session"); + } + + await result.cleanup(); + await result.cleanup(); + await cleanupActiveRunResources(); + expect(stagehand.close).toHaveBeenCalledOnce(); + expect(browser.close).toHaveBeenCalledOnce(); + expect(releaseSession).toHaveBeenCalledTimes(environment === "BROWSERBASE" ? 1 : 0); + }, + ); + + it("releases the connected browser and remote session when Stagehand creation fails", async () => { + const error = new Error("synthetic initialization failure"); + mocks.create.mockRejectedValue(error); + + await expect( + initStagehand({ logger, modelName: "openai/gpt-6-astra", environment: "BROWSERBASE" }), + ).rejects.toBe(error); + + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ systemPrompt: EVAL_SYSTEM_PROMPT }), + ); + await cleanupActiveRunResources(); + expect(browser.close).toHaveBeenCalledOnce(); + expect(releaseSession).toHaveBeenCalledOnce(); + expect(stagehand.close).not.toHaveBeenCalled(); + expect(stagehand.browser.context.activePage).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/evals/tests/logger.test.ts b/packages/evals/tests/logger.test.ts index a33d01db0..9bf0c9623 100644 --- a/packages/evals/tests/logger.test.ts +++ b/packages/evals/tests/logger.test.ts @@ -35,4 +35,16 @@ describe("EvalLogger", () => { expect(logSpy).toHaveBeenCalledTimes(1); expect(logger.getLogs()).toHaveLength(1); }); + + it("keeps level-2 debug lines out of getLogs unless asked for", () => { + const logger = new EvalLogger(false); + logger.log({ category: "session", message: "session", level: 0 }); + logger.log({ category: "trace", message: "step", level: 1 }); + logger.log({ category: "trace", message: "unlevelled" }); + logger.log({ category: "mastra", message: "tool-call-delta event", level: 2 }); + + expect(logger.getLogs().map((line) => line.message)).toEqual(["session", "step", "unlevelled"]); + expect(logger.getLogs({ maxLevel: 2 })).toHaveLength(4); + expect(logger.getLogs({ maxLevel: 0 }).map((line) => line.message)).toEqual(["session"]); + }); }); diff --git a/packages/evals/tests/scripts/updatePricing.test.ts b/packages/evals/tests/scripts/updatePricing.test.ts new file mode 100644 index 000000000..63600633a --- /dev/null +++ b/packages/evals/tests/scripts/updatePricing.test.ts @@ -0,0 +1,110 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { EvalsError } from "../../errors.js"; +import { updatePricing } from "../../scripts/update-pricing.js"; + +let directory: string; +let target: string; +const original = JSON.stringify({ + as_of: "2026-01-01", + models: { + "openai/example": { + input_per_m: 1, + cached_input_per_m: 0.1, + output_per_m: 4, + source: "fixture", + }, + "anthropic/pending": { + input_per_m: null, + cached_input_per_m: null, + output_per_m: null, + source: "needs owner input", + }, + }, +}); + +beforeEach(async () => { + directory = await mkdtemp(path.join(tmpdir(), "stagehand-pricing-test-")); + target = path.join(directory, "pricing.json"); + await writeFile(target, original); + vi.stubEnv("EVAL_PRICING_SOURCE", "gateway"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(async () => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + await rm(directory, { recursive: true, force: true }); +}); + +describe("pricing refresh", () => { + it.each([null, {}, { data: [] }, { data: [null, { id: "broken" }] }])( + "preserves the existing file when HTTP200 has an unusable catalog: %j", + async (body) => { + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json(body)), + ); + await expect(updatePricing(target)).rejects.toBeInstanceOf(EvalsError); + expect(await readFile(target, "utf8")).toBe(original); + }, + ); + + it("does not unprice every model when a catalog has no matching IDs", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ + data: [{ id: "unrelated/model", pricing: { input: "0.000001", output: "0.000004" } }], + }), + ), + ); + await expect(updatePricing(target)).rejects.toThrow(/currently priced models/u); + expect(await readFile(target, "utf8")).toBe(original); + }); + + it("keeps HTTP failures typed and excludes response secrets", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("credential=do-not-emit", { status: 503 })), + ); + const failure = updatePricing(target); + await expect(failure).rejects.toBeInstanceOf(EvalsError); + await expect(failure).rejects.toThrow("gateway models: HTTP 503"); + await expect(failure).rejects.not.toThrow("do-not-emit"); + expect(await readFile(target, "utf8")).toBe(original); + }); + + it("falls back from an unusable gateway and retains observed cache-write rates for owner input", async () => { + vi.stubEnv("EVAL_PRICING_SOURCE", ""); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(Response.json({ data: [] })) + .mockResolvedValueOnce( + Response.json({ + data: [ + { id: "openai/example", pricing: { prompt: "0.000002", completion: "0.000008" } }, + { + id: "anthropic/pending", + pricing: { + prompt: "0.000003", + completion: "0.000015", + input_cache_write: "0.00000375", + }, + }, + ], + }), + ); + vi.stubGlobal("fetch", fetchMock); + await updatePricing(target); + const updated = JSON.parse(await readFile(target, "utf8")); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(updated.models["openai/example"].input_per_m).toBe(2); + expect(updated.models["anthropic/pending"]).toMatchObject({ input_per_m: null }); + expect(updated.models["anthropic/pending"].note).toContain("cache_write=3.75"); + }); +}); diff --git a/packages/evals/tests/tui/helpImports.test.ts b/packages/evals/tests/tui/helpImports.test.ts new file mode 100644 index 000000000..5b0475943 --- /dev/null +++ b/packages/evals/tests/tui/helpImports.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildCommandTree, dispatch } from "../../tui/commandTree.js"; + +vi.mock("../../framework/benchHarness.js", () => { + throw new Error("Help must not initialize the harness runtime"); +}); +vi.mock("../../core/tools/registry.js", () => { + throw new Error("Help must not initialize the tool runtime"); +}); + +afterEach(() => vi.restoreAllMocks()); + +describe("help without runtime imports", () => { + it.each([ + { args: ["--help"], expected: "Commands:" }, + { args: ["list", "--help"], expected: "evals list" }, + { args: ["new", "--help"], expected: "evals new" }, + { args: ["experiments", "--help"], expected: "evals experiments" }, + { args: ["config", "tracing", "--help"], expected: "evals config tracing" }, + ])("prints $args with harness and tool modules unavailable", async ({ args, expected }) => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const getRegistry = vi.fn(async () => { + throw new Error("Help must not discover tasks"); + }); + await dispatch(buildCommandTree(), args, { + entryDir: "/unused", + getRegistry, + setRegistry: vi.fn(), + abortRef: null, + contextPath: null, + }); + expect(log.mock.calls.flat().join("\n")).toContain(expected); + expect(getRegistry).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/evals/tests/tui/run.test.ts b/packages/evals/tests/tui/run.test.ts index 8b5494f8d..95e913509 100644 --- a/packages/evals/tests/tui/run.test.ts +++ b/packages/evals/tests/tui/run.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { DiscoveredTask, TaskRegistry } from "../../framework/types.js"; +import type { RunEvalsResult } from "../../framework/runner.js"; import { canExecuteBenchHarness, deriveCategoryFilter, @@ -8,15 +9,18 @@ import { import { formatBenchHarnessFlags, listBenchHarnessesForTaskKind, + listBenchHarnessesForToolSurface, registerBenchHarness, } from "../../framework/benchHarness.js"; const runEvalsMock = vi.hoisted(() => - vi.fn(async () => ({ - experimentName: "test-experiment", - summary: { passed: 0, failed: 0, total: 0 }, - results: [], - })), + vi.fn( + async (): Promise => ({ + experimentName: "test-experiment", + summary: { passed: 0, failed: 0, total: 0 }, + results: [], + }), + ), ); vi.mock("../../framework/runner.js", () => ({ @@ -209,7 +213,9 @@ describe("deriveCategoryFilter", () => { }, registry, ), - ).rejects.toThrow(formatBenchHarnessFlags(listBenchHarnessesForTaskKind("suite"))); + ).rejects.toThrow( + formatBenchHarnessFlags(listBenchHarnessesForToolSurface("stagehand_facade")), + ); }); it("prints claude_code dry-run matrices without stagehand agent modes", async () => { @@ -253,24 +259,24 @@ describe("deriveCategoryFilter", () => { dataset: "webvoyager", model: "anthropic/claude-sonnet-4-20250514", harness: "claude_code", - toolSurface: "browse_cli", + toolSurface: "stagehand_facade", startupProfile: "tool_create_browserbase", - toolCommand: "browse", - browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + toolCommand: null, + browseCliVersion: null, + browseCliEntrypoint: null, harnessConfig: { harness: "claude_code", model: "anthropic/claude-sonnet-4-20250514", environment: "BROWSERBASE", useApi: false, - toolSurface: "browse_cli", + toolSurface: "stagehand_facade", startupProfile: "tool_create_browserbase", dataset: "webvoyager", }, }); }); - it("prints codex dry-run matrices with browse_cli metadata", async () => { + it("prints codex dry-run matrices with facade metadata", async () => { const registry = makeRegistry([ makeTask({ name: "agent/webvoyager", @@ -311,17 +317,17 @@ describe("deriveCategoryFilter", () => { dataset: "webvoyager", model: "openai/gpt-5.4-mini", harness: "codex", - toolSurface: "browse_cli", + toolSurface: "stagehand_facade", startupProfile: "tool_create_browserbase", - toolCommand: "browse", - browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + toolCommand: null, + browseCliVersion: null, + browseCliEntrypoint: null, harnessConfig: { harness: "codex", model: "openai/gpt-5.4-mini", environment: "BROWSERBASE", useApi: false, - toolSurface: "browse_cli", + toolSurface: "stagehand_facade", startupProfile: "tool_create_browserbase", dataset: "webvoyager", }, @@ -456,6 +462,63 @@ describe("deriveCategoryFilter", () => { expect(output).toContain("Env: BROWSERBASE Harness: stagehand Concurrency: 25"); expect(runEvalsMock).toHaveBeenCalledOnce(); }); + + it("fails a gated batch when a graded pass has zero facade tool calls", async () => { + const previousExitCode = process.exitCode; + const previousLimit = process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA; + const registry = makeRegistry([makeTask()]); + vi.spyOn(console, "log").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + runEvalsMock.mockResolvedValueOnce({ + experimentName: "test-experiment", + summary: { passed: 1, failed: 0, total: 1 }, + results: [ + { + name: "dropdown", + input: { name: "dropdown", modelName: "openai/gpt-4.1-mini" }, + output: { + _success: true, + criterionCount: 1, + evidenceInsufficient: [], + metrics: { facade_tool_calls: { value: 0 } }, + }, + score: 1, + }, + ], + }); + process.exitCode = undefined; + try { + await runCommand( + { + target: "act", + normalizedTarget: "act", + trials: 1, + concurrency: 1, + environment: "LOCAL", + model: "openai/gpt-4.1-mini", + useApi: false, + harness: "stagehand", + envOverrides: { EVAL_MAX_UNVERIFIABLE_CRITERIA: "0" }, + dryRun: false, + preview: false, + successMode: "outcome", + verbose: false, + }, + registry, + ); + + expect(runEvalsMock).toHaveBeenCalledOnce(); + expect(process.exitCode).toBe(1); + expect(error.mock.calls.flat().join("\n")).toMatch( + /passes without (?:any browser tool call|browser use)/, + ); + expect(process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA).toBe(previousLimit); + } finally { + process.exitCode = previousExitCode; + if (previousLimit === undefined) delete process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA; + else process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA = previousLimit; + } + }); }); describe("buildCombinations (preview column-pruning)", () => { diff --git a/packages/evals/tests/tui/verify.test.ts b/packages/evals/tests/tui/verify.test.ts new file mode 100644 index 000000000..b037fec6d --- /dev/null +++ b/packages/evals/tests/tui/verify.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { formatVerdictLine } from "../../tui/commands/verify.js"; + +// eslint-disable-next-line no-control-regex +const plain = (value: string) => value.replace(/\[[0-9;]*m/g, ""); + +describe("evals verify verdict line", () => { + it("shows the gated outcome first and names the gates when they flipped the judge", () => { + const line = plain( + formatVerdictLine({ + outcomeSuccess: false, + judgeOutcomeSuccess: true, + outcomeGates: ["no_final_answer", "no_browser_use"], + processScore: 0.5, + processScoreLenient: 0.75, + }), + ); + expect(line).toContain("outcomeSuccess=false"); + expect(line).toContain("judge=true gated=no_final_answer,no_browser_use"); + expect(line).toContain("processScore=0.500 (lenient=0.750)"); + }); + + it("still reports the judge verdict when nothing was gated", () => { + const line = plain( + formatVerdictLine({ + outcomeSuccess: true, + judgeOutcomeSuccess: true, + outcomeGates: [], + processScore: undefined, + processScoreLenient: undefined, + }), + ); + expect(line).toContain("outcomeSuccess=true judge=true"); + expect(line).toContain("processScore=n/a (lenient=n/a)"); + }); +}); diff --git a/packages/evals/tests/tui/verifyCommand.test.ts b/packages/evals/tests/tui/verifyCommand.test.ts new file mode 100644 index 000000000..a55ec7e66 --- /dev/null +++ b/packages/evals/tests/tui/verifyCommand.test.ts @@ -0,0 +1,177 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_VERIFIER_MODEL } from "../../framework/verifierAdapter.js"; +import { handleVerify } from "../../tui/commands/verify.js"; + +const state = vi.hoisted(() => ({ + options: [] as Record[], + result: {} as Record, + key: "fixture-key" as string | undefined, + error: undefined as Error | undefined, +})); + +vi.mock("stagehand-v3", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + V3: class {}, + V3Evaluator: class { + constructor(_v3: unknown, options: Record) { + state.options.push(options); + } + async verify() { + if (state.error) throw state.error; + return state.result; + } + }, + loadApiKeyFromEnv: () => state.key, + loadTrajectoryFromDisk: async () => ({ + task: { id: "fixture", instruction: "Inspect the page" }, + status: "complete", + steps: [] as unknown[], + }), + nextResultFilename: () => "result_fixture.json", + }; +}); + +describe("offline verifier command", () => { + let dir: string; + let previousExitCode: typeof process.exitCode; + let output: string; + + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "offline-verifier-")); + await fs.writeFile(path.join(dir, "trajectory.json"), "{}"); + state.options = []; + state.result = { outcomeSuccess: true, processScore: 1 }; + state.key = "fixture-key"; + state.error = undefined; + output = ""; + previousExitCode = process.exitCode; + process.exitCode = undefined; + vi.stubEnv("EVAL_VERIFIER_MODEL", ""); + vi.stubEnv("EVAL_VERIFIER_TRACE", "0"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { + output += String(chunk); + return true; + }); + }); + + afterEach(async () => { + process.exitCode = previousExitCode; + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it.each([ + { env: "", args: [], expected: DEFAULT_VERIFIER_MODEL }, + { env: "google/fixture-environment", args: [], expected: "google/fixture-environment" }, + { + env: "google/fixture-environment", + args: ["--model", "anthropic/fixture-cli"], + expected: "anthropic/fixture-cli", + }, + ])("uses shared model policy: $expected", async ({ env, args, expected }) => { + vi.stubEnv("EVAL_VERIFIER_MODEL", env); + await handleVerify([dir, ...args, "--json"]); + expect(state.options).toEqual([ + { backend: "verifier", modelName: expected, modelClientOptions: { apiKey: "fixture-key" } }, + ]); + }); + + it("rejects an explicit CLI model without its provider key", async () => { + state.key = undefined; + await expect(handleVerify([dir, "--model", "anthropic/fixture-cli", "--json"])).rejects.toThrow( + "no API key", + ); + expect(state.options).toEqual([]); + }); + + it("emits uncertainty as ungraded JSON while retaining the raw judge evidence", async () => { + state.result = { + outcomeSuccess: false, + processScore: 0, + findings: [{ category: "verifier_uncertainty", description: "provider unavailable" }], + }; + await handleVerify([dir, "--json"]); + const result = JSON.parse(output); + expect(result).toMatchObject({ graded: false, judge: state.result }); + expect(result.verifierError).toContain("uncertainty"); + expect(result).not.toHaveProperty("outcomeSuccess"); + expect(result).not.toHaveProperty("processScore"); + expect(process.exitCode).toBe(1); + await expect(fs.stat(path.join(dir, "scores"))).rejects.toThrow(); + }); + + it("persists an auditable ungraded result for human output", async () => { + state.result = { + outcomeSuccess: false, + processScore: 0, + findings: [{ category: "verifier_uncertainty", description: "provider unavailable" }], + }; + await handleVerify([dir]); + const result = JSON.parse( + await fs.readFile(path.join(dir, "scores/result_fixture.json"), "utf8"), + ); + expect(result).toMatchObject({ graded: false, judge: state.result }); + expect(result).not.toHaveProperty("outcomeSuccess"); + expect(process.exitCode).toBe(1); + }); + + it("keeps a trustworthy failed verdict scored", async () => { + state.result = { outcomeSuccess: false, processScore: 0 }; + await handleVerify([dir, "--json"]); + expect(JSON.parse(output)).toMatchObject({ outcomeSuccess: false, processScore: 0 }); + expect(JSON.parse(output)).not.toHaveProperty("verifierError"); + expect(process.exitCode).toBeUndefined(); + }); + + it("does not persist uncertainty in dry-run mode", async () => { + state.result = { + outcomeSuccess: false, + processScore: 0, + findings: [{ category: "verifier_uncertainty", description: "provider unavailable" }], + }; + await handleVerify([dir, "--dry-run"]); + expect(process.exitCode).toBe(1); + await expect(fs.stat(path.join(dir, "scores"))).rejects.toThrow(); + }); + + it.each(["human", "json", "dry-run"])( + "preserves thrown verifier errors as ungraded in %s mode", + async (mode) => { + state.error = new Error("provider rejected sk-secret1234567890"); + await handleVerify([ + dir, + ...(mode === "json" ? ["--json"] : mode === "dry-run" ? ["--dry-run"] : []), + ]); + expect(process.exitCode).toBe(1); + if (mode !== "dry-run") { + const result = JSON.parse( + mode === "json" + ? output + : await fs.readFile(path.join(dir, "scores/result_fixture.json"), "utf8"), + ); + expect(result.graded).toBe(false); + expect(result.verifierError).toContain("provider rejected"); + expect(result.verifierError).not.toContain("secret1234567890"); + expect(result).not.toHaveProperty("outcomeSuccess"); + expect(result).not.toHaveProperty("processScore"); + expect(result).not.toHaveProperty("judge"); + } + if (mode !== "human") await expect(fs.stat(path.join(dir, "scores"))).rejects.toThrow(); + }, + ); + + it.each(["../../escape", "..\\escape"])( + "rejects unsafe output label %s before verification", + async (label) => { + await expect(handleVerify([dir, "--label", label])).rejects.toThrow("label"); + expect(state.options).toEqual([]); + }, + ); +}); diff --git a/packages/evals/tui/commands/config.ts b/packages/evals/tui/commands/config.ts index 8a2420f7a..9fd9eee0d 100644 --- a/packages/evals/tui/commands/config.ts +++ b/packages/evals/tui/commands/config.ts @@ -177,7 +177,7 @@ export async function handleConfig(args: string[], entryDir: string): Promise { + const { listBenchHarnesses, listBenchHarnessesForTaskKind } = + await import("../../framework/benchHarness.js"); const suiteHarness = listBenchHarnessesForTaskKind("suite")[0]; print([ "", @@ -153,7 +153,8 @@ export function printNewHelp(): void { ]); } -export function printConfigHelp(): void { +export async function printConfigHelp(): Promise { + const { listCoreRunnableTools } = await import("../../core/tools/registry.js"); print([ "", ` ${dustyCyanHeader("evals config")} ${dim("[subcommand]")}`, @@ -194,7 +195,8 @@ export function printConfigHelp(): void { ]); } -export function printConfigCoreHelp(): void { +export async function printConfigCoreHelp(): Promise { + const { listCoreRunnableTools } = await import("../../core/tools/registry.js"); print([ "", ` ${dustyCyanHeader("evals config core")} ${dim("[subcommand]")}`, diff --git a/packages/evals/tui/commands/run.ts b/packages/evals/tui/commands/run.ts index a86add532..1cc32f138 100644 --- a/packages/evals/tui/commands/run.ts +++ b/packages/evals/tui/commands/run.ts @@ -23,6 +23,7 @@ import type { Harness } from "../../framework/benchTypes.js"; import { formatBenchHarnessFlags, isExecutableBenchHarness } from "../../framework/benchHarness.js"; import { armsOverLimit, + armsWithPassesWithoutBrowserUse, armsWithUngradedRuns, resolveUnverifiableCriteriaLimit, summarizeArmVerifiability, @@ -256,9 +257,13 @@ export async function runCommand( for (const arm of arms) { const ungradedSuffix = arm.ungradedRuns > 0 ? `, ${arm.ungradedRuns} ungraded (self-reported)` : ""; + const browserlessSuffix = + arm.passesWithoutBrowserUse > 0 + ? `, ${arm.passesWithoutBrowserUse} passes without browser use` + : ""; console.log( dim( - ` Verifiability: ${arm.arm} — ${arm.unverifiableCriteria}/${arm.totalCriteria} criteria unverifiable across ${arm.gradedRuns} graded runs${ungradedSuffix}`, + ` Verifiability: ${arm.arm} — ${arm.unverifiableCriteria}/${arm.totalCriteria} criteria unverifiable across ${arm.gradedRuns} graded runs${ungradedSuffix}${browserlessSuffix}`, ), ); } @@ -277,7 +282,13 @@ export async function runCommand( ` ✗ verifiability gate: ${arm.arm} has ${arm.ungradedRuns} ungraded (self-reported) runs`, ); } - if (over.length > 0 || ungraded.length > 0) { + const browserless = armsWithPassesWithoutBrowserUse(arms); + for (const arm of browserless) { + console.error( + ` ✗ verifiability gate: ${arm.arm} has ${arm.passesWithoutBrowserUse} passes without browser use`, + ); + } + if (over.length > 0 || ungraded.length > 0 || browserless.length > 0) { process.exitCode = 1; } } diff --git a/packages/evals/tui/commands/verify.ts b/packages/evals/tui/commands/verify.ts index 763ea745b..f652aa0aa 100644 --- a/packages/evals/tui/commands/verify.ts +++ b/packages/evals/tui/commands/verify.ts @@ -5,25 +5,42 @@ * and returns an EvaluationResult. This command reads the on-disk layout written by * `TrajectoryRecorder.persist()` and feeds it through V3Evaluator.verify(). * + * The judge's verdict is then passed through the same deterministic gates the + * live run applies (see verifierGates.ts), so the offline result matches what + * the row would have scored. The facade gate (`no_browser_use`) needs the live + * tool matcher and is not applied offline. + * * Output: writes a new result file under `scores/result_

+ + + + + First referenceSecond reference + + + + + + + + + + Wrong outer label
+ `); + await page.locator("#shadow-host").evaluate((host) => { + host.attachShadow({ mode: "open" }).innerHTML = + 'Shadow label' + + ''; + }); + const runtime = await createPlaywrightCompatRuntime({ + page, + context: { pages: async () => [page], activePage: async () => page }, + } as unknown as Parameters[0]); + const facade = runtime.page as Pick; + const cases: Array<{ + name: string; + label: string | RegExp; + exact?: boolean; + scope?: string; + expected: string[]; + }> = [ + { + name: "Booking child age aria-label", + expected: ["age"], + label: "Child 1 age", + exact: true, + }, + { + name: "scoped aria-label", + expected: ["age"], + label: "Child 1 age", + exact: true, + scope: "#booking", + }, + { name: "case-insensitive substring", expected: ["age"], label: "CHILD 1" }, + { name: "exact case sensitivity", expected: [], label: "child 1 age", exact: true }, + { name: "native label", expected: ["native"], label: "Native label", exact: true }, + { name: "wrapped label", expected: ["wrapped"], label: "Wrapped label", exact: true }, + { name: "first associated label", expected: ["multi"], label: "First label", exact: true }, + { + name: "second associated label", + expected: ["multi"], + label: "Second label", + exact: true, + }, + { + name: "labels are not concatenated", + expected: [], + label: "First label Second label", + exact: true, + }, + { name: "first ARIA reference", expected: ["refs"], label: "First reference", exact: true }, + { + name: "second ARIA reference", + expected: ["refs"], + label: "Second reference", + exact: true, + }, + { + name: "ARIA references are not concatenated", + expected: [], + label: "First reference Second reference", + exact: true, + }, + { name: "labelledby takes priority", expected: [], label: "Overridden ARIA", exact: true }, + { + name: "empty referenced label takes priority", + expected: [], + label: "Ignored fallback", + exact: true, + }, + { + name: "ARIA takes priority over native label", + expected: [], + label: "Ignored native", + exact: true, + }, + { name: "ARIA priority match", expected: ["aria-first"], label: "ARIA wins", exact: true }, + { + name: "broken labelledby falls back", + expected: ["broken-ref"], + label: "Fallback label", + exact: true, + }, + { + name: "normalized string whitespace", + expected: ["spaces"], + label: "Child 2 age", + exact: true, + }, + { + name: "empty ARIA falls back", + expected: ["empty-aria"], + label: "Empty ARIA fallback", + exact: true, + }, + { + name: "label text excludes script/style", + expected: ["mixed"], + label: "Clean label", + exact: true, + }, + { name: "regular expression", expected: ["age"], label: /^child 1 age$/i }, + { name: "regex keeps original whitespace", expected: ["regex"], label: /^Line\nBreak$/ }, + { + name: "empty query excludes unlabeled elements", + expected: ["blank-ref"], + label: "", + exact: true, + }, + { + name: "shadow-root reference", + expected: ["shadow-input"], + label: "Shadow label", + exact: true, + }, + { name: "shadow-root ARIA", expected: ["shadow-aria"], label: "Shadow ARIA", exact: true }, + { + name: "reference cannot cross shadow boundary", + expected: [], + label: "Wrong outer label", + exact: true, + }, + ]; + const results = []; + for (const test of cases) { + const nativeScope = test.scope ? page.locator(test.scope) : page; + const facadeScope = test.scope ? facade.locator(test.scope) : facade; + const options = { exact: test.exact }; + const native = await nativeScope + .getByLabel(test.label, options) + .evaluateAll((els) => els.map((el) => el.id)); + const actual = await facadeScope + .getByLabel(test.label, options) + .evaluateAll((els) => els.map((el) => el.id)); + results.push({ + name: test.name, + native, + actual, + expected: test.expected, + pass: + JSON.stringify(native) === JSON.stringify(actual) && + JSON.stringify(actual) === JSON.stringify(test.expected), + }); + } + assert.ok( + results.every((r) => r.pass), + `Facade label mismatch: ${JSON.stringify(results.filter((result) => !result.pass))}`, + ); + await facade.getByLabel("Child 1 age", { exact: true }).selectOption("8"); + assert.equal(await page.locator("#age").inputValue(), "8"); + } finally { + await page.close(); + } + }); + it("resolves nested role filters before applying hasNot or invoking callbacks", async () => { + const page = await browser.newPage(); + try { + await page.setContent(` +
+
+ `); + // This snapshot is the browser-computed name that the DOM approximation + // misses for image-only buttons. Native Playwright independently checks it. + const rawPage = { + url: () => page.url(), + evaluate: page.evaluate.bind(page), + snapshot: async () => ({ + formattedTree: "[1] button: Pay now\n[2] button: Cancel order", + xpathMap: { + "1": "/html/body/section[1]/div/button", + "2": "/html/body/section[2]/div/button", + }, + }), + }; + const runtime = await createPlaywrightCompatRuntime({ + page: rawPage, + context: { pages: async () => [rawPage] }, + } as unknown as Parameters[0]); + const facade = runtime.page as Page; + for (const scope of [page, facade]) { + const pay = scope.getByRole("button", { name: "Pay now", exact: true }); + assert.deepEqual( + await scope + .locator("section") + .filter({ has: pay }) + .evaluateAll((els) => els.map((el) => el.id)), + ["pay"], + ); + assert.deepEqual( + await scope + .locator("section") + .filter({ hasNot: pay }) + .evaluateAll((els) => els.map((el) => el.id)), + ["cancel"], + ); + assert.deepEqual( + await scope + .locator("section") + .filter({ + has: scope.locator("div").filter({ has: pay }), + }) + .evaluateAll((els) => els.map((el) => el.id)), + ["pay"], + ); + assert.deepEqual( + await scope + .locator("section") + .filter({ + hasNot: scope.getByRole("button", { name: "Missing", exact: true }), + }) + .evaluateAll((els) => els.map((el) => el.id)), + ["pay", "cancel"], + ); + } + await facade + .locator("section") + .filter({ + hasNot: facade.getByRole("button", { + name: "Pay now", + exact: true, + }), + }) + .evaluate((el) => el.setAttribute("data-mutated", "yes")); + assert.deepEqual( + await page.locator("[data-mutated]").evaluateAll((els) => els.map((el) => el.id)), + ["cancel"], + ); + } finally { + await page.close(); + } + }); + it.each(["open", "closed"] as const)( + "keeps scoped role matches inside nested %s shadow roots", + async (mode) => { + const page = await browser.newPage(); + const cdp = await page.context().newCDPSession(page); + const { evaluateWithShadowRoots } = await import( + new URL("../../../extension/understudy/shadowRootEvaluation.ts", import.meta.url).href + ); + try { + await page.setContent( + '
', + ); + await page.evaluate((mode) => { + (window as unknown as { pageOwnedValue: number }).pageOwnedValue = 42; + const outer = document.querySelector("#host")!.attachShadow({ mode }); + outer.innerHTML = '
'; + const inner = outer.querySelector("#nested")!.attachShadow({ mode }); + inner.innerHTML = + ''; + }, mode); + const rawPage = { + pageId: "fixture", + url: () => page.url(), + evaluate: page.evaluate.bind(page), + snapshot: async () => ({ + formattedTree: "[1] button: Pay now", + xpathMap: { "1": "/html/body/section[1]/div[1]//div[1]//button[1]" }, + }), + }; + const runtime = await createPlaywrightCompatRuntime({ + page: rawPage, + context: { pages: async () => [rawPage] }, + evaluateWithShadowRoots: (_pageId: string, source: string) => + evaluateWithShadowRoots(cdp, (expression: string) => page.evaluate(expression), source), + } as unknown as Parameters[0]); + const facade = runtime.page as Pick; + const pay = facade.getByRole("button", { name: "Pay now", exact: true }); + assert.equal(await pay.count(), 1); + assert.equal( + await facade.locator("#inside").getByRole("button", { name: "Pay now" }).count(), + 1, + ); + assert.equal( + await facade.locator("#outside").getByRole("button", { name: "Pay now" }).count(), + 0, + ); + assert.deepEqual( + await facade + .locator("section") + .filter({ has: pay }) + .evaluateAll((els) => els.map((el) => el.id)), + ["inside"], + ); + assert.deepEqual( + await facade + .locator("section") + .filter({ hasNot: pay }) + .evaluateAll((els) => els.map((el) => el.id)), + ["outside"], + ); + assert.equal(await facade.getByLabel("Amount").inputValue(), "10"); + assert.equal(await facade.locator("#host").locator("#pay").count(), 1); + assert.equal( + await pay.evaluate((el) => { + el.setAttribute("data-checked", "yes"); + return (window as unknown as { pageOwnedValue: number }).pageOwnedValue; + }), + 42, + ); + assert.equal(await facade.locator('[data-checked="yes"]').count(), 1); + // Fresh roots after navigation must not reuse remote references. + await page.goto("about:blank"); + assert.equal(await facade.locator("#pay").count(), 0); + } finally { + await cdp.detach(); + await page.close(); + } + }, + ); + + it("enforces strict reads, bounded waits and non-dispatched trial actions", async () => { + const page = await browser.newPage(); + try { + await page.setContent( + '', + ); + const runtime = await createPlaywrightCompatRuntime({ + page, + context: { pages: async () => [page], activePage: async () => page }, + } as unknown as Parameters[0]); + const facade = runtime.page as Page; + const results: Array<{ name: string; pass: boolean; detail?: string }> = []; + const check = async (name: string, action: () => Promise): Promise => { + try { + await action(); + results.push({ name, pass: true }); + } catch (error) { + results.push({ name, pass: false, detail: String(error) }); + } + }; + // Strictness should be reported before a busy CI browser exhausts its command budget. + const strictTimeout = 5000; + const methods: Array<[string, (locator: Locator) => Promise]> = [ + ["textContent", (locator) => locator.textContent({ timeout: strictTimeout })], + ["innerText", (locator) => locator.innerText({ timeout: strictTimeout })], + ["innerHTML", (locator) => locator.innerHTML({ timeout: strictTimeout })], + ["inputValue", (locator) => locator.inputValue({ timeout: strictTimeout })], + ["getAttribute", (locator) => locator.getAttribute("value", { timeout: strictTimeout })], + ["isChecked", (locator) => locator.isChecked({ timeout: strictTimeout })], + ["isDisabled", (locator) => locator.isDisabled({ timeout: strictTimeout })], + ["isEnabled", (locator) => locator.isEnabled({ timeout: strictTimeout })], + ["isVisible", (locator) => locator.isVisible()], + ["boundingBox", (locator) => locator.boundingBox({ timeout: strictTimeout })], + ["focus", (locator) => locator.focus({ timeout: strictTimeout })], + ["evaluate", (locator) => locator.evaluate((el) => el.setAttribute("data-mutated", "yes"))], + [ + "evaluateHandle", + (locator) => locator.evaluateHandle((el) => el.setAttribute("data-mutated", "yes")), + ], + ]; + for (const [name, run] of methods) { + await check(`strict ${name}`, async () => { + await assert.rejects(run(page.locator(".duplicate")), /strict mode violation/); + await assert.rejects(run(facade.locator(".duplicate")), /strict mode violation/); + }); + } + await check("ambiguous callbacks never execute", async () => { + assert.equal(await page.locator("[data-mutated]").count(), 0); + }); + for (const [name, scope] of [ + ["native", page], + ["facade", facade], + ] as const) { + await check(`${name} waits for delayed element`, async () => { + const id = `delayed-${name}`; + await page.evaluate((id) => { + setTimeout(() => { + const el = document.createElement("input"); + el.id = id; + el.value = "arrived"; + document.body.append(el); + }, 120); + }, id); + assert.equal(await scope.locator(`#${id}`).inputValue({ timeout: 1000 }), "arrived"); + }); + await check(`${name} honors explicit timeout`, async () => { + const start = Date.now(); + await assert.rejects( + scope.locator("#missing").textContent({ timeout: 150 }), + /timed out|Timeout/, + ); + assert.ok(Date.now() - start >= 100 && Date.now() - start < 1500); + }); + await check(`${name} timeout zero waits`, async () => { + const id = `unlimited-${name}`; + await page.evaluate((id) => { + setTimeout(() => { + const el = document.createElement("div"); + el.id = id; + el.textContent = "arrived"; + document.body.append(el); + }, 120); + }, id); + assert.equal(await scope.locator(`#${id}`).textContent({ timeout: 0 }), "arrived"); + }); + } + await check("collection reads and visibility stay immediate", async () => { + assert.deepEqual(await facade.locator(".duplicate").allTextContents(), ["", ""]); + assert.equal(await facade.locator("#missing").count(), 0); + assert.equal(await facade.locator("#missing").isVisible(), false); + }); + await check("nth disambiguates reads", async () => { + assert.equal(await facade.locator(".duplicate").nth(1).inputValue(), "two"); + }); + await check("unsupported trial never clicks, including force", async () => { + await page.evaluate(() => { + document.body.insertAdjacentHTML( + "beforeend", + '', + ); + }); + for (const force of [false, true]) { + await assert.rejects( + facade.locator("#trial").click({ trial: true, force }), + /trial clicks are not supported/, + ); + } + assert.equal(await page.locator("#trial").getAttribute("data-clicked"), null); + }); + assert.ok( + results.every((result) => result.pass), + `Facade locator mismatch: ${JSON.stringify(results.filter((result) => !result.pass))}`, + ); + } finally { + await page.close(); + } + }); +}); diff --git a/packages/integrations/core/integration/facade-frames.test.ts b/packages/integrations/core/integration/facade-frames.test.ts new file mode 100644 index 000000000..bc8bf1973 --- /dev/null +++ b/packages/integrations/core/integration/facade-frames.test.ts @@ -0,0 +1,173 @@ +import { createServer } from "node:http"; +import { expect, it, vi } from "vitest"; +import { localBrowser, Stagehand, type StagehandBrowser } from "@browserbasehq/stagehand"; +import { StagehandFacadeTools } from "../src/facade/tools.js"; + +type FrameState = { + fixture: string; + active: string; + click: number; + input: number; + change: number; + lastClick: string | null; + scrollY: number; + targetScroll: number; + targetInView: boolean; +}; + +it("runs the shared facade against same-origin and out-of-process local frames", async () => { + let port = 0; + const server = createServer((request, response) => { + response.setHeader("content-type", "text/html; charset=utf-8"); + if (request.url === "/nested") { + response.end(""); + return; + } + if (request.url === "/same" || request.url === "/cross") { + const name = request.url === "/same" ? "Same value" : "Cross value"; + response.end(` + + +
+
+
+
+
Scrollable target
+
+ `); + return; + } + response.end( + `

Local frame fixture

`, + ); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + // The fixture uses both 127.0.0.1 and localhost to force site isolation. + // An unspecified bind accepts IPv4 and IPv6 localhost resolution. + server.listen(0, resolve); + }); + port = (server.address() as { port: number }).port; + let browser: StagehandBrowser | undefined; + let stagehand: Stagehand | undefined; + const generate = vi.fn(async (): Promise => { + throw new Error("This local fixture must not call a model"); + }); + try { + browser = await localBrowser.launch({ headless: true, args: ["--site-per-process"] }); + stagehand = await Stagehand.create({ browser, model: { generate }, logging: { level: "off" } }); + const tools = new StagehandFacadeTools(stagehand); + await tools.run(`await page.goto(${JSON.stringify(`http://127.0.0.1:${port}/`)});`); + const page = await stagehand.browser.context.activePage(); + if (!page) throw new Error("Missing local fixture page"); + const inspect = () => + page.evaluate( + () => + new Promise((resolve, reject) => { + const replies: FrameState[] = []; + const timeout = setTimeout(() => { + removeEventListener("message", listener); + reject(new Error("Frame fixture did not report its state")); + }, 2_000); + const listener = (event: MessageEvent) => { + if (!["/same", "/cross"].includes(event.data?.fixture)) return; + replies.push(event.data as FrameState); + if (replies.length === 2) { + clearTimeout(timeout); + removeEventListener("message", listener); + resolve(replies.sort((a, b) => a.fixture.localeCompare(b.fixture))); + } + }; + addEventListener("message", listener); + for (const frame of document.querySelectorAll("iframe")) + frame.contentWindow?.postMessage("inspect-fixture", "*"); + }), + ); + + for (const id of ["same", "cross"]) { + await tools.run(`await page.frameLocator("#${id}").locator("#focus").focus();`); + const state = (await inspect()).find((state) => state.fixture === `/${id}`); + expect(state).toMatchObject({ active: "focus", click: 0, input: 0, change: 0 }); + } + for (const id of ["same", "cross"]) { + await tools.run( + `await page.frameLocator("#${id}").locator("#target").scrollIntoViewIfNeeded();`, + ); + } + for (const state of await inspect()) { + expect(state.targetInView).toBe(true); + expect(state.scrollY).toBeGreaterThan(0); + expect(state.targetScroll).toBe(45); + expect(state).toMatchObject({ click: 0, input: 0, change: 0 }); + } + for (const id of ["same", "cross"]) { + await expect( + tools.run(` + const frame = page.frameLocator("#${id}"); + const choices = frame.locator(".choice"); + return [await frame.getByRole("button", {name:"Frame action", exact:true}).count(), + await (await choices.nth(1).all())[0].textContent(), + await (await choices.last().all())[0].textContent()]; + `), + ).resolves.toEqual([1, "Second", "Second"]); + await tools.run( + `await page.frameLocator("#${id}").locator("#a, #b").locator("button").nth(0).click();`, + ); + await expect( + tools.run(`return await page.frameLocator("#${id}").getByPlaceholder(/value/i).count();`), + ).rejects.toThrow(/regular-expression attribute matching is not supported/); + await tools.run(`await page.frameLocator("#${id}").locator("#value").fill("${id}-origin");`); + } + for (const state of await inspect()) expect(state.lastClick).toBe("first"); + await expect( + tools.run(`return [await page.frameLocator("#same").locator("#value").inputValue(), + await page.frameLocator("#cross").locator("#value").inputValue()];`), + ).resolves.toEqual(["same-origin", "cross-origin"]); + const snapshot = await tools.snapshot({ includeIframes: true }); + expect(snapshot).toContain("Same value"); + expect(snapshot).toContain("Cross value"); + expect(snapshot).toContain("Shadow-only action"); + // Confirm the cross-site frame really is an OOPIF, rather than assuming + // that every cross-origin iframe has a distinct renderer process. + const transport = stagehand.rpcClient.cdp as unknown as { + sendCommand(method: string): Promise<{ targetInfos: Array<{ type: string; url: string }> }>; + }; + const { targetInfos } = await transport.sendCommand("Target.getTargets"); + expect( + targetInfos.some( + (target) => target.type === "iframe" && target.url === `http://localhost:${port}/cross`, + ), + ).toBe(true); + expect(generate).not.toHaveBeenCalled(); + expect(tools.sessionLoss).toBeUndefined(); + } finally { + try { + await stagehand?.close(); + } finally { + try { + await browser?.close(); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + } + } +}, 60_000); diff --git a/packages/integrations/core/package.json b/packages/integrations/core/package.json index 88d771559..d81a79eef 100644 --- a/packages/integrations/core/package.json +++ b/packages/integrations/core/package.json @@ -29,7 +29,8 @@ "build": "tsdown", "test": "pnpm run build && vitest run --root ../../.. packages/integrations/core/tests", "test:unit": "vitest run --root ../../.. packages/integrations/core/tests", - "typecheck": "tsc --noEmit -p tsconfig.json" + "typecheck": "tsc --noEmit -p tsconfig.json", + "test:browser": "vitest run --root ../../.. --config packages/integrations/core/vitest.browser.config.ts" }, "dependencies": { "@browserbasehq/stagehand": "workspace:*", @@ -38,6 +39,7 @@ }, "devDependencies": { "@types/node": "catalog:", + "playwright": ">=1.55.1 <1.57.0", "tsdown": "catalog:", "typescript": "catalog:", "vitest": "catalog:" diff --git a/packages/integrations/core/src/facade/contract.ts b/packages/integrations/core/src/facade/contract.ts index fcf27ed22..d11bc74fd 100644 --- a/packages/integrations/core/src/facade/contract.ts +++ b/packages/integrations/core/src/facade/contract.ts @@ -202,7 +202,7 @@ export function facadeAgentInstructions(env: NodeJS.ProcessEnv = process.env): s */ export const FACADE_AGENT_INSTRUCTIONS = `Browser tool surface: Stagehand Playwright facade. You control one persistent browser through exactly three tools: -- run: execute JavaScript against an initialized Playwright page, context, and browser (page.goto, page.locator(selector).click()/fill(), page.getByRole(...), page.evaluate(...), and the supported Playwright-shaped API). Use await directly and return JSON-serializable values so you can inspect progress. Alternatively, pass snapshot actions. +- run: execute JavaScript against an initialized Playwright page, context, and browser (page.goto, page.locator(selector).click()/fill(), page.getByRole(...), page.evaluate(...), page.waitForURL(...), and the supported Playwright-shaped API). Use await directly and return JSON-serializable values so you can inspect progress. Alternatively, pass snapshot actions. - snapshot: inspect the active page's accessibility tree and hydrate bracketed element IDs for run actions. - screenshot: inspect the rendered page visually. diff --git a/packages/integrations/core/src/facade/index.ts b/packages/integrations/core/src/facade/index.ts index c78f8df1f..9a9271d8b 100644 --- a/packages/integrations/core/src/facade/index.ts +++ b/packages/integrations/core/src/facade/index.ts @@ -37,6 +37,7 @@ export { } from "./contract.js"; export { StagehandFacadeTools, + StagehandFacadeSessionLostError, type StagehandFacadeRunReport, type StagehandFacadeToolsOptions, } from "./tools.js"; diff --git a/packages/integrations/core/src/facade/runtime.ts b/packages/integrations/core/src/facade/runtime.ts index ef6d17105..f4d9faf75 100644 --- a/packages/integrations/core/src/facade/runtime.ts +++ b/packages/integrations/core/src/facade/runtime.ts @@ -27,7 +27,24 @@ type QueryStep = hasNot?: QueryStep[]; visible?: boolean; } - | { kind: "nth"; index: number }; + | { kind: "nth"; index: number } + /** + * A `role` step that was resolved against the browser's accessibility tree. + * `values` are document-relative XPaths for the nodes whose role and name + * matched; the state filters are carried over from the original role step. + */ + | { + kind: "xpaths"; + values: string[]; + checked?: boolean; + disabled?: boolean; + selected?: boolean; + expanded?: boolean; + pressed?: boolean; + level?: number; + }; + +type RoleStep = Extract; type RawLocator = { click(options?: { button?: "left" | "right" | "middle"; clickCount?: number }): Promise; @@ -36,6 +53,16 @@ type RawLocator = { type(text: string, options?: { delay?: number }): Promise; selectOption(values: string | string[]): Promise; setInputFiles(files: unknown): Promise; + count(): Promise; + nth(index: number): RawLocator; + isVisible(): Promise; + isChecked(): Promise; + inputValue(): Promise; + innerText(): Promise; + innerHtml(): Promise; + textContent(): Promise; + scrollTo(percent: number): Promise; + centroid(): Promise<{ x: number; y: number }>; }; type CompatSelectOption = @@ -97,7 +124,11 @@ type RawContext = { }; }; -type BatchStagehandRuntime = { page: RawPage; context: RawContext }; +type BatchStagehandRuntime = { + page: RawPage; + context: RawContext; + evaluateWithShadowRoots?(pageId: string, functionSource: string): Promise; +}; export type PlaywrightCompatTelemetry = { calls: Record; @@ -119,7 +150,7 @@ export type PlaywrightCompatRuntime = { * extension service worker with Function#toString. */ export type PlaywrightCompatRuntimeOptions = { - /** Host-owned pages excluded from the agent context and page events. */ + /** Pages the host keeps for itself (the facade's keeper tab); never surfaced to agent code. */ hiddenPageIds?: string[]; }; @@ -148,6 +179,139 @@ export async function createPlaywrightCompatRuntime( ? { kind: "regexp", source: value.source, flags: value.flags } : { kind: "string", value: String(value), exact }; + // --------------------------------------------------------------------------- + // getByRole fallback through the accessibility tree. + // + // The in-page role matcher reimplements accessible-name computation and + // disagrees with Chrome's on real sites (descendant aria-label / alt / svg + // titles, labelledby across shadow roots, custom elements). When a plan that + // contains a role step matches nothing in the DOM, resolve the role step + // against `page.snapshot()` — the same accessibility tree the `snapshot` + // tool shows the agent — and re-run the plan with those nodes' XPaths. + // --------------------------------------------------------------------------- + + /** Playwright role → roles as they appear in Stagehand's formatted tree. */ + const ACCESSIBILITY_ROLE_ALIASES: Record = { + img: ["image", "img"], + image: ["image", "img"], + textbox: ["textbox", "searchbox"], + cell: ["cell", "gridcell"], + gridcell: ["gridcell", "cell"], + }; + + const ACCESSIBILITY_FALLBACK_CACHE_TTL_MS = 750; + + type AccessibilityTreeNode = { id: string; role: string; name: string }; + + const parseAccessibilityTree = (formattedTree: string): AccessibilityTreeNode[] => { + const nodes: AccessibilityTreeNode[] = []; + for (const rawLine of formattedTree.split("\n")) { + const line = rawLine.match(/^\s*\[([^\]]+)\]\s+(.*)$/u); + if (!line) continue; + let rest = line[2] ?? ""; + // Trailing state flags rendered by formatStateFlags. + rest = rest.replace(/(?:\s\[(?:selected|checked)\])+$/u, ""); + const separator = rest.indexOf(": "); + const roleToken = separator === -1 ? rest : rest.slice(0, separator); + const name = separator === -1 ? "" : rest.slice(separator + 2); + // "scrollable, html" style lines carry the role before the comma. + const role = roleToken.split(",")[0]?.trim() ?? ""; + if (!role) continue; + nodes.push({ id: line[1] ?? "", role, name }); + } + return nodes; + }; + + const matchesAccessibleName = (value: string, expected: JsonMatcher): boolean => { + const normalized = value.replace(/\s+/gu, " ").trim(); + if (expected.kind === "regexp") { + return new RegExp(expected.source, expected.flags).test(normalized); + } + const target = expected.value.replace(/\s+/gu, " ").trim(); + return expected.exact + ? normalized === target + : normalized.toLocaleLowerCase().includes(target.toLocaleLowerCase()); + }; + + const planHasRoleStep = (plan: QueryStep[]): boolean => + plan.some( + (step) => + step.kind === "role" || + (step.kind === "filter" && + ((step.has && planHasRoleStep(step.has)) || + (step.hasNot && planHasRoleStep(step.hasNot)))), + ); + + const resolveRoleStepWithTree = ( + step: RoleStep, + nodes: AccessibilityTreeNode[], + xpathMap: Record, + ): QueryStep => { + const roles = new Set(ACCESSIBILITY_ROLE_ALIASES[step.role] ?? [step.role]); + const values = nodes + .filter( + (node) => + roles.has(node.role) && (!step.name || matchesAccessibleName(node.name, step.name)), + ) + .map((node) => xpathMap[node.id]) + .filter((xpath): xpath is string => typeof xpath === "string" && xpath.length > 0); + if (values.length === 0) record("misses", "getByRole.accessibilityTree:noCandidates"); + const { kind: _kind, role: _role, name: _name, includeHidden: _hidden, ...state } = step; + return { kind: "xpaths", values, ...state }; + }; + + /** + * Resolve role steps recursively, including has/hasNot filters. Empty XPath + * sets preserve negative-filter semantics when no role candidate exists. + * Return null only when no tree is available or the plan contains no roles. + */ + const resolvePlanWithAccessibilityTree = async ( + page: RawPage, + plan: QueryStep[], + ): Promise => { + if (typeof page.snapshot !== "function") { + record("misses", "getByRole.accessibilityTree:snapshotUnavailable"); + return null; + } + let snapshot: unknown; + try { + snapshot = await page.snapshot({ includeIframes: false }); + } catch { + record("misses", "getByRole.accessibilityTree:snapshotError"); + return null; + } + const tree = snapshot as { formattedTree?: unknown; xpathMap?: unknown } | null; + if ( + !tree || + typeof tree.formattedTree !== "string" || + !tree.xpathMap || + typeof tree.xpathMap !== "object" + ) { + record("misses", "getByRole.accessibilityTree:noTree"); + return null; + } + const nodes = parseAccessibilityTree(tree.formattedTree); + const xpathMap = tree.xpathMap as Record; + let replaced = false; + const resolve = (steps: QueryStep[]): QueryStep[] => + steps.map((step) => { + if (step.kind === "role") { + replaced = true; + return resolveRoleStepWithTree(step, nodes, xpathMap); + } + if (step.kind === "filter") { + return { + ...step, + ...(step.has ? { has: resolve(step.has) } : {}), + ...(step.hasNot ? { hasNot: resolve(step.hasNot) } : {}), + }; + } + return step; + }); + const resolved = resolve(plan); + return replaced ? resolved : null; + }; + const unsupported = (surface: string, method: PropertyKey): never => { const name = `${surface}.${String(method)}`; record("misses", name); @@ -166,39 +330,44 @@ export async function createPlaywrightCompatRuntime( // This function is serialized independently by Stagehand page.evaluate, so // every query helper must remain nested inside it rather than closing over // the callback-batch scope. - async function executeQueryInPage(input: { - plan?: QueryStep[]; - operation: - | "inspect" - | "tag" - | "tagAll" - | "untag" - | "textContent" - | "innerText" - | "innerHTML" - | "inputValue" - | "isChecked" - | "isDisabled" - | "isEnabled" - | "getAttribute" - | "boundingBox" - | "focus" - | "blur" - | "selectText" - | "domClick" - | "scrollIntoView" - | "allTextContents" - | "allInnerTexts" - | "evaluate" - | "evaluateAll" - | "pageContent" - | "pageEvaluateHandle" - | "elementEvaluateHandle"; - token?: string; - attribute?: string; - functionSource?: string; - argument?: unknown; - }): Promise { + async function executeQueryInPage( + input: { + plan?: QueryStep[]; + operation: + | "inspect" + | "describe" + | "tag" + | "tagAll" + | "untag" + | "textContent" + | "innerText" + | "innerHTML" + | "inputValue" + | "isChecked" + | "isDisabled" + | "isEnabled" + | "getAttribute" + | "boundingBox" + | "focus" + | "blur" + | "selectText" + | "domClick" + | "scrollIntoView" + | "allTextContents" + | "allInnerTexts" + | "evaluate" + | "evaluateAll" + | "pageContent" + | "pageEvaluateHandle" + | "elementEvaluateHandle"; + token?: string; + attribute?: string; + functionSource?: string; + argument?: unknown; + strict?: boolean; + }, + closedRoots: ShadowRoot[] = [], + ): Promise { type QueryRoot = Document | Element | ShadowRoot; const normalize = (value: string): string => value.replace(/\s+/gu, " ").trim(); @@ -226,13 +395,25 @@ export async function createPlaywrightCompatRuntime( return true; }); }; + const closedRootsByHost = new Map(closedRoots.map((root) => [root.host, root])); + const shadowRootFor = (element: Element): ShadowRoot | null => + element.shadowRoot ?? closedRootsByHost.get(element) ?? null; + const containsAcrossShadowRoots = (root: QueryRoot, element: Element): boolean => { + let node: Node | null = element; + while (node) { + if (node === root) return true; + node = node instanceof ShadowRoot ? node.host : node.parentNode; + } + return false; + }; const queryCssDeep = (root: QueryRoot, selector: string): Element[] => { const direct = [...root.querySelectorAll(selector)]; - const ownShadow = - root instanceof Element && root.shadowRoot ? queryCssDeep(root.shadowRoot, selector) : []; - const nested = [...root.querySelectorAll("*")].flatMap((element) => - element.shadowRoot ? queryCssDeep(element.shadowRoot, selector) : [], - ); + const ownRoot = root instanceof Element ? shadowRootFor(root) : null; + const ownShadow = ownRoot ? queryCssDeep(ownRoot, selector) : []; + const nested = [...root.querySelectorAll("*")].flatMap((element) => { + const shadow = shadowRootFor(element); + return shadow ? queryCssDeep(shadow, selector) : []; + }); return dedupe([...direct, ...ownShadow, ...nested]); }; const smallestTextMatches = (elements: Element[], expected: JsonMatcher): Element[] => @@ -257,6 +438,72 @@ export async function createPlaywrightCompatRuntime( } return elements; }; + /** + * Resolve an XPath produced by Stagehand's accessibility snapshot. Those + * paths are positional (`/html[1]/body[1]/x-host[1]//div[2]/button[1]`) + * and encode a shadow-root boundary as `//`, which native + * `document.evaluate` cannot follow. Mirrors the extension's + * resolveStagehandShadowHopMatches: child steps walk light-DOM children, + * a `//` step after the first walks into the host's shadow root. + */ + const resolveStagehandXPath = (expression: string): Element[] => { + const path = expression.trim().replace(/^xpath=/iu, ""); + if (!path) return []; + type Step = { hop: boolean; tag: string; index?: number }; + const steps: Step[] = []; + let cursor = 0; + while (cursor < path.length) { + let hop = false; + if (path.startsWith("//", cursor)) { + hop = true; + cursor += 2; + } else if (path[cursor] === "/") { + cursor += 1; + } + const start = cursor; + while (cursor < path.length && path[cursor] !== "/") cursor += 1; + const raw = path.slice(start, cursor).trim(); + if (!raw) continue; + const parsed = raw.match(/^([^[]+)(?:\[(\d+)\])?$/u); + if (!parsed) return queryXPath(document, path); + steps.push({ + hop, + tag: (parsed[1] ?? "*").toLowerCase(), + ...(parsed[2] ? { index: Number(parsed[2]) } : {}), + }); + } + const hasShadowHop = steps.some((step, position) => step.hop && position > 0); + if (!hasShadowHop) { + try { + return queryXPath(document, path); + } catch { + return []; + } + } + let current: Array = [document]; + for (const [position, step] of steps.entries()) { + const next: Element[] = []; + for (const root of current) { + let pool: Element[]; + if (root instanceof Document) { + pool = root.documentElement ? [root.documentElement] : []; + } else if (step.hop && position > 0) { + pool = root instanceof Element ? [...(shadowRootFor(root)?.children ?? [])] : []; + } else { + pool = [...root.children]; + } + const tagged = pool.filter( + (element) => step.tag === "*" || element.localName.toLowerCase() === step.tag, + ); + const picked = + step.index === undefined ? tagged : [tagged[step.index - 1]!].filter(Boolean); + for (const element of picked) if (!next.includes(element)) next.push(element); + } + if (!next.length) return []; + current = next; + } + return current as Element[]; + }; const splitSelectorList = (selector: string): string[] => { const parts: string[] = []; let start = 0; @@ -383,6 +630,55 @@ export async function createPlaywrightCompatRuntime( } return ""; }; + const labelNodeText = (node: Node): string => { + if ( + ["SCRIPT", "STYLE", "NOSCRIPT"].includes(node.nodeName) || + node.ownerDocument?.head?.contains(node) + ) { + return ""; + } + if (node instanceof HTMLInputElement && (node.type === "submit" || node.type === "button")) { + return node.value; + } + let text = ""; + for (const child of node.childNodes) { + if (child.nodeType === Node.TEXT_NODE) text += child.nodeValue ?? ""; + else if (child.nodeType === Node.ELEMENT_NODE) text += labelNodeText(child); + } + if (node instanceof Element) { + const shadow = shadowRootFor(node); + if (shadow) text += labelNodeText(shadow); + } + return text; + }; + // Label locators match each label separately, with labelledby > aria-label >