Skip to content

test(evals): exercise the workspace V4 CLI #2838

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
shrey150 wants to merge 1 commit into from
Draft

test(evals): exercise the workspace V4 CLI #2838

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions packages/evals/browseCliPaths.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,9 @@ export const BROWSE_CLI_BUILD_ARTIFACTS = [
];
export const BROWSE_CLI_PACKAGE_JSON = path.join(browseCliRoot, "package.json");
export const BROWSE_SKILL_SOURCE = path.join(browseCliRoot, "skills", "browse", "SKILL.md");

export function createBrowseCliSessionName(): string {
const timestamp = Date.now().toString(36);
const random = Math.random().toString(36).slice(2, 6);
return `eval-${process.pid}-${timestamp}-${random}`;
}
104 changes: 60 additions & 44 deletions packages/evals/core/tools/browse_cli.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@ import type {
ToolStartInput,
ToolStartResult,
} from "../contracts/tool.js";
import { BROWSE_CLI_BUILD_ARTIFACTS, BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js";
import {
BROWSE_CLI_BUILD_ARTIFACTS,
BROWSE_CLI_ENTRYPOINT,
createBrowseCliSessionName,
} from "../../browseCliPaths.js";
import { getRepoRootDir } from "../../runtimePaths.js";

const execFileAsync = promisify(execFile);
Expand Down Expand Up @@ -79,25 +83,38 @@ function buildSelectorQuery(selector: string): string {
`;
}

type BrowseCliPagesResult = {
pages: Array<{
type BrowseCliTabsResult = {
tabs: Array<{
index: number;
url: string;
targetId: string;
targetId?: string;
}>;
};

export function buildBrowseCliProcessArgs(
entrypoint: string,
session: string,
args: string[],
): string[] {
return [entrypoint, ...args, "--session", session];
}

export function browseCliStartupArgs(environment: ToolStartInput["environment"]): string[] {
return ["open", "about:blank", environment === "BROWSERBASE" ? "--remote" : "--local"];
}

class BrowseCliRuntime {
constructor(private readonly session: string) {}

async runJson<T>(args: string[]): Promise<T> {
async runJson<T>(args: string[], timeoutMs?: number): Promise<T> {
const { stdout, stderr } = await execFileAsync(
process.execPath,
[resolveBrowseCliEntrypoint(), "--json", "--session", this.session, ...args],
buildBrowseCliProcessArgs(resolveBrowseCliEntrypoint(), this.session, args),
{
cwd: getRepoRootDir(),
env: process.env,
maxBuffer: 10 * 1024 * 1024,
...(timeoutMs === undefined ? {} : { timeout: timeoutMs }),
},
);

Expand Down Expand Up @@ -162,7 +179,7 @@ class BrowseCliLocatorHandle implements CoreLocatorHandle {
}
}

class BrowseCliPageHandle implements CorePageHandle {
export class BrowseCliPageHandle implements CorePageHandle {
constructor(
private readonly session: BrowseCliSession,
readonly id: string,
Expand Down Expand Up @@ -208,7 +225,7 @@ class BrowseCliPageHandle implements CorePageHandle {
args.push("--wait", opts.waitUntil);
}
if (typeof opts?.timeoutMs === "number") {
args.push("-t", String(opts.timeoutMs));
args.push("--timeout", String(opts.timeoutMs));
}
const result = await this.runCommandAfterSelecting<{ url: string }>(args);
this.cachedUrl = result.url;
Expand Down Expand Up @@ -280,15 +297,15 @@ class BrowseCliPageHandle implements CorePageHandle {
type?: "png" | "jpeg";
quality?: number;
}): Promise<Buffer> {
const args = ["screenshot"];
const args = ["screenshot", "--base64"];
if (opts?.fullPage) {
args.push("-f");
args.push("--full-page");
}
if (opts?.type) {
args.push("-t", opts.type);
args.push("--type", opts.type);
}
if (typeof opts?.quality === "number") {
args.push("-q", String(opts.quality));
args.push("--quality", String(opts.quality));
}

const result = await this.runCommandAfterSelecting<{ base64: string }>(args);
Expand All @@ -310,9 +327,9 @@ class BrowseCliPageHandle implements CorePageHandle {
"wait",
"selector",
spec.selector,
"-t",
"--timeout",
String(spec.timeoutMs ?? 30_000),
"-s",
"--state",
spec.state ?? "visible",
]);
return;
Expand All @@ -324,7 +341,7 @@ class BrowseCliPageHandle implements CorePageHandle {
"wait",
"load",
spec.state,
"-t",
"--timeout",
String(spec.timeoutMs ?? 30_000),
]);
return;
Expand Down Expand Up @@ -372,7 +389,7 @@ class BrowseCliPageHandle implements CorePageHandle {
if (typeof y !== "number") {
throw new Error("click(x, y) requires both numeric coordinates");
}
await this.runCommandAfterSelecting(["click_xy", String(targetOrX), String(y)]);
await this.runCommandAfterSelecting(["mouse", "click", String(targetOrX), String(y)]);
return;
}

Expand All @@ -387,7 +404,7 @@ class BrowseCliPageHandle implements CorePageHandle {
await this.runCommandAfterSelecting(["click", this.refSelector(target.value)]);
return;
case "coords":
await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]);
await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]);
return;
default:
throw new Error(`browse_cli does not support click target kind "${target.kind}" yet`);
Expand All @@ -399,7 +416,7 @@ class BrowseCliPageHandle implements CorePageHandle {
if (typeof y !== "number") {
throw new Error("hover(x, y) requires both numeric coordinates");
}
await this.runCommandAfterSelecting(["hover", String(targetOrX), String(y)]);
await this.runCommandAfterSelecting(["mouse", "hover", String(targetOrX), String(y)]);
return;
}

Expand All @@ -409,11 +426,11 @@ class BrowseCliPageHandle implements CorePageHandle {
switch (target.kind) {
case "selector": {
const point = await this.resolveHoverPoint(target.value);
await this.runCommandAfterSelecting(["hover", String(point.x), String(point.y)]);
await this.runCommandAfterSelecting(["mouse", "hover", String(point.x), String(point.y)]);
return;
}
case "coords":
await this.runCommandAfterSelecting(["hover", String(target.x), String(target.y)]);
await this.runCommandAfterSelecting(["mouse", "hover", String(target.x), String(target.y)]);
return;
default:
throw new Error(`browse_cli does not support hover target kind "${target.kind}" yet`);
Expand All @@ -422,6 +439,7 @@ class BrowseCliPageHandle implements CorePageHandle {

async scroll(x: number, y: number, deltaX: number, deltaY: number): Promise<void> {
await this.runCommandAfterSelecting([
"mouse",
"scroll",
String(x),
String(y),
Expand Down Expand Up @@ -450,7 +468,7 @@ class BrowseCliPageHandle implements CorePageHandle {
await this.runCommandAfterSelecting(["type", text]);
return;
case "selector":
await this.runCommandAfterSelecting(["fill", target.value, text, "--no-press-enter"]);
await this.runCommandAfterSelecting(["fill", target.value, text]);
return;
default:
throw new Error(`browse_cli does not support type target kind "${target.kind}" yet`);
Expand Down Expand Up @@ -485,7 +503,7 @@ class BrowseCliPageHandle implements CorePageHandle {
await this.runCommandAfterSelecting(["press", key]);
return;
case "coords":
await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]);
await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]);
await this.runCommandAfterSelecting(["press", key]);
return;
default:
Expand Down Expand Up @@ -519,7 +537,7 @@ class BrowseCliPageHandle implements CorePageHandle {
}
}

class BrowseCliSession implements CoreSession {
export class BrowseCliSession implements CoreSession {
readonly runtime: BrowseCliRuntime;
private readonly handles = new Map<string, BrowseCliPageHandle>();
private activePageId: string | null = null;
Expand All @@ -541,9 +559,14 @@ class BrowseCliSession implements CoreSession {
return handle;
}

private async fetchPages(): Promise<BrowseCliPagesResult["pages"]> {
const result = await this.runtime.runJson<BrowseCliPagesResult>(["pages"]);
const pages = result.pages ?? [];
private async fetchPages(): Promise<Array<{ index: number; url: string; targetId: string }>> {
const result = await this.runtime.runJson<BrowseCliTabsResult>(["tab", "list"]);
const pages = (result.tabs ?? []).map((tab) => {
if (!tab.targetId) {
throw new Error(`browse tab list returned no targetId for tab index ${tab.index}`);
}
return { ...tab, targetId: tab.targetId };
});

for (const page of pages) {
this.wrap(page);
Expand Down Expand Up @@ -583,12 +606,11 @@ class BrowseCliSession implements CoreSession {
}

async newPage(url?: string): Promise<CorePageHandle> {
const args = ["newpage"];
const args = ["tab", "new"];
if (url) {
args.push(url);
}
const result = await this.runtime.runJson<{
created: boolean;
url: string;
targetId: string;
}>(args);
Expand All @@ -604,7 +626,7 @@ class BrowseCliSession implements CoreSession {
throw new Error(`Unknown page id "${pageId}"`);
}

await this.runtime.runJson(["tab_switch", String(page.index)]);
await this.runtime.runJson(["tab", "switch", page.targetId]);
this.activePageId = pageId;
}

Expand All @@ -615,7 +637,7 @@ class BrowseCliSession implements CoreSession {
throw new Error(`Unknown page id "${pageId}"`);
}

await this.runtime.runJson(["tab_close", String(page.index)]);
await this.runtime.runJson(["tab", "close", page.targetId]);
this.handles.delete(pageId);
const remaining = await this.fetchPages();
this.activePageId = remaining[0]?.targetId ?? null;
Expand All @@ -625,11 +647,7 @@ class BrowseCliSession implements CoreSession {
if (this.closed) return;
this.closed = true;

try {
await this.runtime.runJson(["stop", "--force"]);
} catch {
// best-effort only
}
await this.runtime.runJson(["stop", "--force"], 5_000);
}

async getArtifacts(): Promise<Artifact[]> {
Expand All @@ -655,10 +673,6 @@ function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMo
return "launch";
}

function createSessionName(): string {
return `evals-browse-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
}

export class BrowseCliTool implements CoreTool {
readonly id = "browse_cli";
readonly surface = "cli";
Expand Down Expand Up @@ -687,11 +701,13 @@ export class BrowseCliTool implements CoreTool {
);
}

const session = new BrowseCliSession(createSessionName());
await session.runtime.runJson([
"env",
input.environment === "BROWSERBASE" ? "remote" : "local",
]);
const session = new BrowseCliSession(createBrowseCliSessionName());
try {
await session.runtime.runJson(browseCliStartupArgs(input.environment));
} catch (error) {
await session.close().catch(() => {});
throw error;
}

return {
session,
Expand Down
16 changes: 11 additions & 5 deletions packages/evals/framework/benchHarness.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import { withHarnessAgentSpan } from "./otel.js";
import type { DiscoveredTask, TaskResult } from "./types.js";
import type { BenchMatrixRow, BenchTaskKind, Harness } from "./benchTypes.js";
import { DEFAULT_BENCH_HARNESS } from "./benchTypes.js";
import { onceAsync, registerActiveRunCleanup } from "./activeRunCleanup.js";
import type { StartupProfile, ToolSurface } from "../core/contracts/tool.js";
import type { ExternalHarnessVerifierConfig } from "./verifierAdapter.js";

Expand Down Expand Up @@ -148,6 +149,14 @@ export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<
// the adapter and the carrier.
const carrierV3 = buildVerifierCarrierV3(logger);
let toolAdapter: TAdapter | undefined;
const cleanup = onceAsync(async () => {
try {
await toolAdapter?.cleanup();
} finally {
await carrierV3.close().catch(() => {});
}
});
const unregisterCleanup = registerActiveRunCleanup(cleanup);
try {
toolAdapter = await prepareToolAdapter({
toolSurface: row.config.toolSurface,
Expand Down Expand Up @@ -180,12 +189,9 @@ export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<
);
} finally {
try {
await toolAdapter?.cleanup();
await cleanup();
} finally {
// Deregister the never-init()-ed carrier (instance registry, event
// store, logger binding) so long matrix runs don't accumulate one
// V3 object graph per task.
await carrierV3.close().catch(() => {});
unregisterCleanup();
}
}
},
Expand Down
Loading
Loading