diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index e0ee4b5..106f2ea 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -65,6 +65,12 @@ jobs: CLAWBENCH_JUDGE_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} CLAWBENCH_JUDGE_MODEL: deepseek/deepseek-v4-pro CLAWBENCH_JUDGE_API_TYPE: openai-completions + # Other browser providers, for remote-browser arms. + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + STEEL_API_KEY: ${{ secrets.STEEL_API_KEY }} + HYPERBROWSER_API_KEY: ${{ secrets.HYPERBROWSER_API_KEY }} + ANCHOR_API_KEY: ${{ secrets.ANCHOR_API_KEY }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} steps: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72ff36b..828bfaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,8 @@ jobs: run: uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp,claude-kernel-mcp --tasks v2-1134-chapter-finder-redcross --out "$RUNNER_TEMP/prepared" - name: Prepare the static retail-cart arm run: uv run bench prepare retail-cart --arms codex-repl --out "$RUNNER_TEMP/prepared-retail-cart" + - name: Prepare the retail-cart provider arms + run: uv run bench prepare retail-cart-providers --out "$RUNNER_TEMP/prepared-retail-cart-providers" report: runs-on: ubuntu-latest diff --git a/.github/workflows/retail-cart-providers.yml b/.github/workflows/retail-cart-providers.yml new file mode 100644 index 0000000..affec33 --- /dev/null +++ b/.github/workflows/retail-cart-providers.yml @@ -0,0 +1,45 @@ +name: Retail cart flows on each provider's browser + +on: + workflow_dispatch: + inputs: + arms: + description: Comma-separated arm names (empty runs every arm in benchmarks/retail-cart-providers.toml) + required: false + default: "" + type: string + tasks: + description: Comma-separated task IDs, or all + required: false + default: all + type: string + attempts: + description: Trials per task and arm + required: false + default: "10" + type: string + concurrency: + description: Concurrent trials per arm. Every arm's task containers run on the one runner. + required: false + default: "4" + type: string + playwright_mcp_version: + description: "@playwright/mcp version to install (empty installs the toolset's pinned version)" + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + benchmark: + uses: ./.github/workflows/_run.yml + secrets: inherit + with: + benchmark: retail-cart-providers + arms: ${{ inputs.arms }} + tasks: ${{ inputs.tasks }} + attempts: ${{ inputs.attempts }} + concurrency: ${{ inputs.concurrency }} + toolset_ref: ${{ inputs.playwright_mcp_version }} diff --git a/README.md b/README.md index 17942f6..10bb302 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ A run of two arms on two tasks (Red Cross and Indeed) produced this summary: | `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` | Codex and Claude Code arms; the `site-config` configurator's model | | `PURELY_MAIL_API_KEY`, `PURELY_MAIL_DOMAIN` | ClawBench creates a disposable [PurelyMail](https://purelymail.com) mailbox on your domain for every task, so agents can sign up and receive confirmation emails. | | `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, `CLAWBENCH_JUDGE_API_TYPE` | ClawBench's LLM judge, which checks the intercepted request against the task. Any OpenAI-compatible endpoint works with `CLAWBENCH_JUDGE_API_TYPE=openai-completions`; we use `deepseek/deepseek-v4-pro` through OpenRouter. See [ClawBench](https://github.com/TIGER-AI-Lab/ClawBench) for details. | +| `BROWSERBASE_API_KEY`, `BROWSERBASE_PROJECT_ID`, `STEEL_API_KEY`, `HYPERBROWSER_API_KEY`, `ANCHOR_API_KEY` | Arms on those providers' browsers (`environment = "remote-browser"`); checked before a run that includes them | | `BRAINTRUST_API_KEY`, `BRAINTRUST_PROJECT` | Optional: publishing results | ## Running in GitHub Actions @@ -139,6 +140,26 @@ Run several attempts per task to get a pass rate per site: uv run bench run retail-cart --attempts 10 ``` +### `retail-cart-providers` + +The `retail-cart` tasks on each browser provider's browser. Every arm uses the same agent (Codex, `gpt-6-luna`), the same tools ([Playwright MCP](https://github.com/microsoft/playwright-mcp) attached over CDP) and the same task container on the runner's Docker, so arms differ only in the browser. Each provider gets its own anti-bot settings: + +| Arm | Browser | +|---|---| +| `codex-pw-kernel` | Kernel, `stealth: true` | +| `codex-pw-browserbase` | Browserbase, Advanced Stealth (Scale plan), CAPTCHA solving, its residential proxies | +| `codex-pw-steel` | Steel, residential proxy and CAPTCHA solving | +| `codex-pw-hyperbrowser` | Hyperbrowser, stealth, proxy and CAPTCHA solving | +| `codex-pw-anchor` | Anchor, its US proxy, extra stealth and CAPTCHA solving | + +The start page, the agent and the verifier each connect to the browser separately, so sessions are created to outlive a CDP disconnect (Browserbase's `keepAlive`, Anchor's idle timeout). Scoring is the same as `retail-cart`. + +```bash +uv run bench run retail-cart-providers --attempts 10 --concurrency 4 +``` + +All arms' task containers run on one machine, so keep `--concurrency` low on a GitHub runner. + ## Extending ### Add an arm @@ -162,6 +183,8 @@ steering = true `toolset_args` are passed to the toolset; `kernel-mcp-local` takes `toolsets`, the Kernel MCP toolsets to enable: `playwright` (the default), `repl`, `computer`, or several. `agent_kwargs` are passed to the Harbor agent. +An arm with `environment = "remote-browser"` runs the task container on local Docker instead of in the Kernel browser's VM, and its `config` names a `provider` from `harness/providers.py` and that provider's session create `params`. The environment writes the browser's CDP URL to `/run/browser/cdp-url` in the container; the `playwright-mcp` toolset connects to it. + `config` takes any [`POST /browsers`](https://kernel.sh/docs/api-reference/browsers/create-a-browser-session) parameter, such as `stealth`, `proxy`, `region`, `extensions`, or `viewport`. Browsers are always headful. ### Add a configurator diff --git a/arms/codex-pw-anchor.toml b/arms/codex-pw-anchor.toml new file mode 100644 index 0000000..3ac96c3 --- /dev/null +++ b/arms/codex-pw-anchor.toml @@ -0,0 +1,24 @@ +# Codex driving an Anchor browser through Playwright MCP over CDP, with +# Anchor's anti-bot settings: its US proxy, extra stealth and CAPTCHA solving +# (both need the proxy). +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "anchor" + +[browser.config.params.session] +proxy = { active = true, type = "anchor_proxy", country_code = "us" } +# Minutes. The idle timeout covers the gaps between the start page, agent and +# verifier connections. +timeout = { max_duration = 45, idle_timeout = 30 } + +[browser.config.params.browser] +extra_stealth = { active = true } +captcha_solver = { active = true } diff --git a/arms/codex-pw-browserbase.toml b/arms/codex-pw-browserbase.toml new file mode 100644 index 0000000..18f379a --- /dev/null +++ b/arms/codex-pw-browserbase.toml @@ -0,0 +1,21 @@ +# Codex driving a Browserbase browser through Playwright MCP over CDP, with +# Browserbase's anti-bot settings: Advanced Stealth (Scale plan), CAPTCHA +# solving and its residential proxies. keepAlive keeps the session across the +# start page, agent and verifier connections. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "browserbase" + +[browser.config.params] +keepAlive = true +timeout = 2700 +proxies = true +browserSettings = { advancedStealth = true, solveCaptchas = true } diff --git a/arms/codex-pw-hyperbrowser.toml b/arms/codex-pw-hyperbrowser.toml new file mode 100644 index 0000000..3a6879c --- /dev/null +++ b/arms/codex-pw-hyperbrowser.toml @@ -0,0 +1,14 @@ +# Codex driving a Hyperbrowser browser through Playwright MCP over CDP, with +# Hyperbrowser's anti-bot settings: stealth, proxy and CAPTCHA solving. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "hyperbrowser" +params = { useStealth = true, useProxy = true, solveCaptchas = true, timeoutMinutes = 45 } diff --git a/arms/codex-pw-kernel.toml b/arms/codex-pw-kernel.toml new file mode 100644 index 0000000..bc9b4c4 --- /dev/null +++ b/arms/codex-pw-kernel.toml @@ -0,0 +1,14 @@ +# Codex driving a Kernel stealth browser through Playwright MCP over CDP, the +# same agent, tools and container as the other providers' codex-pw-* arms. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800 } diff --git a/arms/codex-pw-steel.toml b/arms/codex-pw-steel.toml new file mode 100644 index 0000000..b78b0e6 --- /dev/null +++ b/arms/codex-pw-steel.toml @@ -0,0 +1,15 @@ +# Codex driving a Steel browser through Playwright MCP over CDP, with Steel's +# anti-bot settings: residential proxy and CAPTCHA solving. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "steel" +# timeout is in milliseconds. +params = { useProxy = true, solveCaptcha = true, timeout = 2700000 } diff --git a/benchmarks/retail-cart-providers.toml b/benchmarks/retail-cart-providers.toml new file mode 100644 index 0000000..61f9841 --- /dev/null +++ b/benchmarks/retail-cart-providers.toml @@ -0,0 +1,10 @@ +# The retail-cart tasks on each provider's browser, with the same agent, +# Playwright MCP tools and task container, so arms differ only in the browser. +suite = "retail-cart" +arms = [ + "codex-pw-kernel", + "codex-pw-browserbase", + "codex-pw-steel", + "codex-pw-hyperbrowser", + "codex-pw-anchor", +] diff --git a/harness/definitions.py b/harness/definitions.py index 57e12d4..00da835 100644 --- a/harness/definitions.py +++ b/harness/definitions.py @@ -11,6 +11,11 @@ from typing import Any ROOT = Path(__file__).resolve().parents[1] +# Harbor environment each arm `environment` runs its trials in. +ENVIRONMENTS = { + "kernel": "harness.environment:KernelTaskEnvironment", + "remote-browser": "harness.environment:RemoteBrowserEnvironment", +} @dataclass(frozen=True) @@ -26,6 +31,9 @@ class Arm: agent: str model: str toolset: str + # "kernel" runs the task container in the Kernel browser's VM; + # "remote-browser" runs it on local Docker and drives the browser over CDP. + environment: str = "kernel" agent_version: str | None = None agent_kwargs: dict[str, Any] = field(default_factory=dict) browser: Browser = field(default_factory=Browser) @@ -59,7 +67,10 @@ def load_benchmark(name: str) -> Benchmark: def load_arm(name: str) -> Arm: data = _read(ROOT / "arms" / f"{name}.toml") browser = data.pop("browser", {}) - return Arm(name=name, browser=Browser(**browser), **data) + arm = Arm(name=name, browser=Browser(**browser), **data) + if arm.environment not in ENVIRONMENTS: + raise SystemExit(f"Arm {name} has unknown environment {arm.environment!r}") + return arm def load_suite(name: str) -> ModuleType: diff --git a/harness/environment.py b/harness/environment.py index bf47afc..df773f3 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -20,6 +20,11 @@ the persona or WebGL renderer they ask for. `vm_setup` is a list of shell commands then run as root. A trial where either fails fails with `VmSetupError`. + +`RemoteBrowserEnvironment` instead runs the task container on local Docker +and drives a browser from any provider in `harness.providers` over CDP, so +providers are compared with the same agent, tools and container. Its config +file names the `provider` and that provider's session create `params`. """ from __future__ import annotations @@ -33,14 +38,20 @@ from pathlib import Path from typing import Any +from harbor.environments.docker.docker import DockerEnvironment from harbor.environments.kernel import KernelEnvironment from kernel import AsyncKernel, Kernel +from harness import providers + CONFIG_FILE = "kernel-config.json" PROXY_CREATE = "proxy_create" CHROMIUM_ENV = "chromium_env" VM_SETUP = "vm_setup" RUN_ID_ENV = "BENCH_RUN_ID" +# Where the task container finds the remote browser's CDP URL. Agents may not +# pass their environment on to the MCP servers they launch, so it is a file. +CDP_URL_FILE = "/run/browser/cdp-url" class ProxyCreationError(RuntimeError): @@ -51,6 +62,10 @@ class VmSetupError(RuntimeError): """The browser's VM could not be set up for the trial before the first page load.""" +class BrowserCreationError(RuntimeError): + """The provider could not create the trial's remote browser.""" + + SUPERVISOR_PROGRAM = """[program:chromium] command=/usr/bin/env {env} /usr/local/bin/chromium-launcher autostart=false @@ -210,3 +225,49 @@ async def stop(self, delete: bool) -> None: self._proxy_id = None except Exception as error: self.logger.error(f"Failed to delete Kernel proxy {self._proxy_id}: {error}") + + +class RemoteBrowserEnvironment(DockerEnvironment): + def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: + config = json.loads((Path(environment_dir) / CONFIG_FILE).read_text()) + self._provider_name: str = config["provider"] + self._provider = providers.get(self._provider_name) + self._create_params: dict[str, Any] = config.get("params") or {} + self._browser_session_id: str | None = None + super().__init__(environment_dir, *args, **kwargs) + + async def start(self, force_build: bool) -> None: + await super().start(force_build) + try: + session = await self._provider.create(self._create_params) + except Exception as error: + raise BrowserCreationError( + f"Could not create a {self._provider_name} browser: {error}" + ) from error + self._browser_session_id = session.id + self.logger.info(f"Created {self._provider_name} browser {session.id}") + self._persistent_env.update( + {"BROWSER_PROVIDER": self._provider_name, "BROWSER_SESSION_ID": session.id} + ) + directory = os.path.dirname(CDP_URL_FILE) + result = await self.exec( + f'mkdir -p {directory} && printf %s "$CDP_URL" > {CDP_URL_FILE} && chmod 0644 {CDP_URL_FILE}', + env={"CDP_URL": session.cdp_url}, + user="root", + ) + if result.return_code != 0: + raise BrowserCreationError(f"Could not write {CDP_URL_FILE}: {result.stderr}") + + async def stop(self, delete: bool) -> None: + try: + await super().stop(delete) + finally: + if self._browser_session_id is not None: + try: + await self._provider.release(self._browser_session_id) + self._browser_session_id = None + except Exception as error: + self.logger.error( + f"Failed to release {self._provider_name} browser " + f"{self._browser_session_id}: {error}" + ) diff --git a/harness/providers.py b/harness/providers.py new file mode 100644 index 0000000..9326bfa --- /dev/null +++ b/harness/providers.py @@ -0,0 +1,144 @@ +"""Cloud browser providers that a trial can drive over CDP. + +Each provider creates one browser session from an arm's create params and +releases it when the trial ends. The params are the provider's own session +create body, so an arm states exactly which stealth, proxy and CAPTCHA +settings it asked for. Sessions must outlive a CDP disconnect: the start page +is opened, the agent connects, and the verifier connects, each separately. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any + +import httpx +from kernel import AsyncKernel + + +@dataclass(frozen=True) +class Session: + id: str + cdp_url: str + + +class Provider: + env: tuple[str, ...] = () + + async def create(self, params: dict[str, Any]) -> Session: + raise NotImplementedError + + async def release(self, session_id: str) -> None: + raise NotImplementedError + + +class KernelProvider(Provider): + env = ("KERNEL_API_KEY",) + + async def create(self, params: dict[str, Any]) -> Session: + async with AsyncKernel() as client: + browser = await client.browsers.create(**params) + return Session(browser.session_id, browser.cdp_ws_url) + + async def release(self, session_id: str) -> None: + async with AsyncKernel() as client: + await client.browsers.delete_by_id(session_id) + + +class HttpProvider(Provider): + base_url: str + key_header: str + + def _client(self) -> httpx.AsyncClient: + return httpx.AsyncClient( + base_url=self.base_url, + headers={self.key_header: os.environ[self.env[0]]}, + timeout=120, + ) + + +class BrowserbaseProvider(HttpProvider): + env = ("BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID") + base_url = "https://api.browserbase.com/v1" + key_header = "X-BB-API-Key" + + async def create(self, params: dict[str, Any]) -> Session: + body = {"projectId": os.environ["BROWSERBASE_PROJECT_ID"], **params} + async with self._client() as client: + response = await client.post("/sessions", json=body) + response.raise_for_status() + session = response.json() + return Session(session["id"], session["connectUrl"]) + + async def release(self, session_id: str) -> None: + body = {"projectId": os.environ["BROWSERBASE_PROJECT_ID"], "status": "REQUEST_RELEASE"} + async with self._client() as client: + (await client.post(f"/sessions/{session_id}", json=body)).raise_for_status() + + +class SteelProvider(HttpProvider): + env = ("STEEL_API_KEY",) + base_url = "https://api.steel.dev/v1" + key_header = "steel-api-key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/sessions", json=params) + response.raise_for_status() + session = response.json() + url = f"wss://connect.steel.dev?apiKey={os.environ['STEEL_API_KEY']}&sessionId={session['id']}" + return Session(session["id"], url) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + (await client.post(f"/sessions/{session_id}/release")).raise_for_status() + + +class HyperbrowserProvider(HttpProvider): + env = ("HYPERBROWSER_API_KEY",) + base_url = "https://api.hyperbrowser.ai/api" + key_header = "x-api-key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/session", json=params) + response.raise_for_status() + session = response.json() + return Session(session["id"], session["wsEndpoint"]) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + (await client.put(f"/session/{session_id}/stop")).raise_for_status() + + +class AnchorProvider(HttpProvider): + env = ("ANCHOR_API_KEY",) + base_url = "https://api.anchorbrowser.io/v1" + key_header = "anchor-api-key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/sessions", json=params) + response.raise_for_status() + session = response.json()["data"] + return Session(session["id"], session["cdp_url"]) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + (await client.delete(f"/sessions/{session_id}")).raise_for_status() + + +PROVIDERS: dict[str, Provider] = { + "kernel": KernelProvider(), + "browserbase": BrowserbaseProvider(), + "steel": SteelProvider(), + "hyperbrowser": HyperbrowserProvider(), + "anchor": AnchorProvider(), +} + + +def get(name: str) -> Provider: + if name not in PROVIDERS: + raise SystemExit(f"Unknown browser provider {name!r}; known: {', '.join(PROVIDERS)}") + return PROVIDERS[name] diff --git a/harness/run.py b/harness/run.py index f18b53f..a1aa941 100644 --- a/harness/run.py +++ b/harness/run.py @@ -11,9 +11,16 @@ from pathlib import Path from typing import Any -from harness import preflight +from harness import preflight, providers from harness.configure import configure_tasks -from harness.definitions import Arm, load_arm, load_benchmark, load_suite, load_toolset +from harness.definitions import ( + ENVIRONMENTS, + Arm, + load_arm, + load_benchmark, + load_suite, + load_toolset, +) from harness.environment import RUN_ID_ENV, delete_run_proxies # Transient failures worth retrying as a whole trial: Kernel API hiccups and @@ -104,6 +111,10 @@ def run( prepared = prepare( benchmark_name, arm_names, task_ids, work_dir, toolset_ref, suite_args, toolset_args ) + for item in prepared: + if item.arm.environment == "remote-browser": + for configuration in item.configurations.values(): + preflight.require_env(*providers.get(configuration["browser"]["provider"]).env) jobs_dir.mkdir(parents=True, exist_ok=True) manifest_path = jobs_dir / f"run-{run_id}.json" @@ -146,7 +157,7 @@ def harbor_command( "--path", str(item.dataset), "--agent", arm.agent, "--model", arm.model, - "--env", "harness.environment:KernelTaskEnvironment", + "--env", ENVIRONMENTS[arm.environment], "--job-name", job_name, "--jobs-dir", str(jobs_dir), "--n-concurrent", str(concurrency), diff --git a/suites/retail-cart/image/Dockerfile b/suites/retail-cart/image/Dockerfile index 4362d77..78620f0 100644 --- a/suites/retail-cart/image/Dockerfile +++ b/suites/retail-cart/image/Dockerfile @@ -1,13 +1,18 @@ # Task image for retail-cart. The browser is the Kernel VM this container runs -# in, so the image only holds the finalize tool, the start-URL setup and the -# scorer. Toolsets append their own layers. +# in, or a remote browser reached over CDP, so the image only holds the +# finalize tool, the start-URL setup and the scorer. Toolsets append their own +# layers. FROM node:22-bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl git python3 \ && rm -rf /var/lib/apt/lists/* +# cdp-execute drives a remote browser; it needs no browser of its own. +RUN npm install --global playwright-core@1.64.1 +ENV NODE_PATH=/usr/local/lib/node_modules + COPY bench/ /opt/bench/ -RUN chmod +x /opt/bench/finalize-mcp /opt/bench/open-start-url /opt/bench/score +RUN chmod +x /opt/bench/cdp-execute /opt/bench/finalize-mcp /opt/bench/open-start-url /opt/bench/score WORKDIR /app diff --git a/suites/retail-cart/image/bench/cdp-execute b/suites/retail-cart/image/bench/cdp-execute new file mode 100755 index 0000000..17010ba --- /dev/null +++ b/suites/retail-cart/image/bench/cdp-execute @@ -0,0 +1,34 @@ +#!/usr/bin/env node +// Run Playwright code against the trial's remote browser over CDP, the way +// Kernel's playwright/execute API runs it: the code on stdin is the body of an +// async function given `page`, `context` and `browser`. Prints +// {success, result} or {success: false, error} as JSON. +const fs = require("fs"); +const { chromium } = require("playwright-core"); + +const CDP_URL_FILE = "/run/browser/cdp-url"; + +async function main() { + const code = fs.readFileSync(0, "utf8"); + const timeoutMs = Number(process.argv[2] || 60) * 1000; + const timer = setTimeout(() => { + console.log(JSON.stringify({ success: false, error: `timed out after ${timeoutMs} ms` })); + process.exit(0); + }, timeoutMs); + const browser = await chromium.connectOverCDP(fs.readFileSync(CDP_URL_FILE, "utf8").trim()); + const context = browser.contexts()[0] ?? (await browser.newContext()); + // The page the agent was last working in: the newest one that is not blank. + const pages = context.pages().filter((p) => p.url() !== "about:blank"); + const page = pages.at(-1) ?? context.pages()[0] ?? (await context.newPage()); + const run = new (Object.getPrototypeOf(async () => {}).constructor)("page", "context", "browser", code); + const result = await run(page, context, browser); + clearTimeout(timer); + console.log(JSON.stringify({ success: true, result: result ?? null })); + // Exit without closing the browser: the session belongs to the trial. + process.exit(0); +} + +main().catch((error) => { + console.log(JSON.stringify({ success: false, error: String(error && error.message ? error.message : error) })); + process.exit(0); +}); diff --git a/suites/retail-cart/image/bench/finalize-mcp b/suites/retail-cart/image/bench/finalize-mcp index 9339d7a..3d07f23 100755 --- a/suites/retail-cart/image/bench/finalize-mcp +++ b/suites/retail-cart/image/bench/finalize-mcp @@ -35,8 +35,8 @@ OUTCOMES = { "to load." ), "kernel_error": ( - "the Kernel browser or its tools kept you from finishing: the browser REPL or the " - "browser itself errored, hung, or crashed." + "the browser or its tools kept you from finishing: the browser tools or the browser " + "itself errored, hung, or crashed." ), } diff --git a/suites/retail-cart/image/bench/open-start-url b/suites/retail-cart/image/bench/open-start-url index d6dd7f3..3ee95f5 100755 --- a/suites/retail-cart/image/bench/open-start-url +++ b/suites/retail-cart/image/bench/open-start-url @@ -9,10 +9,15 @@ from __future__ import annotations import json import os +import subprocess import sys import time import urllib.error import urllib.request +from pathlib import Path + +# Written by the remote-browser environment; absent in a Kernel browser's VM. +CDP_URL_FILE = Path("/run/browser/cdp-url") CODE = """ await page.goto(%s, { waitUntil: "domcontentloaded", timeout: 45000 }); @@ -22,10 +27,29 @@ return page.url(); def main(url: str) -> int: + if CDP_URL_FILE.exists(): + output = subprocess.run( + ["/opt/bench/cdp-execute", "90"], + input=CODE % json.dumps(url), + capture_output=True, + text=True, + timeout=120, + ) + result = json.loads(output.stdout) + else: + result = execute_with_kernel(CODE % json.dumps(url)) + if not result.get("success"): + print(f"Could not open {url}: {result.get('error')}", file=sys.stderr) + return 1 + print(f"Opened {result.get('result')}") + return 0 + + +def execute_with_kernel(code: str) -> dict: base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") request = urllib.request.Request( f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", - data=json.dumps({"code": CODE % json.dumps(url), "timeout_sec": 90}).encode(), + data=json.dumps({"code": code, "timeout_sec": 90}).encode(), headers={ "Authorization": f"Bearer {os.environ['KERNEL_API_KEY']}", "Content-Type": "application/json", @@ -44,11 +68,7 @@ def main(url: str) -> int: if attempt == 4: raise time.sleep(2 * (attempt + 1)) - if not result.get("success"): - print(f"Could not open {url}: {result.get('error')}", file=sys.stderr) - return 1 - print(f"Opened {result.get('result')}") - return 0 + return result if __name__ == "__main__": diff --git a/suites/retail-cart/image/bench/score b/suites/retail-cart/image/bench/score index 175d4a8..4bcff6e 100644 --- a/suites/retail-cart/image/bench/score +++ b/suites/retail-cart/image/bench/score @@ -7,7 +7,9 @@ finalize, and `outcome_` which failure it reported, so bot blocks can be told apart from other website errors, Kernel errors and the agent's own failures. -The browser's final URL and title are saved, then the verifier checks the +The browser is the Kernel VM's, reached through Kernel's API, or a remote +browser reached over CDP when the environment wrote its URL. The browser's +final URL and title are saved, then the verifier checks the cart itself: it reads the page the agent ended on when that is already the cart, otherwise opens the task's `cart_url`, or fetches its `cart_api` from the page, and sets `cart_verified` when the text matches `cart_match`. When @@ -22,12 +24,15 @@ import json import os import re import shutil +import subprocess import urllib.parse import urllib.request from pathlib import Path REPORT = Path("/logs/agent/finalize.json") TASK = Path("/opt/bench/task.json") +# Written by the remote-browser environment; absent in a Kernel browser's VM. +CDP_URL_FILE = Path("/run/browser/cdp-url") VERIFIER = Path("/logs/verifier") FAILURES = ("failed", "blocked", "website_error", "kernel_error") @@ -69,6 +74,8 @@ return await page.evaluate(async (url) => { def execute(code: str, timeout_sec: int) -> dict: + if CDP_URL_FILE.exists(): + return execute_over_cdp(code, timeout_sec) base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") request = urllib.request.Request( f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", @@ -89,6 +96,23 @@ def execute(code: str, timeout_sec: int) -> dict: return result.get("result") or {} +def execute_over_cdp(code: str, timeout_sec: int) -> dict: + try: + output = subprocess.run( + ["/opt/bench/cdp-execute", str(timeout_sec)], + input=code, + capture_output=True, + text=True, + timeout=timeout_sec + 30, + ) + result = json.loads(output.stdout) + except (OSError, ValueError, subprocess.SubprocessError) as error: + return {"error": str(error)} + if not result.get("success"): + return {"error": result.get("error")} + return result.get("result") or {} + + def same_page(url: str, other: str) -> bool: a, b = urllib.parse.urlsplit(url), urllib.parse.urlsplit(other) return (a.netloc, a.path.rstrip("/").lower()) == (b.netloc, b.path.rstrip("/").lower()) diff --git a/tests/test_definitions.py b/tests/test_definitions.py index 2ca3d28..30fba7e 100644 --- a/tests/test_definitions.py +++ b/tests/test_definitions.py @@ -1,3 +1,6 @@ +import pytest + +from harness import definitions, providers from harness.definitions import ROOT, load_arm, load_benchmark @@ -10,3 +13,22 @@ def test_every_benchmark_and_arm_resolves(): assert (ROOT / "toolsets" / arm.toolset / "toolset.py").is_file() if arm.browser.configurator: assert (ROOT / "configurators" / arm.browser.configurator / "configure").is_file() + + +def test_provider_arms_run_on_remote_browsers(): + benchmark = load_benchmark("retail-cart-providers") + for name in benchmark.arms: + arm = load_arm(name) + assert arm.environment == "remote-browser" + assert arm.toolset == "playwright-mcp" + assert arm.browser.config["provider"] in providers.PROVIDERS + + +def test_unknown_environment_is_refused(tmp_path, monkeypatch): + (tmp_path / "arms").mkdir() + (tmp_path / "arms/bad.toml").write_text( + 'agent = "codex"\nmodel = "m"\ntoolset = "t"\nenvironment = "elsewhere"\n' + ) + monkeypatch.setattr(definitions, "ROOT", tmp_path) + with pytest.raises(SystemExit, match="elsewhere"): + load_arm("bad") diff --git a/tests/test_environment.py b/tests/test_environment.py index 2f282eb..0f0b523 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,5 +1,6 @@ import asyncio import json +import logging import pytest from harbor.environments.kernel import KernelEnvironment @@ -195,3 +196,74 @@ def test_vm_setup_commands_run_as_root_after_the_browser_is_created(tmp_path, mo asyncio.run(env._create_browser()) assert calls == [("exec", ["-c", "echo hi"], True), ("exec", ["-c", "true"], True)] + + +class FakeProvider: + env = () + + def __init__(self, fail=False): + self.fail = fail + self.calls = [] + + async def create(self, params): + self.calls.append(("create", params)) + if self.fail: + raise RuntimeError("no capacity") + return environment.providers.Session("remote-1", "wss://remote/1") + + async def release(self, session_id): + self.calls.append(("release", session_id)) + + +def _remote_environment(tmp_path, monkeypatch, provider): + from harbor.environments.base import ExecResult + from harbor.environments.docker.docker import DockerEnvironment + + calls = [] + + async def start(self, force_build): + calls.append("container started") + + async def stop(self, delete): + calls.append("container stopped") + + async def exec(self, command, env=None, user=None, **kwargs): + calls.append((command, env, user)) + return ExecResult(return_code=0) + + def init(self, environment_dir, **kwargs): + self._persistent_env = {} + self.logger = logging.getLogger("test") + + monkeypatch.setattr(DockerEnvironment, "__init__", init) + monkeypatch.setattr(DockerEnvironment, "start", start) + monkeypatch.setattr(DockerEnvironment, "stop", stop) + monkeypatch.setattr(DockerEnvironment, "exec", exec) + monkeypatch.setitem(environment.providers.PROVIDERS, "fake", provider) + (tmp_path / "kernel-config.json").write_text( + json.dumps({"provider": "fake", "params": {"stealth": True}}) + ) + return environment.RemoteBrowserEnvironment(environment_dir=tmp_path), calls + + +def test_remote_browser_is_created_after_the_container_and_released_after(tmp_path, monkeypatch): + provider = FakeProvider() + env, calls = _remote_environment(tmp_path, monkeypatch, provider) + + asyncio.run(env.start(force_build=False)) + asyncio.run(env.stop(delete=True)) + + assert calls[0] == "container started" + command, exec_env, user = calls[1] + assert environment.CDP_URL_FILE in command + assert exec_env == {"CDP_URL": "wss://remote/1"} + assert user == "root" + assert env._persistent_env["BROWSER_SESSION_ID"] == "remote-1" + assert calls[2] == "container stopped" + assert provider.calls == [("create", {"stealth": True}), ("release", "remote-1")] + + +def test_remote_browser_creation_failure_is_identifiable(tmp_path, monkeypatch): + env, _ = _remote_environment(tmp_path, monkeypatch, FakeProvider(fail=True)) + with pytest.raises(environment.BrowserCreationError, match="no capacity"): + asyncio.run(env.start(force_build=False)) diff --git a/tests/test_playwright_mcp.py b/tests/test_playwright_mcp.py new file mode 100644 index 0000000..35e8aaa --- /dev/null +++ b/tests/test_playwright_mcp.py @@ -0,0 +1,20 @@ +import tomllib + +from harness.definitions import load_toolset + + +def test_apply_adds_the_server_install_and_instruction(task_dir): + load_toolset("playwright-mcp").apply([task_dir]) + + environment = task_dir / "environment" + assert (environment / "playwright-mcp/bin/browser-mcp").is_file() + assert "npm install --global @playwright/mcp@0.0.84" in (environment / "Dockerfile").read_text() + config = tomllib.loads((task_dir / "task.toml").read_text()) + assert config["environment"]["mcp_servers"][-1]["command"] == "/usr/local/bin/browser-mcp" + assert config["metadata"]["playwright_mcp_version"] == "0.0.84" + assert "`browser` MCP server" in (task_dir / "steps/run/instruction.md").read_text() + + +def test_ref_picks_the_playwright_mcp_version(task_dir): + load_toolset("playwright-mcp").apply([task_dir], ref="0.0.80") + assert "@playwright/mcp@0.0.80" in (task_dir / "environment/Dockerfile").read_text() diff --git a/tests/test_providers.py b/tests/test_providers.py new file mode 100644 index 0000000..9a9cbdf --- /dev/null +++ b/tests/test_providers.py @@ -0,0 +1,86 @@ +import asyncio +import json + +import httpx +import pytest + +from harness import providers + + +def _serve(monkeypatch, provider, handler): + requests = [] + + def record(request): + requests.append(request) + return handler(request) + + monkeypatch.setattr( + provider, + "_client", + lambda: httpx.AsyncClient( + base_url=provider.base_url, + headers={provider.key_header: "key"}, + transport=httpx.MockTransport(record), + ), + ) + return requests + + +def test_browserbase_sends_the_project_and_releases_the_session(monkeypatch): + monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj") + provider = providers.get("browserbase") + requests = _serve( + monkeypatch, + provider, + lambda request: httpx.Response(200, json={"id": "bb1", "connectUrl": "wss://bb/1"}), + ) + + session = asyncio.run(provider.create({"keepAlive": True})) + asyncio.run(provider.release(session.id)) + + assert session == providers.Session("bb1", "wss://bb/1") + assert json.loads(requests[0].content) == {"projectId": "proj", "keepAlive": True} + assert requests[1].url.path == "/v1/sessions/bb1" + assert json.loads(requests[1].content)["status"] == "REQUEST_RELEASE" + + +def test_steel_connects_with_the_session_and_key(monkeypatch): + monkeypatch.setenv("STEEL_API_KEY", "sk") + provider = providers.get("steel") + requests = _serve(monkeypatch, provider, lambda request: httpx.Response(200, json={"id": "s1"})) + + session = asyncio.run(provider.create({"useProxy": True})) + asyncio.run(provider.release("s1")) + + assert session.cdp_url == "wss://connect.steel.dev?apiKey=sk&sessionId=s1" + assert [request.url.path for request in requests] == ["/v1/sessions", "/v1/sessions/s1/release"] + + +def test_hyperbrowser_and_anchor_read_their_cdp_urls(monkeypatch): + hyperbrowser = providers.get("hyperbrowser") + _serve( + monkeypatch, + hyperbrowser, + lambda request: httpx.Response(200, json={"id": "h1", "wsEndpoint": "wss://hb/1"}), + ) + anchor = providers.get("anchor") + _serve( + monkeypatch, + anchor, + lambda request: httpx.Response(200, json={"data": {"id": "a1", "cdp_url": "wss://a/1"}}), + ) + + assert asyncio.run(hyperbrowser.create({})) == providers.Session("h1", "wss://hb/1") + assert asyncio.run(anchor.create({})) == providers.Session("a1", "wss://a/1") + + +def test_failed_create_raises(monkeypatch): + provider = providers.get("hyperbrowser") + _serve(monkeypatch, provider, lambda request: httpx.Response(402, json={})) + with pytest.raises(httpx.HTTPStatusError): + asyncio.run(provider.create({})) + + +def test_unknown_provider_is_refused(): + with pytest.raises(SystemExit, match="nope"): + providers.get("nope") diff --git a/tests/test_run.py b/tests/test_run.py index e74d2bb..e91909a 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -67,3 +67,9 @@ def test_harbor_command_passes_agent_kwargs_as_json(tmp_path): assert "version=0.163.0" in command assert "disable_web_search=true" in command assert command[command.index("--n-attempts") + 1] == "3" + + +def test_harbor_command_picks_the_arms_environment(tmp_path): + arm = Arm(name="a", agent="codex", model="m", toolset="t", environment="remote-browser") + command = run.harbor_command(run.PreparedArm(arm, tmp_path, {}), tmp_path, "job", 1) + assert command[command.index("--env") + 1] == "harness.environment:RemoteBrowserEnvironment" diff --git a/toolsets/playwright-mcp/image/bin/browser-mcp b/toolsets/playwright-mcp/image/bin/browser-mcp new file mode 100755 index 0000000..8063fa4 --- /dev/null +++ b/toolsets/playwright-mcp/image/bin/browser-mcp @@ -0,0 +1,6 @@ +#!/bin/sh +# Playwright MCP attached over CDP to the trial's remote browser, whose URL the +# environment writes to a file before the agent starts. +set -eu +exec playwright-mcp --cdp-endpoint "$(cat /run/browser/cdp-url)" \ + --cdp-timeout 60000 --output-dir /tmp/playwright-mcp "$@" diff --git a/toolsets/playwright-mcp/image/layer.Dockerfile b/toolsets/playwright-mcp/image/layer.Dockerfile new file mode 100644 index 0000000..d3753c1 --- /dev/null +++ b/toolsets/playwright-mcp/image/layer.Dockerfile @@ -0,0 +1,3 @@ +# playwright-mcp toolset. Appended to the suite's Dockerfile, which must have +# Node.js. The toolset appends the line installing its @playwright/mcp version. +COPY playwright-mcp/bin/ /usr/local/bin/ diff --git a/toolsets/playwright-mcp/toolset.py b/toolsets/playwright-mcp/toolset.py new file mode 100644 index 0000000..c6639fe --- /dev/null +++ b/toolsets/playwright-mcp/toolset.py @@ -0,0 +1,61 @@ +"""Playwright MCP, attached over CDP to a browser from any provider. + +The same tools drive every provider's browser, so arms that differ only in +provider compare the browsers rather than the tools. Needs an arm with +`environment = "remote-browser"`, which writes the browser's CDP URL into the +task container. The agent works in the browser's existing page and context. + +Toolset ref: the @playwright/mcp version to install, default `VERSION`. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import tomlkit + +from harness.tasks import append_instruction, read_config, write_config + +VERSION = "0.0.84" +IMAGE = Path(__file__).with_name("image") + +INSTRUCTION = """ +## Browser tools + +- Use the `browser` MCP server's tools for all browser interaction. They drive a browser that is already open on the start page; keep working in it. +- Do not install or launch another browser. +""" + + +def apply(task_dirs: list[Path], ref: str | None = None) -> None: + for task_dir in task_dirs: + _apply(task_dir, ref or VERSION) + + +def _apply(task_dir: Path, version: str) -> None: + context = task_dir / "environment" / "playwright-mcp" + shutil.copytree(IMAGE / "bin", context / "bin") + dockerfile = task_dir / "environment" / "Dockerfile" + dockerfile.write_text( + dockerfile.read_text().rstrip() + + "\n\n" + + (IMAGE / "layer.Dockerfile").read_text() + + f"RUN npm install --global @playwright/mcp@{version}\n" + ) + + config = read_config(task_dir) + server = tomlkit.table() + server.update( + { + "name": "browser", + "transport": "stdio", + "command": "/usr/local/bin/browser-mcp", + "args": [], + } + ) + servers = config["environment"].setdefault("mcp_servers", tomlkit.aot()) + servers.append(server) + config.setdefault("metadata", tomlkit.table())["playwright_mcp_version"] = version + write_config(task_dir, config) + append_instruction(task_dir, INSTRUCTION)