diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index 106f2ea..0b09d50 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -71,6 +71,8 @@ jobs: STEEL_API_KEY: ${{ secrets.STEEL_API_KEY }} HYPERBROWSER_API_KEY: ${{ secrets.HYPERBROWSER_API_KEY }} ANCHOR_API_KEY: ${{ secrets.ANCHOR_API_KEY }} + EV_APP_UUID: ${{ secrets.EV_APP_UUID }} + EV_API_KEY: ${{ secrets.EV_API_KEY }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} steps: diff --git a/README.md b/README.md index 10bb302..f966b63 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ A run of two arms on two tasks (Red Cross and Indeed) produced this summary: | `PURELY_MAIL_API_KEY`, `PURELY_MAIL_DOMAIN` | ClawBench creates a disposable [PurelyMail](https://purelymail.com) mailbox on your domain for every task, so agents can sign up and receive confirmation emails. | | `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, `CLAWBENCH_JUDGE_API_TYPE` | ClawBench's LLM judge, which checks the intercepted request against the task. Any OpenAI-compatible endpoint works with `CLAWBENCH_JUDGE_API_TYPE=openai-completions`; we use `deepseek/deepseek-v4-pro` through OpenRouter. See [ClawBench](https://github.com/TIGER-AI-Lab/ClawBench) for details. | | `BROWSERBASE_API_KEY`, `BROWSERBASE_PROJECT_ID`, `STEEL_API_KEY`, `HYPERBROWSER_API_KEY`, `ANCHOR_API_KEY` | Arms on those providers' browsers (`environment = "remote-browser"`); checked before a run that includes them | +| `EV_APP_UUID`, `EV_API_KEY` | `codex-pw-kernel-evervault`, which mints an Evervault Browser Tokens session per trial for its proxy. The key needs `browserToken:create`. | | `BRAINTRUST_API_KEY`, `BRAINTRUST_PROJECT` | Optional: publishing results | ## Running in GitHub Actions @@ -147,6 +148,7 @@ The `retail-cart` tasks on each browser provider's browser. Every arm uses the s | Arm | Browser | |---|---| | `codex-pw-kernel` | Kernel, `stealth: true` | +| `codex-pw-kernel-evervault` | Kernel, `stealth: true`, with the whole session sent through Evervault's browser proxy: a new Evervault session and Kernel custom proxy with Evervault's CA per trial | | `codex-pw-browserbase` | Browserbase, Advanced Stealth (Scale plan), CAPTCHA solving, its residential proxies | | `codex-pw-steel` | Steel, residential proxy and CAPTCHA solving | | `codex-pw-hyperbrowser` | Hyperbrowser, stealth, proxy and CAPTCHA solving | @@ -160,6 +162,19 @@ uv run bench run retail-cart-providers --attempts 10 --concurrency 4 All arms' task containers run on one machine, so keep `--concurrency` low on a GitHub runner. +### `retail-cart-geo` + +The `retail-cart` tasks reached from each site's own market, for sites outside the US. Tasks need a `country`, so pass a task file of the same shape as `suites/retail-cart/tasks.toml` with `--suite-arg tasks=`. + +| Arm | Browser | +|---|---| +| `codex-pw-kernel-geo` | Kernel, `stealth: true`, with a new residential proxy in the task's country per trial (the `geo` configurator) | +| `codex-pw-kernel-evervault-eu` | Kernel, `stealth: true`, with the whole session sent through Evervault's EU (Ireland) relay | + +```bash +uv run bench run retail-cart-geo --suite-arg tasks=path/to/tasks.toml --attempts 5 +``` + ## Extending ### Add an arm diff --git a/arms/codex-pw-kernel-evervault-eu.toml b/arms/codex-pw-kernel-evervault-eu.toml new file mode 100644 index 0000000..5c102c8 --- /dev/null +++ b/arms/codex-pw-kernel-evervault-eu.toml @@ -0,0 +1,23 @@ +# Same as codex-pw-kernel-evervault, but through Evervault's EU (Ireland) +# relay, for sites that serve European markets. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel-evervault" + +[browser.config.params] +stealth = true +timeout_seconds = 1800 + +# See codex-pw-kernel-evervault for the TTL. +[browser.config.params.evervault] +ttl = 900 +hosts = ["checkout-test.adyen.com"] +relay_host = "eu-ie.browser.relay.evervault.app" diff --git a/arms/codex-pw-kernel-evervault.toml b/arms/codex-pw-kernel-evervault.toml new file mode 100644 index 0000000..612c60c --- /dev/null +++ b/arms/codex-pw-kernel-evervault.toml @@ -0,0 +1,26 @@ +# Same as codex-pw-kernel, but the whole browser session goes through +# Evervault's browser proxy, so sites see Evervault's egress instead of +# Kernel's stealth proxy pool. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel-evervault" + +[browser.config.params] +stealth = true +timeout_seconds = 1800 + +# The proxy closes its tunnels when the mint's TTL ends. Sandbox apps cap the +# TTL at 900 seconds, the same as the agent timeout, so a trial that uses all +# of it can lose the proxy before the verifier runs. A production app allows +# up to 4 hours. No card is minted, so the host only has to be valid. +[browser.config.params.evervault] +ttl = 900 +hosts = ["checkout-test.adyen.com"] diff --git a/arms/codex-pw-kernel-geo.toml b/arms/codex-pw-kernel-geo.toml new file mode 100644 index 0000000..6054e7d --- /dev/null +++ b/arms/codex-pw-kernel-geo.toml @@ -0,0 +1,18 @@ +# Same as codex-pw-kernel, but each trial gets a new residential proxy in the +# country of the task's site (the task's `country`). +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser] +configurator = "geo" +configurator_args = { proxy = "residential" } + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800 } diff --git a/benchmarks/retail-cart-geo.toml b/benchmarks/retail-cart-geo.toml new file mode 100644 index 0000000..3c6747b --- /dev/null +++ b/benchmarks/retail-cart-geo.toml @@ -0,0 +1,6 @@ +# The retail-cart tasks reached from each site's own market: a Kernel browser +# with a residential proxy in the site's country, and a Kernel browser sent +# through Evervault's EU relay. Tasks need a `country`; pass a task file with +# `--suite-arg tasks=`. +suite = "retail-cart" +arms = ["codex-pw-kernel-geo", "codex-pw-kernel-evervault-eu"] diff --git a/benchmarks/retail-cart-providers.toml b/benchmarks/retail-cart-providers.toml index 61f9841..79c1a82 100644 --- a/benchmarks/retail-cart-providers.toml +++ b/benchmarks/retail-cart-providers.toml @@ -3,6 +3,7 @@ suite = "retail-cart" arms = [ "codex-pw-kernel", + "codex-pw-kernel-evervault", "codex-pw-browserbase", "codex-pw-steel", "codex-pw-hyperbrowser", diff --git a/configurators/geo/configure b/configurators/geo/configure new file mode 100755 index 0000000..b63a0b4 --- /dev/null +++ b/configurators/geo/configure @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Give each trial a proxy in the country of the task's site. + +The country comes from `metadata.country` in the task's task.toml, so a site +is reached from the market it serves. The proxy type is the `proxy` arg +(default `residential`). The proxy is created per trial and deleted after it. + +stdin: {"task": {"config": str, "instruction": str}, "args": {"proxy": str}} +stdout: {"browser": {...}, "steering": null, "provenance": {...}} +""" + +from __future__ import annotations + +import json +import sys +import tomllib + + +def main() -> int: + payload = json.load(sys.stdin) + metadata = tomllib.loads(payload["task"]["config"]).get("metadata", {}) + country = metadata.get("country") + if not country: + raise SystemExit(f"Task for {metadata.get('site')!r} has no metadata.country") + proxy = (payload.get("args") or {}).get("proxy", "residential") + json.dump( + { + "browser": {"proxy_create": {"type": proxy, "config": {"country": country}}}, + "steering": None, + "provenance": {"site": metadata.get("site"), "proxy": proxy, "country": country}, + }, + sys.stdout, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/environment.py b/harness/environment.py index df773f3..fb4affe 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -24,7 +24,9 @@ `RemoteBrowserEnvironment` instead runs the task container on local Docker and drives a browser from any provider in `harness.providers` over CDP, so providers are compared with the same agent, tools and container. Its config -file names the `provider` and that provider's session create `params`. +file names the `provider` and that provider's session create `params`. A +`proxy_create` entry, as a configurator writes it, is passed on in `params`; +only the Kernel providers accept it. """ from __future__ import annotations @@ -232,7 +234,9 @@ def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: config = json.loads((Path(environment_dir) / CONFIG_FILE).read_text()) self._provider_name: str = config["provider"] self._provider = providers.get(self._provider_name) - self._create_params: dict[str, Any] = config.get("params") or {} + self._create_params: dict[str, Any] = dict(config.get("params") or {}) + if PROXY_CREATE in config: + self._create_params[PROXY_CREATE] = config[PROXY_CREATE] self._browser_session_id: str | None = None super().__init__(environment_dir, *args, **kwargs) diff --git a/harness/providers.py b/harness/providers.py index 9326bfa..8da52c3 100644 --- a/harness/providers.py +++ b/harness/providers.py @@ -10,6 +10,7 @@ from __future__ import annotations import os +import uuid from dataclasses import dataclass from typing import Any @@ -34,16 +35,109 @@ async def release(self, session_id: str) -> None: class KernelProvider(Provider): + """A Kernel browser. A `proxy_create` param, a POST /proxies body, creates a + proxy for the session and deletes it after the browser, as in + harness.environment.""" + env = ("KERNEL_API_KEY",) - async def create(self, params: dict[str, Any]) -> Session: - async with AsyncKernel() as client: + def __init__(self) -> None: + self._proxies: dict[str, str] = {} + + async def _create_browser( + self, client: AsyncKernel, params: dict[str, Any], proxy: dict[str, Any] | None + ) -> Session: + from harness.environment import RUN_ID_ENV, proxy_prefix + + if proxy is None: browser = await client.browsers.create(**params) + return Session(browser.session_id, browser.cdp_ws_url) + run_id = os.environ.get(RUN_ID_ENV) or "local" + name = f"{proxy_prefix(run_id)}{uuid.uuid4().hex[:12]}" + kernel_proxy = await client.proxies.create(**proxy, name=name) + try: + browser = await client.browsers.create(**params, proxy_id=kernel_proxy.id) + except Exception: + await client.proxies.delete(kernel_proxy.id) + raise + self._proxies[browser.session_id] = kernel_proxy.id return Session(browser.session_id, browser.cdp_ws_url) + async def create(self, params: dict[str, Any]) -> Session: + params = dict(params) + proxy = params.pop("proxy_create", None) + async with AsyncKernel() as client: + return await self._create_browser(client, params, proxy) + async def release(self, session_id: str) -> None: + # The browser goes first: without its proxy it would reach sites directly. async with AsyncKernel() as client: await client.browsers.delete_by_id(session_id) + proxy_id = self._proxies.pop(session_id, None) + if proxy_id is not None: + await client.proxies.delete(proxy_id) + + +class KernelEvervaultProvider(KernelProvider): + """A Kernel browser whose whole session goes through Evervault's browser proxy. + + Each session mints its own Evervault Browser Tokens session, which carries + the proxy credentials, and creates a Kernel custom proxy from them with + Evervault's CA bundle, since the proxy terminates TLS. The mint holds one + placeholder credential: the session is only used for its proxy, and no + real value is ever revealed. `params` holds the Kernel browser create + params, plus an `evervault` table with the mint's `ttl`, the `hosts` it + may reveal to, optionally `relay_host` to use another region's relay than + the one the mint returns (the credentials work on any), and optionally + `proxy_settings` for Evervault's egress. + """ + + env = ("KERNEL_API_KEY", "EV_APP_UUID", "EV_API_KEY") + mint_url = "https://api.evervault.com/browser-tokens" + ca_url = "https://ca.evervault.com" + + def __init__(self) -> None: + super().__init__() + self._ca_bundle: str | None = None + + async def _mint(self, settings: dict[str, Any]) -> dict[str, Any]: + body: dict[str, Any] = { + "ttl": settings.get("ttl", 900), + "data": [{"id": "bench", "category": "credential.password", "value": "bench"}], + "permissions": [{"action": "decrypt", "target": host} for host in settings["hosts"]], + } + if "proxy_settings" in settings: + body["proxySettings"] = settings["proxy_settings"] + async with httpx.AsyncClient(timeout=30) as client: + if self._ca_bundle is None: + response = await client.get(self.ca_url) + response.raise_for_status() + self._ca_bundle = response.text + response = await client.post( + self.mint_url, + json=body, + auth=(os.environ["EV_APP_UUID"], os.environ["EV_API_KEY"]), + ) + # The response body carries the proxy password, so it is never logged. + if response.is_error: + raise RuntimeError(f"Evervault mint failed with HTTP {response.status_code}") + return response.json()["proxyConfiguration"] + + async def create(self, params: dict[str, Any]) -> Session: + browser_params = dict(params) + settings = browser_params.pop("evervault") + proxy = await self._mint(settings) + config = { + "host": settings.get("relay_host") or proxy["hostname"], + "port": proxy["port"], + "username": proxy["username"], + "password": proxy["password"], + "ca_bundle": self._ca_bundle, + } + async with AsyncKernel() as client: + return await self._create_browser( + client, browser_params, {"type": "custom", "protocol": "https", "config": config} + ) class HttpProvider(Provider): @@ -131,6 +225,7 @@ async def release(self, session_id: str) -> None: PROVIDERS: dict[str, Provider] = { "kernel": KernelProvider(), + "kernel-evervault": KernelEvervaultProvider(), "browserbase": BrowserbaseProvider(), "steel": SteelProvider(), "hyperbrowser": HyperbrowserProvider(), diff --git a/suites/retail-cart/suite.py b/suites/retail-cart/suite.py index 879c005..f09e838 100644 --- a/suites/retail-cart/suite.py +++ b/suites/retail-cart/suite.py @@ -13,6 +13,10 @@ itself (`cart_url`, or `cart_api` fetched from the page) and records `cart_verified` when it lists the item (`cart_match`, a case-insensitive regular expression). + +Tasks come from tasks.toml, or from another file of the same shape given as +the `tasks` suite arg. A task's optional `country` is recorded in its +task.toml metadata, where the `geo` configurator reads it. """ from __future__ import annotations @@ -52,6 +56,7 @@ class Task: cart_match: str cart_url: str | None = None cart_api: str | None = None + country: str | None = None timeout_sec: int = 900 @@ -59,12 +64,12 @@ def preflight() -> None: """Nothing to check: tasks need only the Kernel browser Harbor creates.""" -def load_tasks() -> list[Task]: - return [Task(**entry) for entry in tomllib.loads((HERE / "tasks.toml").read_text())["tasks"]] +def load_tasks(path: Path = HERE / "tasks.toml") -> list[Task]: + return [Task(**entry) for entry in tomllib.loads(path.read_text())["tasks"]] -def generate(task_ids: list[str] | None, out_dir: Path) -> list[Path]: - tasks = load_tasks() +def generate(task_ids: list[str] | None, out_dir: Path, tasks: str | None = None) -> list[Path]: + tasks = load_tasks(Path(tasks)) if tasks else load_tasks() if task_ids: known = {task.id: task for task in tasks} missing = [task_id for task_id in task_ids if task_id not in known] @@ -108,6 +113,8 @@ def _write(task: Task, task_dir: Path) -> Path: config["source"] = "retail-cart" config["task"] = {"name": f"retail-cart/{task.id}", "keywords": [task.site]} config["metadata"] = {"site": task.site, "start_url": task.start_url} + if task.country: + config["metadata"]["country"] = task.country config["environment"] = { "build_timeout_sec": 1200.0, "network_mode": "public", diff --git a/tests/test_geo.py b/tests/test_geo.py new file mode 100644 index 0000000..4f6d10b --- /dev/null +++ b/tests/test_geo.py @@ -0,0 +1,30 @@ +import json +import subprocess + +from harness.definitions import ROOT + +CONFIGURE = ROOT / "configurators/geo/configure" + + +def _run(config, args=None): + payload = {"task": {"config": config, "instruction": ""}, "args": args or {}} + return subprocess.run( + [CONFIGURE], input=json.dumps(payload), capture_output=True, text=True, check=False + ) + + +def test_the_proxy_is_in_the_task_country(): + result = _run('[metadata]\nsite = "shop.example"\ncountry = "PL"\n', {"proxy": "isp"}) + output = json.loads(result.stdout) + assert output["browser"] == {"proxy_create": {"type": "isp", "config": {"country": "PL"}}} + + +def test_a_task_without_a_country_is_refused(): + result = _run('[metadata]\nsite = "shop.example"\n') + assert result.returncode != 0 + assert "country" in result.stderr + + +def test_the_default_proxy_is_residential(): + output = json.loads(_run('[metadata]\ncountry = "GB"\n').stdout) + assert output["browser"]["proxy_create"]["type"] == "residential" diff --git a/tests/test_providers.py b/tests/test_providers.py index 9a9cbdf..95b7884 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -84,3 +84,111 @@ def test_failed_create_raises(monkeypatch): def test_unknown_provider_is_refused(): with pytest.raises(SystemExit, match="nope"): providers.get("nope") + + +class _FakeKernel: + def __init__(self, calls): + self.calls = calls + self.proxies = self + self.browsers = self + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def create(self, **kwargs): + if "type" in kwargs: + self.calls.append(("proxy.create", kwargs)) + return type("Proxy", (), {"id": "p1"}) + self.calls.append(("browser.create", kwargs)) + return type("Browser", (), {"session_id": "b1", "cdp_ws_url": "wss://k/1"}) + + async def delete(self, proxy_id): + self.calls.append(("proxy.delete", proxy_id)) + + async def delete_by_id(self, session_id): + self.calls.append(("browser.delete", session_id)) + + +def test_kernel_evervault_proxies_the_browser_through_a_minted_session(monkeypatch): + monkeypatch.setenv("EV_APP_UUID", "app_1") + monkeypatch.setenv("EV_API_KEY", "ev:key:1") + monkeypatch.setenv("BENCH_RUN_ID", "r1") + requests = [] + + def handler(request): + requests.append(request) + if request.url.host == "ca.evervault.com": + return httpx.Response(200, text="PEM") + proxy = {"hostname": "us-va.browser.relay.evervault.app", "port": 443} + return httpx.Response( + 200, json={"proxyConfiguration": {**proxy, "username": "app_1", "password": "jwe"}} + ) + + real_client = httpx.AsyncClient + monkeypatch.setattr( + providers.httpx, + "AsyncClient", + lambda **kwargs: real_client(transport=httpx.MockTransport(handler), **kwargs), + ) + calls = [] + monkeypatch.setattr(providers, "AsyncKernel", lambda: _FakeKernel(calls)) + provider = providers.KernelEvervaultProvider() + + session = asyncio.run( + provider.create({"stealth": True, "evervault": {"ttl": 1800, "hosts": ["a.example"]}}) + ) + asyncio.run(provider.release(session.id)) + + assert session == providers.Session("b1", "wss://k/1") + mint = json.loads(requests[1].content) + assert mint["ttl"] == 1800 + assert mint["permissions"] == [{"action": "decrypt", "target": "a.example"}] + proxy = calls[0][1] + assert proxy["type"] == "custom" and proxy["protocol"] == "https" + assert proxy["name"].startswith("bench-r1-") + assert proxy["config"] == { + "host": "us-va.browser.relay.evervault.app", + "port": 443, + "username": "app_1", + "password": "jwe", + "ca_bundle": "PEM", + } + assert calls[1] == ("browser.create", {"stealth": True, "proxy_id": "p1"}) + assert calls[2:] == [("browser.delete", "b1"), ("proxy.delete", "p1")] + + +def test_kernel_evervault_mint_failure_does_not_leak_the_body(monkeypatch): + monkeypatch.setenv("EV_APP_UUID", "app_1") + monkeypatch.setenv("EV_API_KEY", "ev:key:1") + + def handler(request): + if request.url.host == "ca.evervault.com": + return httpx.Response(200, text="PEM") + return httpx.Response(403, text="secret detail") + + real_client = httpx.AsyncClient + monkeypatch.setattr( + providers.httpx, + "AsyncClient", + lambda **kwargs: real_client(transport=httpx.MockTransport(handler), **kwargs), + ) + with pytest.raises(RuntimeError, match="HTTP 403") as error: + asyncio.run(providers.KernelEvervaultProvider().create({"evervault": {"hosts": ["a"]}})) + assert "secret" not in str(error.value) + + +def test_kernel_creates_and_deletes_a_proxy_per_session(monkeypatch): + calls = [] + monkeypatch.setattr(providers, "AsyncKernel", lambda: _FakeKernel(calls)) + provider = providers.KernelProvider() + proxy = {"type": "residential", "config": {"country": "GB"}} + + session = asyncio.run(provider.create({"stealth": True, "proxy_create": proxy})) + asyncio.run(provider.release(session.id)) + + assert calls[0][0] == "proxy.create" and calls[0][1]["config"] == {"country": "GB"} + assert calls[1] == ("browser.create", {"stealth": True, "proxy_id": "p1"}) + assert calls[2:] == [("browser.delete", "b1"), ("proxy.delete", "p1")] diff --git a/tests/test_retail_cart.py b/tests/test_retail_cart.py index 962c9a5..b80d626 100644 --- a/tests/test_retail_cart.py +++ b/tests/test_retail_cart.py @@ -32,6 +32,18 @@ def test_generate_writes_a_harbor_task(tmp_path): assert (task_dir / "environment/bench/finalize-mcp").is_file() +def test_a_tasks_file_and_country_can_be_given(tmp_path): + tasks = tmp_path / "tasks.toml" + tasks.write_text( + '[[tasks]]\nid = "shop"\nsite = "shop.example"\nstart_url = "https://shop.example/"\n' + 'item = "a pair of socks"\ncart_url = "https://shop.example/cart"\n' + 'cart_match = "sock"\ncountry = "GB"\n' + ) + [task_dir] = suite.generate(None, tmp_path / "out", tasks=str(tasks)) + + assert tomllib.loads((task_dir / "task.toml").read_text())["metadata"]["country"] == "GB" + + def test_unknown_task_is_refused(tmp_path): with pytest.raises(SystemExit, match="nope"): suite.generate(["nope"], tmp_path)