Add retail-cart benchmark: guest add-to-cart flows with the browser REPL

.github/workflows/_run.yml

@@ -27,6 +27,21 @@ on:
         required: false
         default: ""
         type: string
+      suite_args:
+        description: Space-separated KEY=VALUE overrides of the benchmark's suite_args
+        required: false
+        default: ""
+        type: string
+      toolset_args:
+        description: Space-separated KEY=VALUE overrides of the arms' toolset_args
+        required: false
+        default: ""
+        type: string
+      attempts:
+        description: Trials per task and arm
+        required: false
+        default: "1"
+        type: string
 
 permissions:
   contents: read
@@ -79,14 +94,19 @@ jobs:
           TASKS: ${{ inputs.tasks }}
           CONCURRENCY: ${{ inputs.concurrency }}
           TOOLSET_REF: ${{ inputs.toolset_ref }}
+          SUITE_ARGS: ${{ inputs.suite_args }}
+          TOOLSET_ARGS: ${{ inputs.toolset_args }}
+          ATTEMPTS: ${{ inputs.attempts }}
         run: |
           run_id="$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
           jobs_dir="$RUNNER_TEMP/jobs"
           echo "run_file=$jobs_dir/run-$run_id.json" >>"$GITHUB_OUTPUT"
           echo "experiment=$BENCHMARK_NAME-$run_id" >>"$GITHUB_OUTPUT"
-          args=(--tasks "$TASKS" --concurrency "$CONCURRENCY" --jobs-dir "$jobs_dir" --run-id "$run_id")
+          args=(--tasks "$TASKS" --concurrency "$CONCURRENCY" --attempts "$ATTEMPTS" --jobs-dir "$jobs_dir" --run-id "$run_id")
           [[ -n "$ARMS" ]] && args+=(--arms "$ARMS")
           [[ -n "$TOOLSET_REF" ]] && args+=(--toolset-ref "$TOOLSET_REF")
+          for pair in $SUITE_ARGS; do args+=(--suite-arg "$pair"); done
+          for pair in $TOOLSET_ARGS; do args+=(--toolset-arg "$pair"); done
           uv run bench run "$BENCHMARK_NAME" "${args[@]}"
 
       - name: Report

.github/workflows/ci.yml

@@ -24,6 +24,8 @@ jobs:
       - run: uv run pytest -q
       - name: Prepare every static arm for one task
         run: uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp,claude-kernel-mcp --tasks v2-1134-chapter-finder-redcross --out "$RUNNER_TEMP/prepared"
+      - name: Prepare the static retail-cart arm
+        run: uv run bench prepare retail-cart --arms codex-repl --out "$RUNNER_TEMP/prepared-retail-cart"
 
   report:
     runs-on: ubuntu-latest

.github/workflows/retail-cart.yml

@@ -0,0 +1,45 @@
+name: Retail cart flows with the browser REPL
+
+on:
+  workflow_dispatch:
+    inputs:
+      arms:
+        description: Comma-separated arm names (empty runs every arm in benchmarks/retail-cart.toml)
+        required: false
+        default: ""
+        type: string
+      tasks:
+        description: Comma-separated task IDs, or all
+        required: false
+        default: all
+        type: string
+      attempts:
+        description: Trials per task and arm
+        required: false
+        default: "10"
+        type: string
+      concurrency:
+        description: Concurrent trials per arm
+        required: false
+        default: "16"
+        type: string
+      kernel_mcp_server_ref:
+        description: kernel/kernel-mcp-server git ref to build (empty builds main)
+        required: false
+        default: ""
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    uses: ./.github/workflows/_run.yml
+    secrets: inherit
+    with:
+      benchmark: retail-cart
+      arms: ${{ inputs.arms }}
+      tasks: ${{ inputs.tasks }}
+      attempts: ${{ inputs.attempts }}
+      concurrency: ${{ inputs.concurrency }}
+      toolset_ref: ${{ inputs.kernel_mcp_server_ref }}

README.md

@@ -119,6 +119,21 @@ The Kernel MCP server on ClawBench.
 
 ClawBench's primary score is the lenient reward. `kernel_mcp_valid` records whether the agent called Kernel MCP's `execute_playwright_code` and used its assigned browser on every call, so a failed task can be told apart from a trial that never used the toolset.
 
+### `retail-cart`
+
+Guest add-to-cart flows on 16 retail and industrial-supply sites with heavy bot protection. Each task starts on the site's home page and asks the agent to search for one item, open its product page, add it to the cart without logging in, and report the cart.
+
+| Arm | Agent | Browser |
+|---|---|---|
+| `codex-repl` | Codex, `gpt-6-luna`, Kernel MCP's `browser_repl` only | Stealth |
+| `codex-repl-site-config` | Same; `gpt-6-luna` also picks the site and proxy for the configurator | From Kernel's config registry per site, plus its guidance. Each trial gets its own proxy. |
+
+The agent ends each task by calling the `bench` MCP server's `finalize` tool with its evidence and an outcome: `success`, `failed`, `blocked` (bot protection stopped it), `website_error` or `kernel_error`. `reward` is 1 for a reported success, and `outcome_<name>` records the failure, so bot blocks are counted apart from the agent's own failures. A trial whose proxy cannot be created fails with `ProxyCreationError`, listed under infrastructure failures in the report. The verifier also saves the browser's final URL and title. Run several attempts per task to get a pass rate per site:
+
+```bash
+uv run bench run retail-cart --attempts 10
+```
+
 ## Extending
 
 ### Add an arm
@@ -140,6 +155,8 @@ model = "anthropic/claude-haiku-5-5"
 steering = true
 ```
 
+`toolset_args` are passed to the toolset; `kernel-mcp-local` takes `toolsets`, the Kernel MCP toolsets to enable (`playwright`, the default, or `repl`). `agent_kwargs` are passed to the Harbor agent.
+
 `config` takes any [`POST /browsers`](https://kernel.sh/docs/api-reference/browsers/create-a-browser-session) parameter, such as `stealth`, `proxy`, `region`, `extensions`, or `viewport`. Browsers are always headful.
 
 ### Add a configurator
@@ -151,12 +168,12 @@ stdin:  {"task": {"config": "<task.toml>", "instruction": "<instruction>"}, "arg
 stdout: {"browser": {"stealth": true}, "steering": "advice for the agent" | null, "provenance": {...}}
 ```
 
-It never sees the verifier's files. `steering` is appended to the task instruction under "Site guidance". Each task's result is recorded in `jobs/run-<id>.json`. Configurators may be nondeterministic, e.g. when they call an LLM.
+`browser` may include `proxy_create`, a [`POST /proxies`](https://kernel.sh/docs/proxies/overview) body: each trial then gets its own proxy, deleted when the trial ends. It never sees the verifier's files. `steering` is appended to the task instruction under "Site guidance". Each task's result is recorded in `jobs/run-<id>.json`. Configurators may be nondeterministic, e.g. when they call an LLM.
 
 `site-config` works like this:
 1. A model names the task's primary website.
 2. The [config registry](https://kernel.sh/docs/config-registry) looks it up. The registry is Kernel's record of which browser and proxy settings have reached a site, and what its bot protection reacted to.
-3. A model translates the recommended proxy recipe into a Kernel proxy, which is created once and reused.
+3. A model translates the recommended proxy recipe into a Kernel proxy. Each trial creates its own proxy before its browser and deletes it afterwards, so trials never share an ISP IP; `bench run` deletes any a crashed trial left behind.
 4. With `steering = true`, the registry's guidance for the site is added to the instruction.
 
 ### Add a toolset

arms/codex-repl-site-config.toml

@@ -0,0 +1,21 @@
+# Same as codex-repl, but each task's browser and proxy come from Kernel's
+# config registry for the task's site, and the registry's guidance for that
+# site is added to the instruction.
+agent = "codex"
+model = "gpt-6-luna"
+agent_version = "0.163.0"
+toolset = "kernel-mcp-local"
+
+[agent_kwargs]
+disable_web_search = true
+
+[toolset_args]
+toolsets = ["repl"]
+
+[browser]
+config = { stealth = true }
+configurator = "site-config"
+
+[browser.configurator_args]
+model = "openai/gpt-6-luna"
+steering = true

arms/codex-repl.toml

@@ -0,0 +1,16 @@
+# Codex driving the browser only through Kernel MCP's browser REPL, with a
+# fixed stealth browser.
+agent = "codex"
+model = "gpt-6-luna"
+agent_version = "0.163.0"
+toolset = "kernel-mcp-local"
+
+# Codex's own web tool would let the agent answer without the browser.
+[agent_kwargs]
+disable_web_search = true
+
+[toolset_args]
+toolsets = ["repl"]
+
+[browser]
+config = { stealth = true }

benchmarks/retail-cart.toml

@@ -0,0 +1,5 @@
+# Guest add-to-cart flows on bot-protected retail and industrial-supply sites,
+# driven through the browser REPL, with a stealth browser and with the config
+# registry's per-site settings.
+suite = "retail-cart"
+arms = ["codex-repl", "codex-repl-site-config"]

configurators/site-config/configure

@@ -5,8 +5,10 @@
    website the task runs on, if there is one.
 2. Kernel's config registry looks that site up and returns a recommended
    browser and proxy recipe, plus guidance for navigating the site.
-3. An LLM translates the proxy recipe into a Kernel proxy, which is created
-   once per distinct recipe and then reused by name.
+3. An LLM translates the proxy recipe into Kernel proxy settings. A recipe
+   that needs a proxy becomes `proxy_create`, the POST /proxies body, which
+   harness.environment creates for each trial and deletes when it ends, so
+   no two trials share a proxy's IP.
 4. The guidance becomes steering for the agent when `steering` is true.
 
 stdin:  {"task": {"config": str, "instruction": str}, "args": {"model": str, "steering": bool}}
@@ -15,7 +17,6 @@ stdout: {"browser": {...}, "steering": str | null, "provenance": {...}}
 
 from __future__ import annotations
 
-import hashlib
 import json
 import re
 import sys
@@ -94,23 +95,11 @@ def parse_proxy(answer: dict[str, Any]) -> dict[str, Any]:
     return {"mode": answer["mode"]}
 
 
-def proxy_settings(client: Kernel, model: str, recipe: dict[str, Any]) -> dict[str, Any]:
+def proxy_settings(model: str, recipe: dict[str, Any]) -> dict[str, Any]:
     proxy = ask(model, PROXY_PROMPT.format(recipe=json.dumps(recipe, indent=2)), parse_proxy)
-    if "create" not in proxy:
-        return proxy
-    body = proxy["create"]
-    digest = hashlib.sha256(json.dumps(body, sort_keys=True).encode()).hexdigest()[:12]
-    name = f"site-config-{body['type']}-{digest}"
-    for existing in client.proxies.list():
-        if existing.name == name:
-            return {"name": name}
-    try:
-        client.proxies.create(**body, name=name)
-    except Exception:
-        # Another task may have created the same proxy concurrently.
-        if not any(existing.name == name for existing in client.proxies.list()):
-            raise
-    return {"name": name}
+    if "create" in proxy:
+        return {"proxy_create": proxy["create"]}
+    return {"proxy": proxy}
 
 
 def main() -> int:
@@ -138,7 +127,7 @@ def main() -> int:
     )
     if recommendation.get("type") == "recommendation":
         output["browser"] = dict(recommendation["browser"])
-        output["browser"]["proxy"] = proxy_settings(client, model, recommendation["proxy"])
+        output["browser"].update(proxy_settings(model, recommendation["proxy"]))
     if args.get("steering", False):
         output["steering"] = lookup.get("guidance")
     json.dump(output, sys.stdout)

harness/cli.py

@@ -16,6 +16,16 @@ def _split(value: str | None) -> list[str] | None:
     return [item.strip() for item in value.split(",") if item.strip()]
 
 
+def _pairs(values: list[str]) -> dict[str, str]:
+    pairs = {}
+    for value in values:
+        key, separator, item = value.partition("=")
+        if not separator or not key:
+            raise SystemExit(f"Expected key=value, got {value!r}")
+        pairs[key] = item
+    return pairs
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(prog="bench")
     commands = parser.add_subparsers(dest="command", required=True)
@@ -28,21 +38,41 @@ def main(argv: list[str] | None = None) -> int:
         )
         command.add_argument("--tasks", default="all", help="comma-separated task IDs, or all")
         command.add_argument("--toolset-ref", help="git ref for toolsets built from source")
+        command.add_argument(
+            "--suite-arg",
+            action="append",
+            default=[],
+            metavar="KEY=VALUE",
+            help="override one of the benchmark's suite_args, e.g. split=validation",
+        )
+        command.add_argument(
+            "--toolset-arg",
+            action="append",
+            default=[],
+            metavar="KEY=VALUE",
+            help="override a toolset_args key in the arms that set it, e.g. toolsets=repl",
+        )
 
     commands.choices["prepare"].add_argument("--out", type=Path, required=True)
     run_command = commands.choices["run"]
     run_command.add_argument("--jobs-dir", type=Path, default=Path("jobs"))
     run_command.add_argument(
         "--concurrency", type=int, default=10, help="concurrent trials per arm"
     )
+    run_command.add_argument("--attempts", type=int, default=1, help="trials per task and arm")
     run_command.add_argument("--run-id", default=datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ"))
 
     args = parser.parse_args(argv)
     arms = _split(args.arms)
     tasks = _split(args.tasks)
+    suite_args = _pairs(args.suite_arg)
+    toolset_args = _pairs(args.toolset_arg)
 
     if args.command == "prepare":
-        for item in prepare(args.benchmark, arms, tasks, args.out, args.toolset_ref):
+        prepared = prepare(
+            args.benchmark, arms, tasks, args.out, args.toolset_ref, suite_args, toolset_args
+        )
+        for item in prepared:
             print(f"{item.arm.name}: {item.dataset}")
         return 0
 
@@ -56,6 +86,9 @@ def main(argv: list[str] | None = None) -> int:
             args.run_id,
             args.concurrency,
             args.toolset_ref,
+            suite_args,
+            toolset_args,
+            args.attempts,
         )
     for arm, status in statuses.items():
         print(f"{arm}: harbor exited {status} -> {args.jobs_dir / f'{arm}-{args.run_id}'}")

harness/definitions.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import importlib.util
+import sys
 import tomllib
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -26,14 +27,17 @@ class Arm:
     model: str
     toolset: str
     agent_version: str | None = None
+    agent_kwargs: dict[str, Any] = field(default_factory=dict)
     browser: Browser = field(default_factory=Browser)
+    toolset_args: dict[str, Any] = field(default_factory=dict)
 
 
 @dataclass(frozen=True)
 class Benchmark:
     name: str
     suite: str
     arms: list[str]
+    suite_args: dict[str, Any] = field(default_factory=dict)
 
 
 def _read(path: Path) -> dict[str, Any]:
@@ -44,7 +48,12 @@ def _read(path: Path) -> dict[str, Any]:
 
 def load_benchmark(name: str) -> Benchmark:
     data = _read(ROOT / "benchmarks" / f"{name}.toml")
-    return Benchmark(name=name, suite=data["suite"], arms=list(data["arms"]))
+    return Benchmark(
+        name=name,
+        suite=data["suite"],
+        arms=list(data["arms"]),
+        suite_args=data.get("suite_args", {}),
+    )
 
 
 def load_arm(name: str) -> Arm:
@@ -69,5 +78,7 @@ def _load_module(path: Path) -> ModuleType:
     )
     assert spec is not None and spec.loader is not None
     module = importlib.util.module_from_spec(spec)
+    # Registered before it runs, as an import would be, so dataclasses work.
+    sys.modules[spec.name] = module
     spec.loader.exec_module(module)
     return module