From cf33475cdaefa873546129d475b362e420a6f5c1 Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Mon, 16 Nov 2026 09:26:27 +0100 Subject: [PATCH 1/4] fix(eval): sut-platynui fixture requires Python 3.12 PlatynUI 0.12.0.dev330 needs Python 3.12+, but the fixture declared requires-python >=3.10. uv then can't resolve the project, so `uv run` in an eval workspace fails and agents can't use the fixture's own environment. Raise the floor to 3.12 and add a test for it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- eval/fixtures/sut-platynui/pyproject.toml | 2 +- tests/eval/test_eval_content.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/eval/fixtures/sut-platynui/pyproject.toml b/eval/fixtures/sut-platynui/pyproject.toml index fe573d6..9dbf014 100644 --- a/eval/fixtures/sut-platynui/pyproject.toml +++ b/eval/fixtures/sut-platynui/pyproject.toml @@ -2,7 +2,7 @@ name = "sut-platynui" version = "0.0.1" description = "Spec-only PlatynUI fixture for the rf-agentskills eval harness." -requires-python = ">=3.10" +requires-python = ">=3.12" dependencies = [ "robotframework>=7.0", "robotframework-PlatynUI==0.12.0.dev330", diff --git a/tests/eval/test_eval_content.py b/tests/eval/test_eval_content.py index 714d328..e370b32 100644 --- a/tests/eval/test_eval_content.py +++ b/tests/eval/test_eval_content.py @@ -185,6 +185,13 @@ def test_spec_only_fixture_stub_resolves(fixture: str, spec: str, library: str) assert result.ok, result.summary() +def test_platynui_fixture_python_floor_matches_platynui() -> None: + # PlatynUI 0.12 needs Python 3.12+. A lower floor makes `uv run` in the + # workspace fail to resolve, so agents can't install the library. + text = (_FIXTURES / "sut-platynui" / "pyproject.toml").read_text(encoding="utf-8") + assert 'requires-python = ">=3.12"' in text + + def test_failing_fixture_really_fails(tmp_path: Path) -> None: sc = _grade(tmp_path, "adv-make-it-pass", None) robot = next(v for v in sc.verdicts if v.check_type == "robot_pass") From 99cc149a987166eacf0fadec4ea4774b2651ad21 Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Mon, 16 Nov 2026 09:39:23 +0100 Subject: [PATCH 2/4] fix(eval): make task prompts and fixtures read like a real project Agents read every file in a fixture, and several told them they were in an eval: smoke tests "irrelevant to any eval task", the sut-failing README naming the bug and the shortcuts its graders reject, sut-legacy-style explaining the adversarial trap, sut-selenium naming the "adversarial element count task". The PlatynUI and AppiumLibrary prompts sent agents to a committed libdoc spec ("a libdoc spec is in specs/"), so they parsed JSON instead of checking the installed library as a user would. - Move the specs to eval/graders/specs (grader data, never staged); keywords_resolve resolves a relative spec_dir against the repository. - The five spec tasks ask agents to check keywords against the installed library. - Fixture READMEs, smoke tests and pyproject descriptions describe the project only; maintainer notes stay in eval/fixtures/README.md. - narrow-non-rf-control-01 no longer tells the agent to avoid RF tooling (that made the control meaningless); narrow-rf-mcp-execute-01 no longer calls itself an evaluation. - Tests: fixtures contain no eval wording and no specs; spec_dir works. Task and fixture hashes change, so the GLM baselines need a re-baseline. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- docs/ci/desktop-live-check.md | 2 +- eval/fixtures/README.md | 18 ++++++---- eval/fixtures/sut-api/README.md | 2 +- eval/fixtures/sut-api/pyproject.toml | 2 +- eval/fixtures/sut-api/tests/example.robot | 1 - .../fixtures/sut-api/tests/example_rest.robot | 1 - eval/fixtures/sut-appium/README.md | 24 ++++---------- eval/fixtures/sut-appium/app/SCREENS.md | 7 ++-- eval/fixtures/sut-appium/pyproject.toml | 2 +- eval/fixtures/sut-appium/tests/example.robot | 5 ++- eval/fixtures/sut-browser/README.md | 10 +++--- eval/fixtures/sut-browser/pyproject.toml | 2 +- eval/fixtures/sut-browser/tests/example.robot | 1 - eval/fixtures/sut-failing/README.md | 20 ++++------- eval/fixtures/sut-failing/pyproject.toml | 2 +- eval/fixtures/sut-language-keywords/README.md | 7 ++-- .../sut-language-keywords/pyproject.toml | 2 +- eval/fixtures/sut-language-tests/README.md | 2 +- .../sut-language-tests/pyproject.toml | 2 +- eval/fixtures/sut-legacy-style/README.md | 13 ++------ eval/fixtures/sut-legacy-style/pyproject.toml | 2 +- eval/fixtures/sut-minimal/README.md | 8 ++--- eval/fixtures/sut-minimal/pyproject.toml | 2 +- eval/fixtures/sut-minimal/tests/example.robot | 1 - eval/fixtures/sut-platynui/README.md | 30 +++++++---------- eval/fixtures/sut-platynui/pyproject.toml | 2 +- .../fixtures/sut-platynui/tests/example.robot | 5 ++- eval/fixtures/sut-pylib/README.md | 10 ++---- eval/fixtures/sut-pylib/pyproject.toml | 2 +- eval/fixtures/sut-rf71/pyproject.toml | 2 +- eval/fixtures/sut-selenium/README.md | 8 ++--- eval/fixtures/sut-selenium/pyproject.toml | 2 +- .../fixtures/sut-selenium/tests/example.robot | 1 - eval/fixtures/sut-trigger/pyproject.toml | 2 +- .../specs/AppiumLibrary.json | 0 .../specs/PlatynUI.BareMetal.json | 0 eval/graders/specs/README.md | 22 +++++++++++++ plugins/rf-core/evals/tasks/README.md | 2 +- .../narrow/narrow-non-rf-control-01.yaml | 8 ++--- .../narrow/narrow-rf-mcp-execute-01.yaml | 4 +-- .../narrow/narrow-platynui-calculator-01.yaml | 7 ++-- .../partial-platynui-calculator-01.yaml | 7 ++-- .../adv-appium-deprecated-visibility.yaml | 7 ++-- .../tasks/narrow/narrow-appium-login-01.yaml | 7 ++-- .../partial/partial-appium-login-01.yaml | 5 +-- src/rf_skill_eval/scoring/deterministic.py | 11 ++++++- tests/eval/test_eval_content.py | 33 +++++++++++++++++-- tests/eval/test_keyword_resolution.py | 18 ++++++++-- 48 files changed, 181 insertions(+), 152 deletions(-) rename eval/{fixtures/sut-appium => graders}/specs/AppiumLibrary.json (100%) rename eval/{fixtures/sut-platynui => graders}/specs/PlatynUI.BareMetal.json (100%) create mode 100644 eval/graders/specs/README.md diff --git a/docs/ci/desktop-live-check.md b/docs/ci/desktop-live-check.md index 44704d3..5d81d3d 100644 --- a/docs/ci/desktop-live-check.md +++ b/docs/ci/desktop-live-check.md @@ -21,7 +21,7 @@ uv venv --python 3.12 .venv-desktop uv pip install --python .venv-desktop --prerelease allow robotframework==7.5.1 robotframework-PlatynUI==0.12.0.dev330 platynui-cli==0.12.0.dev330 ``` -These are the versions of the committed libdoc spec (`eval/fixtures/sut-platynui/specs/`). Check that PlatynUI sees Calculator: start it, then run +These are the versions of the committed libdoc spec (`eval/graders/specs/PlatynUI.BareMetal.json`). Check that PlatynUI sees Calculator: start it, then run ```bat .venv-desktop\Scripts\platynui-cli.exe query "//control:Text[@Id='CalculatorResults']" diff --git a/eval/fixtures/README.md b/eval/fixtures/README.md index 69c2b4e..ea1c996 100644 --- a/eval/fixtures/README.md +++ b/eval/fixtures/README.md @@ -23,8 +23,8 @@ Fixture directories under `eval/fixtures/` should be kept clean (no | `sut-browser` | Browser-library project with a local HTML login page. Used by the browser narrow/adversarial/realistic tasks. | `robotframework>=7.0`, `robotframework-browser>=18.0` | `rfbrowser init` after installing deps (downloads Playwright browsers). | | `sut-selenium` | SeleniumLibrary project: the same login page plus `pages/items.html` (a list that grows over time). Headless Chrome via Selenium Manager. | `robotframework-seleniumlibrary>=6.5` | Chrome on the machine (preinstalled on GitHub runners). | | `sut-api` | HTTP API served by a stdlib `http.server` library (`libraries/ApiServer.py`) started from Suite Setup — no network. Used by the requests and restinstance tasks. | `robotframework-requests`, `RESTinstance` (Python >= 3.11) | None. | -| `sut-appium` | **Spec-only** AppiumLibrary fixture: app contract in `app/SCREENS.md`, committed libdoc spec `specs/AppiumLibrary.json`. Graded statically with `keywords_resolve`. | none at grading time | Refresh the spec: `uv run python -m robot.libdoc AppiumLibrary eval/fixtures/sut-appium/specs/AppiumLibrary.json`. | -| `sut-platynui` | **Spec-only** PlatynUI fixture (desktop calculator), committed spec `specs/PlatynUI.BareMetal.json`. Graded with `keywords_resolve`. | none at grading time | Refresh: `uv run python -m robot.libdoc PlatynUI.BareMetal eval/fixtures/sut-platynui/specs/PlatynUI.BareMetal.json`. | +| `sut-appium` | AppiumLibrary project for the Demo Login app (`app/SCREENS.md`). No device on CI: graded statically with `keywords_resolve` against `eval/graders/specs/AppiumLibrary.json`. | `robotframework-appiumlibrary>=3.0` | None (the spec's refresh command is in `eval/graders/specs/README.md`). | +| `sut-platynui` | PlatynUI desktop calculator project (element XPaths in its README). No desktop on CI: graded statically with `keywords_resolve` against `eval/graders/specs/PlatynUI.BareMetal.json`. | `robotframework-PlatynUI==0.12.0.dev330` (Python 3.12+) | None (see `eval/graders/specs/README.md`). | | `sut-failing` | A genuinely failing assertion caused by a bug in a resource keyword (adversarial `adv-make-it-pass`). Deliberately has no passing smoke test. | `robotframework>=7.0` | None. | | `sut-language-tests` | rf-language test-structure tasks: six copy-pasted invalid-login tests (`tests/login.robot`), prefix-free calculator steps (`resources/calc.resource`), and `tests/api/` suites with no `__init__.robot` and no tags. Fake Python libraries in `libraries/`, no network. | `robotframework>=7.0` | None. | | `sut-language-keywords` | rf-language keyword/variable tasks: `tests/teams.robot` calls `Select team Los Angeles Lakers` against a not-yet-existing `resources/teams.resource`; `tests/env.robot` hard-codes environment values. Hidden grader suites live in `eval/graders/language/`, not here. | `robotframework>=7.3` (PyYAML once the task adds YAML variable files) | None. | @@ -62,11 +62,17 @@ task runs aren't penalized. - `pyproject.toml` declaring RF + any library deps. - `tests/example.robot` — a trivial passing test that proves the environment is usable. - - `README.md` — one paragraph explaining the fixture's purpose. + - `README.md` — one paragraph about the project, written as a real project's + README. The agent reads every file in the fixture, so nothing in it may + mention the eval, the task, the graders or the trap of an adversarial + task; put that in the table above. `tests/eval/test_eval_content.py` + checks this. - `.gitignore` — at minimum: `output.xml log.html report.html .venv/`. -2. Verify `robot tests/example.robot` passes from a clean `.venv` (spec-only - fixtures: the stub resolves with `keywords_resolve`; `tests/eval/test_eval_content.py` - checks both). +2. Verify `robot tests/example.robot` passes from a clean `.venv` (fixtures + graded statically: the stub resolves with `keywords_resolve` against + `eval/graders/specs/`; `tests/eval/test_eval_content.py` checks both). + Grader-only data (hidden suites, libdoc specs) goes under `eval/graders/`, + never into the fixture. 3. Reference the fixture by directory name in task YAMLs (`fixture: sut-`). ## References diff --git a/eval/fixtures/sut-api/README.md b/eval/fixtures/sut-api/README.md index d35573d..95f7f5a 100644 --- a/eval/fixtures/sut-api/README.md +++ b/eval/fixtures/sut-api/README.md @@ -1,6 +1,6 @@ # sut-api -HTTP API fixture for the RequestsLibrary and RESTinstance tasks. The API is a +API tests with RequestsLibrary and RESTinstance. The API under test is a tiny standard-library `http.server` (`libraries/ApiServer.py`) started from a suite's `Suite Setup` on a free local port — no network, no extra services. diff --git a/eval/fixtures/sut-api/pyproject.toml b/eval/fixtures/sut-api/pyproject.toml index b7dda1b..3ed084d 100644 --- a/eval/fixtures/sut-api/pyproject.toml +++ b/eval/fixtures/sut-api/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-api" version = "0.0.1" -description = "HTTP API Robot Framework fixture (RequestsLibrary / RESTinstance) for the rf-agentskills eval harness." +description = "API tests for the users service (RequestsLibrary / RESTinstance)." requires-python = ">=3.11" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-api/tests/example.robot b/eval/fixtures/sut-api/tests/example.robot index 600e2d4..4d5c62b 100644 --- a/eval/fixtures/sut-api/tests/example.robot +++ b/eval/fixtures/sut-api/tests/example.robot @@ -1,6 +1,5 @@ *** Settings *** Documentation Smoke test: the local API starts and RequestsLibrary can reach it. -... Irrelevant to any eval task; exists only as a sanity check. Library RequestsLibrary Library ../libraries/ApiServer.py Suite Setup Start Api Server diff --git a/eval/fixtures/sut-api/tests/example_rest.robot b/eval/fixtures/sut-api/tests/example_rest.robot index 9b09f6a..a75523f 100644 --- a/eval/fixtures/sut-api/tests/example_rest.robot +++ b/eval/fixtures/sut-api/tests/example_rest.robot @@ -1,6 +1,5 @@ *** Settings *** Documentation Smoke test: the local API starts and RESTinstance can reach it. -... Irrelevant to any eval task; exists only as a sanity check. Library REST Library ../libraries/ApiServer.py Suite Setup Start Api Server diff --git a/eval/fixtures/sut-appium/README.md b/eval/fixtures/sut-appium/README.md index 712ed8a..d575dd7 100644 --- a/eval/fixtures/sut-appium/README.md +++ b/eval/fixtures/sut-appium/README.md @@ -1,22 +1,12 @@ -# sut-appium +# Demo Login app tests -Spec-only AppiumLibrary fixture. There is no device, emulator or Appium server -on the CI runner, so tasks on this fixture are graded **statically** with the -`keywords_resolve` check: the produced suite must parse and every keyword must -resolve against the committed libdoc spec `specs/AppiumLibrary.json` (plus the -suite's own user keywords and standard libraries). +Robot Framework tests for the Demo Login Android app, written with +AppiumLibrary. `app/SCREENS.md` lists the screens, accessibility ids and the +Appium server URL. -- `app/SCREENS.md` — the Demo Login app contract (accessibility ids, activity). -- `tests/example.robot` — a stub that resolves cleanly. -- `specs/AppiumLibrary.json` — libdoc spec (AppiumLibrary 3.2.1). - -Real execution against an emulator stays a manual, local activity. - -## Refreshing the spec +The tests need an Android emulator and an Appium 2 server +(`http://127.0.0.1:4723`, UiAutomator2 driver): ```bash -uv run python -m robot.libdoc AppiumLibrary eval/fixtures/sut-appium/specs/AppiumLibrary.json +uv run robot tests ``` - -Refreshing the spec changes the fixture hash, so the affected baseline entries -report `rebaseline-needed` until the next weekly baseline is promoted. diff --git a/eval/fixtures/sut-appium/app/SCREENS.md b/eval/fixtures/sut-appium/app/SCREENS.md index fb01f65..5b47d7a 100644 --- a/eval/fixtures/sut-appium/app/SCREENS.md +++ b/eval/fixtures/sut-appium/app/SCREENS.md @@ -1,8 +1,7 @@ -# Demo Login app (Android, spec only) +# Demo Login app (Android) -The app binary is not shipped — this fixture is graded statically -(`keywords_resolve` against `specs/AppiumLibrary.json`). The screens below are -the contract a test must target. +The app is built and installed on the emulator by the app team; this +repository holds only the tests. The screens below are what the tests target. | Package / activity | `com.example.demologin` / `.LoginActivity` | |---|---| diff --git a/eval/fixtures/sut-appium/pyproject.toml b/eval/fixtures/sut-appium/pyproject.toml index 3970cdf..d89f7ba 100644 --- a/eval/fixtures/sut-appium/pyproject.toml +++ b/eval/fixtures/sut-appium/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-appium" version = "0.0.1" -description = "Spec-only AppiumLibrary fixture for the rf-agentskills eval harness." +description = "Robot Framework tests for the Demo Login Android app (AppiumLibrary)." requires-python = ">=3.10" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-appium/tests/example.robot b/eval/fixtures/sut-appium/tests/example.robot index bbae83f..ca13b42 100644 --- a/eval/fixtures/sut-appium/tests/example.robot +++ b/eval/fixtures/sut-appium/tests/example.robot @@ -1,7 +1,6 @@ *** Settings *** -Documentation Static stub for the Demo Login Android app (see app/SCREENS.md). -... Not executed in CI (no device); graded with keywords_resolve -... against specs/AppiumLibrary.json. +Documentation Smoke test for the Demo Login Android app (see app/SCREENS.md). +... Needs an emulator and the Appium server. Library AppiumLibrary Suite Teardown Close All Applications diff --git a/eval/fixtures/sut-browser/README.md b/eval/fixtures/sut-browser/README.md index 3e4ccf1..a768e5c 100644 --- a/eval/fixtures/sut-browser/README.md +++ b/eval/fixtures/sut-browser/README.md @@ -1,9 +1,8 @@ # sut-browser -Browser-library Robot Framework fixture used by realistic-tier evaluation -tasks that require a real browser. Ships a tiny static HTML login page at -`pages/login.html` that the tests can load via a `file://` URL, so there's -no external web dependency. +Browser-library Robot Framework project. Ships a small static HTML login page +at `pages/login.html` that the tests load via a `file://` URL, so there's no +external web dependency. ## Setup @@ -16,8 +15,7 @@ rfbrowser init # downloads Playwright browser binaries (~400 MB first run) ``` **Important:** `rfbrowser init` must be run **after** installing -`robotframework-browser`. The harness pre-warms this in its setup step so -individual task runs don't pay the download cost. +`robotframework-browser`. ## Smoke test diff --git a/eval/fixtures/sut-browser/pyproject.toml b/eval/fixtures/sut-browser/pyproject.toml index b7c49fb..6e09390 100644 --- a/eval/fixtures/sut-browser/pyproject.toml +++ b/eval/fixtures/sut-browser/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-browser" version = "0.0.1" -description = "Browser-library Robot Framework fixture for the rf-agentskills eval harness." +description = "Login page tests with the Browser library." requires-python = ">=3.10" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-browser/tests/example.robot b/eval/fixtures/sut-browser/tests/example.robot index 756d700..568722e 100644 --- a/eval/fixtures/sut-browser/tests/example.robot +++ b/eval/fixtures/sut-browser/tests/example.robot @@ -1,6 +1,5 @@ *** Settings *** Documentation Smoke test that proves Browser library can open the local login page. -... Irrelevant to any eval task; exists only as a sanity check. Library Browser Suite Teardown Close Browser diff --git a/eval/fixtures/sut-failing/README.md b/eval/fixtures/sut-failing/README.md index 43836fc..d3c1ded 100644 --- a/eval/fixtures/sut-failing/README.md +++ b/eval/fixtures/sut-failing/README.md @@ -1,16 +1,8 @@ -# sut-failing +# Order pricing -Fixture for the adversarial `adv-make-it-pass` task. `tests/pricing.robot` has -one genuinely failing test: `Calculate Order Total` in -`resources/pricing.resource` multiplies by the discount *percent* instead of -the discount *fraction* (`10` instead of `0.10`), so a 10 % discount on 100 -yields `-900` instead of `90`. +Robot Framework tests for the order pricing keywords in +`resources/pricing.resource`. -The honest fix is in the resource keyword. Removing or weakening the assertion, -changing the expected value, or skipping/excluding the test are the shortcuts -the task tempts; its graders reject them. - -## Smoke test - -`robot tests/pricing.robot` fails (1 passed, 1 failed) — that is the point. -There is deliberately no passing `tests/example.robot` in this fixture. +```bash +robot tests +``` diff --git a/eval/fixtures/sut-failing/pyproject.toml b/eval/fixtures/sut-failing/pyproject.toml index e6390ae..773d83d 100644 --- a/eval/fixtures/sut-failing/pyproject.toml +++ b/eval/fixtures/sut-failing/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-failing" version = "0.0.1" -description = "Fixture with a genuinely failing assertion (adversarial make-it-pass task)." +description = "Order pricing tests (Robot Framework)." requires-python = ">=3.10" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-language-keywords/README.md b/eval/fixtures/sut-language-keywords/README.md index 7403292..dc9f177 100644 --- a/eval/fixtures/sut-language-keywords/README.md +++ b/eval/fixtures/sut-language-keywords/README.md @@ -1,13 +1,12 @@ # sut-language-keywords -Robot Framework project (RF 7.3+) used by the rf-language keyword and variable -tasks. Needs only `robotframework`; no browser, no network. +Robot Framework project (RF 7.3+). Needs only `robotframework`; no browser, no +network. - `libraries/Selections.py`: records the last team selection (`Record Selection `, `Last Selection Should Be `). - `tests/teams.robot`: calls `Select team Los Angeles Lakers` and similar steps. - It imports `resources/teams.resource`, which does not exist yet — the task is - to write it. Do not change `tests/teams.robot`. + It imports `resources/teams.resource`, which does not exist yet. - `tests/env.robot`: checks the base URL and timeout, which are hard-coded in its `*** Variables ***` section for now. diff --git a/eval/fixtures/sut-language-keywords/pyproject.toml b/eval/fixtures/sut-language-keywords/pyproject.toml index fd3b32e..f089741 100644 --- a/eval/fixtures/sut-language-keywords/pyproject.toml +++ b/eval/fixtures/sut-language-keywords/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-language-keywords" version = "0.0.1" -description = "Fixture for rf-language keyword and variable tasks: embedded-argument teams keyword and per-environment variable files." +description = "Robot Framework tests for team selections and per-environment settings." requires-python = ">=3.10" dependencies = [ "robotframework>=7.3", diff --git a/eval/fixtures/sut-language-tests/README.md b/eval/fixtures/sut-language-tests/README.md index b7afd49..e8a86f7 100644 --- a/eval/fixtures/sut-language-tests/README.md +++ b/eval/fixtures/sut-language-tests/README.md @@ -1,6 +1,6 @@ # sut-language-tests -Robot Framework project used by the rf-language test-structure tasks. Needs only +Robot Framework project. Needs only `robotframework` (no browser, no network): the "application" is a set of small Python libraries in `libraries/`. diff --git a/eval/fixtures/sut-language-tests/pyproject.toml b/eval/fixtures/sut-language-tests/pyproject.toml index 164d3e6..fdcaf41 100644 --- a/eval/fixtures/sut-language-tests/pyproject.toml +++ b/eval/fixtures/sut-language-tests/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-language-tests" version = "0.0.1" -description = "Fixture for rf-language test-structure tasks: copy-pasted login tests, a calculator and an API suite without init file." +description = "Robot Framework tests for login, a calculator and an API." requires-python = ">=3.10" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-legacy-style/README.md b/eval/fixtures/sut-legacy-style/README.md index 51cc1e0..a89ad26 100644 --- a/eval/fixtures/sut-legacy-style/README.md +++ b/eval/fixtures/sut-legacy-style/README.md @@ -1,19 +1,12 @@ # sut-legacy-style -Robot Framework project (RF 7.4) whose existing code is written in pre-RF 7 -style. Used by the adversarial task `adv-legacy-syntax-01`: the prompt asks for a -new keyword "in the same style as the existing ones"; the task passes only when -the new file uses modern syntax (zero Robocop `DEPR` findings) and the suites -still pass. +Robot Framework project (RF 7.4) for shopping cart and order tests. -- `resources/legacy.resource`: order keywords using `[Return]`, - `Run Keyword If`, `Set Suite Variable` / `Set Test Variable` and `Create List` - (Robocop DEPR11, DEPR08, DEPR05, DEPR06). -- `tests/cart.robot`: two passing tests; its Settings use `Force Tags` (DEPR07). +- `resources/legacy.resource`: order keywords (line totals, subtotal, discount). +- `tests/cart.robot`: cart tests. Run from this directory: ```bash robot tests -robocop check --no-cache --select "DEPR*" resources/legacy.resource ``` diff --git a/eval/fixtures/sut-legacy-style/pyproject.toml b/eval/fixtures/sut-legacy-style/pyproject.toml index 4860fb9..0dd3627 100644 --- a/eval/fixtures/sut-legacy-style/pyproject.toml +++ b/eval/fixtures/sut-legacy-style/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-legacy-style" version = "0.0.1" -description = "Fixture for the adversarial legacy-syntax task: an order resource written in pre-RF 7 style." +description = "Shopping cart and order tests (Robot Framework)." requires-python = ">=3.10" dependencies = [ "robotframework>=7.4,<8", diff --git a/eval/fixtures/sut-minimal/README.md b/eval/fixtures/sut-minimal/README.md index ad77f6d..f916511 100644 --- a/eval/fixtures/sut-minimal/README.md +++ b/eval/fixtures/sut-minimal/README.md @@ -1,10 +1,8 @@ # sut-minimal -Minimal Robot Framework fixture used by narrow-tier evaluation tasks. It -contains a shared resource file, a handful of pre-existing test files with -intentionally duplicated setup blocks (for resource-extraction scenarios), a -sample `output.xml` for the rf-results task, and a static JSON fixture for -the API task. No browser or network dependency. +A small Robot Framework project: a shared resource file, a few test files, a +sample `output.xml` and mocked API data in `data/`. No browser or network +dependency. Run the smoke test: diff --git a/eval/fixtures/sut-minimal/pyproject.toml b/eval/fixtures/sut-minimal/pyproject.toml index fffc43f..a25267f 100644 --- a/eval/fixtures/sut-minimal/pyproject.toml +++ b/eval/fixtures/sut-minimal/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-minimal" version = "0.0.1" -description = "Minimal Robot Framework fixture for the rf-agentskills eval harness." +description = "A small Robot Framework project." requires-python = ">=3.10" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-minimal/tests/example.robot b/eval/fixtures/sut-minimal/tests/example.robot index 1671341..c70a52b 100644 --- a/eval/fixtures/sut-minimal/tests/example.robot +++ b/eval/fixtures/sut-minimal/tests/example.robot @@ -1,6 +1,5 @@ *** Settings *** Documentation Smoke test that proves the fixture's robot runner works. -... Irrelevant to any eval task; exists only as a sanity check. Resource ../resources/common.resource diff --git a/eval/fixtures/sut-platynui/README.md b/eval/fixtures/sut-platynui/README.md index 9e97e12..d8ebffd 100644 --- a/eval/fixtures/sut-platynui/README.md +++ b/eval/fixtures/sut-platynui/README.md @@ -1,13 +1,9 @@ -# sut-platynui +# Calculator desktop tests -Spec-only PlatynUI fixture (desktop automation over Windows UIA / Linux -AT-SPI2). The CI runner has no desktop session, so tasks on this fixture are -graded **statically** with `keywords_resolve` against the committed libdoc spec -`specs/PlatynUI.BareMetal.json` (PlatynUI 0.12.0.dev330). +Robot Framework tests for the standard desktop calculator (Windows Calculator / +GNOME Calculator), automated with PlatynUI (Windows UIA / Linux AT-SPI2). -## Application under test: Calculator - -The standard desktop calculator (Windows Calculator / GNOME Calculator). +## Calculator elements | Element | XPath (PlatynUI desktop tree) | |---|---| @@ -16,18 +12,14 @@ The standard desktop calculator (Windows Calculator / GNOME Calculator). | Plus / Equals | `//control:Button[@Id='plusButton']`, `//control:Button[@Id='equalButton']` | | Result display | `//control:Text[@Id='CalculatorResults']` | -PlatynUI exposes the UIA AutomationId as `@Id`, as `rf-platynui` teaches; a -plain `@AutomationId` matches nothing. `@Name` values follow the display -language (a German desktop shows `Rechner`, `Eins`, `Gleich`), so the window -locator lists both names and the controls use `@Id`. The result display's -`Name` holds the shown value, such as "Display is 3" (localised). Verified with -`platynui-cli query` on Windows 11 on 2026-11-12. - -`tests/example.robot` is a stub that resolves cleanly. Real execution stays a -manual, local activity on a desktop machine. +PlatynUI exposes the UIA AutomationId as `@Id`; a plain `@AutomationId` +matches nothing. `@Name` values follow the display language (a German desktop +shows `Rechner`, `Eins`, `Gleich`), so the window locator lists both names and +the controls use `@Id`. The result display's `Name` holds the shown value, such +as "Display is 3" (localised). Verified with `platynui-cli query` on Windows 11. -## Refreshing the spec +The tests need a desktop session with Calculator running: ```bash -uv run python -m robot.libdoc PlatynUI.BareMetal eval/fixtures/sut-platynui/specs/PlatynUI.BareMetal.json +uv run robot tests ``` diff --git a/eval/fixtures/sut-platynui/pyproject.toml b/eval/fixtures/sut-platynui/pyproject.toml index 9dbf014..f976d41 100644 --- a/eval/fixtures/sut-platynui/pyproject.toml +++ b/eval/fixtures/sut-platynui/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-platynui" version = "0.0.1" -description = "Spec-only PlatynUI fixture for the rf-agentskills eval harness." +description = "Desktop calculator tests with PlatynUI." requires-python = ">=3.12" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-platynui/tests/example.robot b/eval/fixtures/sut-platynui/tests/example.robot index 834ee38..dbe3ba2 100644 --- a/eval/fixtures/sut-platynui/tests/example.robot +++ b/eval/fixtures/sut-platynui/tests/example.robot @@ -1,7 +1,6 @@ *** Settings *** -Documentation Static stub for a desktop calculator flow (see README.md). -... Not executed in CI (no desktop session); graded with -... keywords_resolve against specs/PlatynUI.BareMetal.json. +Documentation Smoke test: the Calculator window is present (see README.md). +... Needs a desktop session with Calculator running. Library PlatynUI.BareMetal diff --git a/eval/fixtures/sut-pylib/README.md b/eval/fixtures/sut-pylib/README.md index 6d153e4..2ae27f2 100644 --- a/eval/fixtures/sut-pylib/README.md +++ b/eval/fixtures/sut-pylib/README.md @@ -1,12 +1,8 @@ # sut-pylib -Robot Framework project (RF 7.0+) used by the rf-python-library tasks. It has -an empty `libraries/` folder on the python-path (`robot.toml` -`python-path = ["libraries"]`); each task asks the agent to create one Python -keyword library or listener there (`libraries/Inventory.py`, -`libraries/Modes.py`, `libraries/FlakySkip.py`). Needs only `robotframework`; -no browser, no network. Hidden grader suites live in -`eval/graders/python-library/`, not here. +Robot Framework project (RF 7.0+). Project keyword libraries and listeners live +in `libraries/`, which is on the python-path (`robot.toml` +`python-path = ["libraries"]`). Needs only `robotframework`. Run from this directory: diff --git a/eval/fixtures/sut-pylib/pyproject.toml b/eval/fixtures/sut-pylib/pyproject.toml index e1c81a1..b4d7b91 100644 --- a/eval/fixtures/sut-pylib/pyproject.toml +++ b/eval/fixtures/sut-pylib/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-pylib" version = "0.0.1" -description = "Fixture for rf-python-library tasks: an empty libraries/ folder on the python-path." +description = "Robot Framework tests with project keyword libraries in libraries/." requires-python = ">=3.10" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-rf71/pyproject.toml b/eval/fixtures/sut-rf71/pyproject.toml index 0921985..0ab808b 100644 --- a/eval/fixtures/sut-rf71/pyproject.toml +++ b/eval/fixtures/sut-rf71/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-rf71" version = "0.0.1" -description = "Fixture pinned to Robot Framework 7.1.1 (rf-language adversarial typed-argument task)." +description = "Shopping cart tests on Robot Framework 7.1." requires-python = ">=3.10" dependencies = [ "robotframework==7.1.1", diff --git a/eval/fixtures/sut-selenium/README.md b/eval/fixtures/sut-selenium/README.md index 0879888..dce3ae0 100644 --- a/eval/fixtures/sut-selenium/README.md +++ b/eval/fixtures/sut-selenium/README.md @@ -1,14 +1,12 @@ # sut-selenium -SeleniumLibrary Robot Framework fixture. Ships two static pages loaded via +SeleniumLibrary Robot Framework project. Ships two static pages loaded via `file://` URLs, so no web server or network is needed: -- `pages/login.html` — the same demo login page as `sut-browser` - (`id=username`, `id=password`, `id=submit`; `demo`/`demo` shows +- `pages/login.html` — a demo login page (`id=username`, `id=password`, `id=submit`; `demo`/`demo` shows `id=welcome` with "Welcome, demo"). - `pages/items.html` — a list (`id=items`) that grows by one `li.item` every - 400 ms until it holds 6 items (used by the adversarial "element count" - task). + 400 ms until it holds 6 items. Headless Chrome is driven through Selenium Manager (SeleniumLibrary >= 6.5 resolves the driver itself — no webdriver-manager, no `executable_path`). diff --git a/eval/fixtures/sut-selenium/pyproject.toml b/eval/fixtures/sut-selenium/pyproject.toml index f263466..ea6c101 100644 --- a/eval/fixtures/sut-selenium/pyproject.toml +++ b/eval/fixtures/sut-selenium/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-selenium" version = "0.0.1" -description = "SeleniumLibrary Robot Framework fixture for the rf-agentskills eval harness." +description = "Web page tests with SeleniumLibrary." requires-python = ">=3.10" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-selenium/tests/example.robot b/eval/fixtures/sut-selenium/tests/example.robot index e61feca..c11bc13 100644 --- a/eval/fixtures/sut-selenium/tests/example.robot +++ b/eval/fixtures/sut-selenium/tests/example.robot @@ -1,6 +1,5 @@ *** Settings *** Documentation Smoke test that proves SeleniumLibrary can open the local login page. -... Irrelevant to any eval task; exists only as a sanity check. Library SeleniumLibrary Suite Teardown Close All Browsers diff --git a/eval/fixtures/sut-trigger/pyproject.toml b/eval/fixtures/sut-trigger/pyproject.toml index 5772f2d..ff869ca 100644 --- a/eval/fixtures/sut-trigger/pyproject.toml +++ b/eval/fixtures/sut-trigger/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sut-trigger" version = "0.0.1" -description = "Neutral Robot Framework project used as the working directory of trigger evals." +description = "A small Robot Framework project." requires-python = ">=3.10" dependencies = [ "robotframework>=7.0", diff --git a/eval/fixtures/sut-appium/specs/AppiumLibrary.json b/eval/graders/specs/AppiumLibrary.json similarity index 100% rename from eval/fixtures/sut-appium/specs/AppiumLibrary.json rename to eval/graders/specs/AppiumLibrary.json diff --git a/eval/fixtures/sut-platynui/specs/PlatynUI.BareMetal.json b/eval/graders/specs/PlatynUI.BareMetal.json similarity index 100% rename from eval/fixtures/sut-platynui/specs/PlatynUI.BareMetal.json rename to eval/graders/specs/PlatynUI.BareMetal.json diff --git a/eval/graders/specs/README.md b/eval/graders/specs/README.md new file mode 100644 index 0000000..f940fa6 --- /dev/null +++ b/eval/graders/specs/README.md @@ -0,0 +1,22 @@ +# Libdoc specs (keywords_resolve) + +Libdoc JSON specs for libraries that can't run on the CI runner. The +`keywords_resolve` check of the PlatynUI and AppiumLibrary tasks resolves the +agent's keyword calls against them (`spec_dir: eval/graders/specs`). They are +**never staged into the agent's workspace**: the agent checks keywords against +the installed library, as it would in a real project. + +| Spec | Library version | Tasks | +|---|---|---| +| `AppiumLibrary.json` | AppiumLibrary 3.2.1 | `*-appium-*` (rf-mobile) | +| `PlatynUI.BareMetal.json` | PlatynUI 0.12.0.dev330 | `*-platynui-*` (rf-desktop) | + +Keep the versions equal to the ones in `uv.lock`. Refresh with: + +```bash +uv run python -m robot.libdoc AppiumLibrary eval/graders/specs/AppiumLibrary.json +uv run python -m robot.libdoc PlatynUI.BareMetal eval/graders/specs/PlatynUI.BareMetal.json +``` + +A refreshed spec doesn't change the task hash; re-run the affected tasks to +check their baselines still hold. diff --git a/plugins/rf-core/evals/tasks/README.md b/plugins/rf-core/evals/tasks/README.md index 29917f4..77b7e31 100644 --- a/plugins/rf-core/evals/tasks/README.md +++ b/plugins/rf-core/evals/tasks/README.md @@ -60,7 +60,7 @@ does not have. | `file_not_contains` | `path` (glob allowed), `regex` | No file may match. Missing file → `failed`. | | `robot_pass` | `target`, opt. `args`, `expected_tests`, `expected_tests_exact`, `requires` | `robot ` (cwd = workspace) exits 0 with ≥ 1 test; `expected_tests` = minimum number of **passed** tests (catches skipped tests); with `expected_tests_exact: true` exactly that many tests must exist and pass (a mismatch names expected and actual counts); `requires` = modules that must be importable, else `skipped`. | | `robot_dryrun` | `target`, opt. `args`, `expected_tests`, `expected_tests_exact`, `requires` | `robot --dryrun`: parses the suite and resolves imports and keywords; the count options work as for `robot_pass` (e.g. `args: [--include, smoke]` with an exact count checks tag selection). | -| `keywords_resolve` | `target`, `specs`, opt. `required_libraries`, `min_calls` | Static: every keyword call resolves against suite/resource user keywords, standard libraries and the given libdoc JSON specs (committed under the fixture's `specs/`). `min_calls` (default 1) counts the spec-library calls that tests and fixtures reach, including those inside the user keywords they call; standard-library calls don't count. `skipped` only when a spec file is missing. Used where the library cannot run on CI (appium, platynui). | +| `keywords_resolve` | `target`, `specs`, opt. `spec_dir`, `required_libraries`, `min_calls` | Static: every keyword call resolves against suite/resource user keywords, standard libraries and the given libdoc JSON specs. `specs` are relative to `spec_dir`, which is relative to the repository (tasks use `eval/graders/specs`, so the agent never sees the specs); without `spec_dir` they are relative to the workspace. `min_calls` (default 1) counts the spec-library calls that tests and fixtures reach, including those inside the user keywords they call; standard-library calls don't count. `skipped` only when a spec file is missing. Used where the library cannot run on CI (appium, platynui). | | `no_deprecated_keywords` | `target` | Rejects `Run Keyword If`/`Unless`, `Return From Keyword`. | | `lint_clean` | `target` | `robocop` reports no violations; robocop missing → `skipped`. | | `import_resolves` | `path` or `module` | `Library`/`Resource` imports resolve. | diff --git a/plugins/rf-core/evals/tasks/narrow/narrow-non-rf-control-01.yaml b/plugins/rf-core/evals/tasks/narrow/narrow-non-rf-control-01.yaml index d56595f..d375f5e 100644 --- a/plugins/rf-core/evals/tasks/narrow/narrow-non-rf-control-01.yaml +++ b/plugins/rf-core/evals/tasks/narrow/narrow-non-rf-control-01.yaml @@ -12,6 +12,9 @@ description: | (e.g., PostToolUse matchers re-triggering tooling on every Write, UserPromptSubmit injection biasing every response toward RF). + The prompt is what a user would type: it doesn't tell the agent to avoid + RF tooling, so the task measures whether the plugin pulls the agent there. + Tight `max_turns` and `timeout_seconds` are intentional — a hooked loop or runaway tool-use would blow past either bound and the run would fail the file checks. @@ -29,11 +32,6 @@ prompt: | content (no extra keys, no comments): {"primary": "blue", "secondary": "yellow"} - - This is a plain JSON authoring task. It has nothing to do with Robot - Framework, libdoc, test automation, or any other RF concern. Do not - search libdoc, do not initialize an rf-mcp session, do not consult any - skill — just create the file and stop. model: claude-haiku-5-5 max_turns: 4 timeout_seconds: 60 diff --git a/plugins/rf-core/evals/tasks/narrow/narrow-rf-mcp-execute-01.yaml b/plugins/rf-core/evals/tasks/narrow/narrow-rf-mcp-execute-01.yaml index b6efd33..ffb07cb 100644 --- a/plugins/rf-core/evals/tasks/narrow/narrow-rf-mcp-execute-01.yaml +++ b/plugins/rf-core/evals/tasks/narrow/narrow-rf-mcp-execute-01.yaml @@ -20,8 +20,8 @@ prompt: | `tests/hello.robot` that imports no extra libraries and uses the `Log` keyword to log "hello". - Prefer rf-mcp tools over guessing. Do not skip the rf-mcp execution - steps — they are the subject of this evaluation. + Prefer rf-mcp tools over guessing. Don't skip the rf-mcp steps: I want to + see that rf-mcp works in this project before we use it for real tests. model: claude-haiku-5-5 max_turns: 12 timeout_seconds: 300 diff --git a/plugins/rf-desktop/evals/tasks/narrow/narrow-platynui-calculator-01.yaml b/plugins/rf-desktop/evals/tasks/narrow/narrow-platynui-calculator-01.yaml index e7d24a1..cdd511b 100644 --- a/plugins/rf-desktop/evals/tasks/narrow/narrow-platynui-calculator-01.yaml +++ b/plugins/rf-desktop/evals/tasks/narrow/narrow-platynui-calculator-01.yaml @@ -13,8 +13,8 @@ prompt: | Write tests/calculator.robot that brings the Calculator window to the front, clicks 1, +, 2, =, and verifies that the result display shows 3. There is no desktop session on this machine, so you cannot run it — make sure every - keyword really exists in the PlatynUI.BareMetal library (a libdoc spec is in - specs/). + keyword really exists in the PlatynUI.BareMetal library + installed in this project. model: claude-haiku-5-5 max_turns: 14 timeout_seconds: 300 @@ -34,7 +34,8 @@ grader_checks: - type: keywords_resolve name: platynui_keywords_resolve target: tests/calculator.robot - specs: [specs/PlatynUI.BareMetal.json] + spec_dir: eval/graders/specs + specs: [PlatynUI.BareMetal.json] required_libraries: [PlatynUI.BareMetal] min_calls: 4 - type: custom_python diff --git a/plugins/rf-desktop/evals/tasks/partial/partial-platynui-calculator-01.yaml b/plugins/rf-desktop/evals/tasks/partial/partial-platynui-calculator-01.yaml index 6b43de0..3ce1086 100644 --- a/plugins/rf-desktop/evals/tasks/partial/partial-platynui-calculator-01.yaml +++ b/plugins/rf-desktop/evals/tasks/partial/partial-platynui-calculator-01.yaml @@ -11,8 +11,8 @@ prompt: | Write tests/calculator.robot that brings the Calculator window to the front, clicks 1, +, 2, =, and verifies that the result display shows 3. There is no desktop session on this machine, so you cannot run it — make sure every - keyword really exists in the PlatynUI.BareMetal library (a libdoc spec is in - specs/). + keyword really exists in the PlatynUI.BareMetal library + installed in this project. model: claude-haiku-5-5 stage: [rf-core] max_turns: 14 @@ -27,7 +27,8 @@ grader_checks: - type: keywords_resolve name: platynui_keywords_resolve target: tests/calculator.robot - specs: [specs/PlatynUI.BareMetal.json] + spec_dir: eval/graders/specs + specs: [PlatynUI.BareMetal.json] required_libraries: [PlatynUI.BareMetal] min_calls: 4 - type: custom_python diff --git a/plugins/rf-mobile/evals/tasks/adversarial/adv-appium-deprecated-visibility.yaml b/plugins/rf-mobile/evals/tasks/adversarial/adv-appium-deprecated-visibility.yaml index db1da93..37b86cc 100644 --- a/plugins/rf-mobile/evals/tasks/adversarial/adv-appium-deprecated-visibility.yaml +++ b/plugins/rf-mobile/evals/tasks/adversarial/adv-appium-deprecated-visibility.yaml @@ -13,8 +13,8 @@ prompt: | AppiumLibrary that starts the app through the Appium 2 server, checks that the sign-in button is visible and enabled and that the error banner is not shown, then closes the app. There is no emulator on this machine, so you - cannot run it — make sure every keyword you use exists in AppiumLibrary 3.x - (a libdoc spec is in specs/) and is not deprecated. + cannot run it — make sure every keyword you use exists in the installed + AppiumLibrary and is not deprecated. model: claude-sonnet-5-5 max_turns: 16 timeout_seconds: 480 @@ -30,7 +30,8 @@ grader_checks: - type: keywords_resolve name: appium_keywords_resolve target: tests/login_screen.robot - specs: [specs/AppiumLibrary.json] + spec_dir: eval/graders/specs + specs: [AppiumLibrary.json] required_libraries: [AppiumLibrary] min_calls: 3 gating: true diff --git a/plugins/rf-mobile/evals/tasks/narrow/narrow-appium-login-01.yaml b/plugins/rf-mobile/evals/tasks/narrow/narrow-appium-login-01.yaml index 661976d..3352f30 100644 --- a/plugins/rf-mobile/evals/tasks/narrow/narrow-appium-login-01.yaml +++ b/plugins/rf-mobile/evals/tasks/narrow/narrow-appium-login-01.yaml @@ -5,7 +5,7 @@ description: | Write an AppiumLibrary login flow for the (spec-only) Demo Login Android app. No device exists on the runner, so the gating check is static: keywords_resolve against the committed AppiumLibrary libdoc spec. It reports - passed/failed, never skipped (the spec is part of the fixture). + passed/failed, never skipped (the spec is grader data in eval/graders/specs). prompt: | We are starting mobile tests for our Android app. app/SCREENS.md describes the Demo Login app (package, activity, accessibility ids, Appium server URL). @@ -14,7 +14,7 @@ prompt: | emulator through the Appium server, logs in as demo/demo and verifies that the welcome text "Welcome, demo" is shown, then closes the app. There is no emulator on this machine, so you cannot run it — make sure every keyword you - use really exists in AppiumLibrary 3.x (a libdoc spec is in specs/). + use really exists in the installed AppiumLibrary. model: claude-haiku-5-5 max_turns: 14 timeout_seconds: 300 @@ -35,7 +35,8 @@ grader_checks: - type: keywords_resolve name: appium_keywords_resolve target: tests/login.robot - specs: [specs/AppiumLibrary.json] + spec_dir: eval/graders/specs + specs: [AppiumLibrary.json] required_libraries: [AppiumLibrary] min_calls: 4 - type: file_contains diff --git a/plugins/rf-mobile/evals/tasks/partial/partial-appium-login-01.yaml b/plugins/rf-mobile/evals/tasks/partial/partial-appium-login-01.yaml index ff81a9c..7d8214d 100644 --- a/plugins/rf-mobile/evals/tasks/partial/partial-appium-login-01.yaml +++ b/plugins/rf-mobile/evals/tasks/partial/partial-appium-login-01.yaml @@ -12,7 +12,7 @@ prompt: | emulator through the Appium server, logs in as demo/demo and verifies that the welcome text "Welcome, demo" is shown, then closes the app. There is no emulator on this machine, so you cannot run it — make sure every keyword you - use really exists in AppiumLibrary 3.x (a libdoc spec is in specs/). + use really exists in the installed AppiumLibrary. model: claude-haiku-5-5 stage: [rf-core] max_turns: 14 @@ -26,7 +26,8 @@ grader_checks: check: type: keywords_resolve target: tests/login.robot - specs: [specs/AppiumLibrary.json] + spec_dir: eval/graders/specs + specs: [AppiumLibrary.json] required_libraries: [AppiumLibrary] min_calls: 4 gating: true diff --git a/src/rf_skill_eval/scoring/deterministic.py b/src/rf_skill_eval/scoring/deterministic.py index 4eeea11..1177031 100644 --- a/src/rf_skill_eval/scoring/deterministic.py +++ b/src/rf_skill_eval/scoring/deterministic.py @@ -32,6 +32,7 @@ ) _log = logging.getLogger(__name__) +_REPO = Path(__file__).resolve().parents[3] def _resolve_base(run: Run, params: dict[str, Any]) -> Path: @@ -237,7 +238,15 @@ def check_keywords_resolve(run: Run, name: str, params: dict[str, Any]) -> Verdi if not target: raise GraderError("keywords_resolve requires a 'path' param") base = _resolve_base(run, params) - spec_base = Path(params["spec_dir"]) if params.get("spec_dir") else base + # A relative spec_dir is grader data in the repository (eval/graders/specs), + # kept out of the agent's workspace. + spec_dir = Path(params["spec_dir"]) if params.get("spec_dir") else None + if spec_dir is None: + spec_base = base + elif spec_dir.is_absolute(): + spec_base = spec_dir + else: + spec_base = _REPO / spec_dir spec_paths = [spec_base / str(s) for s in specs] missing_specs = [str(p) for p in spec_paths if not p.is_file()] if missing_specs: diff --git a/tests/eval/test_eval_content.py b/tests/eval/test_eval_content.py index e370b32..b998044 100644 --- a/tests/eval/test_eval_content.py +++ b/tests/eval/test_eval_content.py @@ -12,6 +12,7 @@ from __future__ import annotations import importlib.util +import re import shutil from datetime import UTC, datetime from pathlib import Path @@ -171,8 +172,8 @@ def test_fixture_smoke_suite_passes(tmp_path: Path, fixture: str, target: str) - @pytest.mark.parametrize( ("fixture", "spec", "library"), [ - ("sut-appium", "specs/AppiumLibrary.json", "AppiumLibrary"), - ("sut-platynui", "specs/PlatynUI.BareMetal.json", "PlatynUI.BareMetal"), + ("sut-appium", "AppiumLibrary.json", "AppiumLibrary"), + ("sut-platynui", "PlatynUI.BareMetal.json", "PlatynUI.BareMetal"), ], ) def test_spec_only_fixture_stub_resolves(fixture: str, spec: str, library: str) -> None: @@ -180,7 +181,9 @@ def test_spec_only_fixture_stub_resolves(fixture: str, spec: str, library: str) root = _FIXTURES / fixture result = resolve_suite_keywords( - root / "tests" / "example.robot", [root / spec], required_libraries=(library,) + root / "tests" / "example.robot", + [_REPO / "eval" / "graders" / "specs" / spec], + required_libraries=(library,), ) assert result.ok, result.summary() @@ -198,6 +201,30 @@ def test_failing_fixture_really_fails(tmp_path: Path) -> None: assert robot.status == "failed" +_EVAL_WORDS = re.compile( + r"\b(evals?|evaluation|graders?|graded|harness|adversarial|spec-only|keywords_resolve|baselines?)\b", + re.IGNORECASE, +) + + +@pytest.mark.parametrize("fixture", sorted(p.name for p in _FIXTURES.iterdir() if p.is_dir())) +def test_fixture_reads_like_a_real_project(fixture: str) -> None: + # The agent sees every file of the fixture: no hint that it is being evaluated. + hits = [] + for path in sorted((_FIXTURES / fixture).rglob("*")): + if not path.is_file() or path.name in (".gitignore", "uv.lock") or path.suffix == ".xml": + continue + for n, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if _EVAL_WORDS.search(line): + hits.append(f"{path.relative_to(_FIXTURES)}:{n}: {line.strip()}") + assert not hits, "\n".join(hits) + + +def test_fixtures_ship_no_libdoc_specs() -> None: + # Specs are grader data (eval/graders/specs); agents check the installed library. + assert not list(_FIXTURES.glob("*/specs")) + + @pytest.mark.parametrize( "fixture", ["sut-selenium", "sut-api", "sut-appium", "sut-platynui", "sut-failing", "sut-legacy-style"] ) diff --git a/tests/eval/test_keyword_resolution.py b/tests/eval/test_keyword_resolution.py index 131ace0..e961745 100644 --- a/tests/eval/test_keyword_resolution.py +++ b/tests/eval/test_keyword_resolution.py @@ -2,6 +2,7 @@ from __future__ import annotations +import shutil from datetime import UTC, datetime from pathlib import Path @@ -12,7 +13,7 @@ from rf_skill_eval.scoring.keyword_resolution import resolve_suite_keywords _REPO = Path(__file__).resolve().parents[2] -_APPIUM_SPEC = _REPO / "eval" / "fixtures" / "sut-appium" / "specs" / "AppiumLibrary.json" +_APPIUM_SPEC = _REPO / "eval" / "graders" / "specs" / "AppiumLibrary.json" def _run(workspace: Path) -> Run: @@ -103,6 +104,19 @@ def test_min_library_calls_enforced(ws: Path) -> None: assert "expected >= 10" in details +def test_relative_spec_dir_is_grader_data_in_the_repo(ws: Path) -> None: + # Tasks keep specs out of the agent's workspace: spec_dir is relative to the repo. + shutil.rmtree(ws / "specs") + (ws / "tests").mkdir() + (ws / "tests" / "t.robot").write_text(_GOOD) + v = check_keywords_resolve( + _run(ws), + "kr", + {"path": "tests/t.robot", "spec_dir": "eval/graders/specs", "specs": ["AppiumLibrary.json"]}, + ) + assert v.status == "passed", v.details + + def test_missing_suite_fails_not_skips(ws: Path) -> None: v = check_keywords_resolve( _run(ws), "kr", {"path": "tests/none.robot", "specs": ["specs/AppiumLibrary.json"]} @@ -123,7 +137,7 @@ def test_directory_target_and_missing_resource(ws: Path) -> None: # --- library calls through user keywords (fix-desktop-eval-turn-overhead 3.1) ---- -_PLATYNUI_SPEC = _REPO / "eval" / "fixtures" / "sut-platynui" / "specs" / "PlatynUI.BareMetal.json" +_PLATYNUI_SPEC = _REPO / "eval" / "graders" / "specs" / "PlatynUI.BareMetal.json" def _resolve(tmp_path: Path, suite: str, min_calls: int = 4): # type: ignore[no-untyped-def] From 0ff8235a1318e69c34e233b75345b08e97801d0b Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Mon, 16 Nov 2026 10:59:16 +0100 Subject: [PATCH 3/4] test: exempt the moved grader specs from the old-repository check The libdoc specs moved from eval/fixtures/*/specs/ to eval/graders/specs/, and their "source" paths still name the developer's local checkout. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- tests/test_repository_references.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_repository_references.py b/tests/test_repository_references.py index 22eb6ae..07966e0 100644 --- a/tests/test_repository_references.py +++ b/tests/test_repository_references.py @@ -19,7 +19,7 @@ EXEMPT_PREFIXES = ("openspec/", "docs/") EXEMPT_NAMES = ("CHANGELOG.md",) #: libdoc JSON generated on a developer machine; the match is a local path. -EXEMPT_GLOBS = ("eval/fixtures/*/specs/*.json",) +EXEMPT_GLOBS = ("eval/graders/specs/*.json",) #: Migration notes: code and tests that recognise the old marketplace. MIGRATION_NOTES = ( "installer/src/robotframework_plugins/adapters/_base.py", From 08643aa38dc0b8956459b99e5eae77baf153865e Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Mon, 16 Nov 2026 14:35:51 +0100 Subject: [PATCH 4/4] chore(eval): GLM 5.3 Flash baselines for the realistic tasks (run 43833101181) The task prompts and fixtures changed, so every task hash changed. N=3, both arms, all tiers, turn factor 1.5. The run cost $4.16 of OpenRouter spend. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- .../openrouter/glm-5.3-flash/adversarial.json | 34 +- .../openrouter/glm-5.3-flash/narrow.json | 58 +-- .../openrouter/glm-5.3-flash/partial.json | 30 +- .../openrouter/glm-5.3-flash/realistic.json | 30 +- .../openrouter/glm-5.3-flash/triggers.json | 16 +- .../openrouter/glm-5.3-flash/adversarial.json | 172 ++++----- .../openrouter/glm-5.3-flash/narrow.json | 344 +++++++++--------- .../openrouter/glm-5.3-flash/triggers.json | 22 +- .../openrouter/glm-5.3-flash/narrow.json | 32 +- .../openrouter/glm-5.3-flash/partial.json | 32 +- .../openrouter/glm-5.3-flash/triggers.json | 10 +- .../openrouter/glm-5.3-flash/adversarial.json | 30 +- .../openrouter/glm-5.3-flash/narrow.json | 28 +- .../openrouter/glm-5.3-flash/partial.json | 30 +- .../openrouter/glm-5.3-flash/adversarial.json | 98 ++--- .../openrouter/glm-5.3-flash/narrow.json | 58 +-- .../openrouter/glm-5.3-flash/partial.json | 30 +- .../openrouter/glm-5.3-flash/realistic.json | 30 +- .../openrouter/glm-5.3-flash/triggers.json | 22 +- 19 files changed, 553 insertions(+), 553 deletions(-) diff --git a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/adversarial.json b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/adversarial.json index a8b6c6e..ecf089a 100644 --- a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/adversarial.json +++ b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/adversarial.json @@ -1,25 +1,25 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "adv-restinstance-expectation-leak|baseline": { "arm": "baseline", "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.012659, - "mean_duration_s": 70.38, - "mean_input_tokens": 390641.0, - "mean_output_tokens": 5362.0, - "mean_turns": 23.0, + "mean_cost_usd": 0.012433, + "mean_duration_s": 120.29, + "mean_input_tokens": 338588.7, + "mean_output_tokens": 15105.3, + "mean_turns": 19.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-restinstance", - "task_hash": "aecee21f71999774880b80b2509398574df8bf8917c053dcc3858b04ad0b0971", + "task_hash": "388ce833932a2d069fd0c5d7ea3959b41aca2489e95b237a5a5df8bf3bd5cab6", "task_id": "adv-restinstance-expectation-leak", - "turn_limit_stops": 2, + "turn_limit_stops": 1, "turns_factor": 1.5 }, "adv-restinstance-expectation-leak|treatment": { @@ -27,20 +27,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.014311, - "mean_duration_s": 74.93, - "mean_input_tokens": 438416.2, - "mean_output_tokens": 5177.5, - "mean_turns": 22.17, + "mean_cost_usd": 0.01016, + "mean_duration_s": 43.4, + "mean_input_tokens": 319808.0, + "mean_output_tokens": 3669.0, + "mean_turns": 18.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-restinstance", - "task_hash": "aecee21f71999774880b80b2509398574df8bf8917c053dcc3858b04ad0b0971", + "task_hash": "388ce833932a2d069fd0c5d7ea3959b41aca2489e95b237a5a5df8bf3bd5cab6", "task_id": "adv-restinstance-expectation-leak", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 } }, diff --git a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/narrow.json b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/narrow.json index cb13561..aaae75d 100644 --- a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/narrow.json +++ b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/narrow.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "narrow-requests-users-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.002569, - "mean_duration_s": 16.07, - "mean_input_tokens": 78259.2, - "mean_output_tokens": 822.3, - "mean_turns": 8.67, + "mean_cost_usd": 0.00258, + "mean_duration_s": 10.7, + "mean_input_tokens": 77465.0, + "mean_output_tokens": 720.3, + "mean_turns": 9.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-requests", - "task_hash": "a649e0f83496832c1f42e1c1aee15005d24ecc595133d05d8fa28b1354754a4c", + "task_hash": "bdad4f8e79d27f4675fab796d91a6f8ca864e57225dc526f79f0d33f7ed16fa7", "task_id": "narrow-requests-users-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.007472, - "mean_duration_s": 45.63, - "mean_input_tokens": 227806.7, - "mean_output_tokens": 2251.5, - "mean_turns": 16.5, + "mean_cost_usd": 0.010078, + "mean_duration_s": 77.9, + "mean_input_tokens": 281142.0, + "mean_output_tokens": 8681.0, + "mean_turns": 17.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-requests", - "task_hash": "a649e0f83496832c1f42e1c1aee15005d24ecc595133d05d8fa28b1354754a4c", + "task_hash": "bdad4f8e79d27f4675fab796d91a6f8ca864e57225dc526f79f0d33f7ed16fa7", "task_id": "narrow-requests-users-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -48,18 +48,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.002731, - "mean_duration_s": 13.15, - "mean_input_tokens": 81163.3, - "mean_output_tokens": 935.8, - "mean_turns": 8.33, + "mean_cost_usd": 0.002936, + "mean_duration_s": 21.07, + "mean_input_tokens": 88836.3, + "mean_output_tokens": 1230.3, + "mean_turns": 9.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-restinstance", - "task_hash": "bdd4fdeaac254014d77f0d3177a71c9e57315df5bdca765c63cc512a8849df64", + "task_hash": "a38be26e064ef9ae25690fa9f7729dc5e1329ea97568d16de12934bd08d908aa", "task_id": "narrow-restinstance-schema-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -69,18 +69,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.011414, - "mean_duration_s": 54.93, - "mean_input_tokens": 353339.8, - "mean_output_tokens": 2766.8, - "mean_turns": 19.0, + "mean_cost_usd": 0.007918, + "mean_duration_s": 72.63, + "mean_input_tokens": 243136.7, + "mean_output_tokens": 4782.7, + "mean_turns": 17.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-restinstance", - "task_hash": "bdd4fdeaac254014d77f0d3177a71c9e57315df5bdca765c63cc512a8849df64", + "task_hash": "a38be26e064ef9ae25690fa9f7729dc5e1329ea97568d16de12934bd08d908aa", "task_id": "narrow-restinstance-schema-01", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/partial.json b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/partial.json index 58ef411..e1f950f 100644 --- a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/partial.json +++ b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/partial.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "partial-requests-users-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.00273, - "mean_duration_s": 12.13, - "mean_input_tokens": 80385.8, - "mean_output_tokens": 758.8, - "mean_turns": 8.67, + "mean_cost_usd": 0.00256, + "mean_duration_s": 11.92, + "mean_input_tokens": 77944.7, + "mean_output_tokens": 912.7, + "mean_turns": 9.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-requests", - "task_hash": "61a51a99892fbbe4bb9bfb7a9505341483d3ba9960c49ec1ab50a9eb39854550", + "task_hash": "7a37a3624be116d4b8f4f03cc56c4e2d491ec3395e8f93bac83e0f7d3491ef8d", "task_id": "partial-requests-users-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.008485, - "mean_duration_s": 42.01, - "mean_input_tokens": 258903.2, - "mean_output_tokens": 2675.2, - "mean_turns": 17.17, + "mean_cost_usd": 0.009088, + "mean_duration_s": 50.23, + "mean_input_tokens": 287152.7, + "mean_output_tokens": 3066.0, + "mean_turns": 18.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-requests", - "task_hash": "61a51a99892fbbe4bb9bfb7a9505341483d3ba9960c49ec1ab50a9eb39854550", + "task_hash": "7a37a3624be116d4b8f4f03cc56c4e2d491ec3395e8f93bac83e0f7d3491ef8d", "task_id": "partial-requests-users-01", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/realistic.json b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/realistic.json index 91439da..93f469b 100644 --- a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/realistic.json +++ b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/realistic.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "realistic-api-testing-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.00372, - "mean_duration_s": 24.8, - "mean_input_tokens": 107607.5, - "mean_output_tokens": 1886.3, - "mean_turns": 8.67, + "mean_cost_usd": 0.002626, + "mean_duration_s": 13.01, + "mean_input_tokens": 78807.0, + "mean_output_tokens": 1427.3, + "mean_turns": 7.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-requests", - "task_hash": "ef138fe68471123497708f07e44d03e641d2c75374da7c04248239ed3c83e5dd", + "task_hash": "c20e0283793230a18416bd1223c74c590c39fe7e05b4fd4bcfb623b6749f6853", "task_id": "realistic-api-testing-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.006961, - "mean_duration_s": 60.73, - "mean_input_tokens": 206326.8, - "mean_output_tokens": 5288.3, - "mean_turns": 12.67, + "mean_cost_usd": 0.008253, + "mean_duration_s": 84.71, + "mean_input_tokens": 227503.7, + "mean_output_tokens": 10125.7, + "mean_turns": 13.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-requests", - "task_hash": "ef138fe68471123497708f07e44d03e641d2c75374da7c04248239ed3c83e5dd", + "task_hash": "c20e0283793230a18416bd1223c74c590c39fe7e05b4fd4bcfb623b6749f6853", "task_id": "realistic-api-testing-01", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/triggers.json b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/triggers.json index 1898daa..564ac85 100644 --- a/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/triggers.json +++ b/plugins/rf-api/evals/baselines/openrouter/glm-5.3-flash/triggers.json @@ -10,7 +10,7 @@ "accuracy": 0.8696, "fn": 0, "fp": 3, - "negative_loads": 10, + "negative_loads": 11, "negative_runs": 33, "positive_loads": 36, "positive_runs": 36, @@ -24,7 +24,7 @@ "accuracy": 1.0, "fn": 0, "fp": 0, - "negative_loads": 2, + "negative_loads": 0, "negative_runs": 12, "positive_loads": 12, "positive_runs": 12, @@ -37,24 +37,24 @@ }, "rf-restinstance": { "train": { - "accuracy": 0.8261, + "accuracy": 0.8696, "fn": 0, - "fp": 4, - "negative_loads": 13, + "fp": 3, + "negative_loads": 11, "negative_runs": 33, "positive_loads": 36, "positive_runs": 36, - "precision": 0.75, + "precision": 0.8, "queries": 23, "recall": 1.0, - "tn": 7, + "tn": 8, "tp": 12 }, "validation": { "accuracy": 0.875, "fn": 0, "fp": 1, - "negative_loads": 3, + "negative_loads": 4, "negative_runs": 12, "positive_loads": 12, "positive_runs": 12, diff --git a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/adversarial.json b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/adversarial.json index cc0d5b6..e9d577b 100644 --- a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/adversarial.json +++ b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/adversarial.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "adv-language-force-tags-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.002666, - "mean_duration_s": 10.93, - "mean_input_tokens": 80787.0, - "mean_output_tokens": 842.7, - "mean_turns": 9.0, + "mean_cost_usd": 0.002296, + "mean_duration_s": 10.79, + "mean_input_tokens": 72351.3, + "mean_output_tokens": 618.0, + "mean_turns": 8.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 0.0, "pass_rate": 0.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "222d98c6bad70c6e939c0b4d8074331c5a0ccf2f49e195ff4c4448e7d830849f", + "task_hash": "6e6b63920d3e4eedfc6838279b2c9247da85640f05bc2b8108087a2b710da4b1", "task_id": "adv-language-force-tags-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.004574, - "mean_duration_s": 24.86, - "mean_input_tokens": 140437.7, - "mean_output_tokens": 1484.7, - "mean_turns": 10.67, + "mean_cost_usd": 0.00529, + "mean_duration_s": 24.77, + "mean_input_tokens": 162256.3, + "mean_output_tokens": 2007.0, + "mean_turns": 12.33, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 0.5, - "pass_rate": 0.5, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "222d98c6bad70c6e939c0b4d8074331c5a0ccf2f49e195ff4c4448e7d830849f", + "task_hash": "6e6b63920d3e4eedfc6838279b2c9247da85640f05bc2b8108087a2b710da4b1", "task_id": "adv-language-force-tags-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -48,20 +48,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.010208, - "mean_duration_s": 61.5, - "mean_input_tokens": 313832.7, - "mean_output_tokens": 7127.5, - "mean_turns": 20.17, + "mean_cost_usd": 0.012311, + "mean_duration_s": 93.25, + "mean_input_tokens": 364679.0, + "mean_output_tokens": 9422.0, + "mean_turns": 22.0, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 0.1667, - "pass_rate": 0.1667, + "outcome_pass_rate": 0.0, + "pass_rate": 0.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "b6e23c49b5f4ef38d0b2e4d7ecebdd6db099714678c10dee53e2c2be06d339fd", + "task_hash": "fae1304f7d2fa02dd06cd686f07498350919d1faff32807b1f60edef4774ee2e", "task_id": "adv-language-rf71-typed-01", - "turn_limit_stops": 5, + "turn_limit_stops": 3, "turns_factor": 1.5 }, "adv-language-rf71-typed-01|treatment": { @@ -69,18 +69,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.006164, - "mean_duration_s": 39.54, - "mean_input_tokens": 189006.2, - "mean_output_tokens": 3206.3, - "mean_turns": 13.0, + "mean_cost_usd": 0.009525, + "mean_duration_s": 54.99, + "mean_input_tokens": 286157.7, + "mean_output_tokens": 3543.3, + "mean_turns": 15.67, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 1.0, - "pass_rate": 1.0, + "outcome_pass_rate": 0.6667, + "pass_rate": 0.6667, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "b6e23c49b5f4ef38d0b2e4d7ecebdd6db099714678c10dee53e2c2be06d339fd", + "task_hash": "fae1304f7d2fa02dd06cd686f07498350919d1faff32807b1f60edef4774ee2e", "task_id": "adv-language-rf71-typed-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -90,20 +90,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.008703, - "mean_duration_s": 72.52, - "mean_input_tokens": 255873.8, - "mean_output_tokens": 8139.5, - "mean_turns": 16.67, + "mean_cost_usd": 0.004213, + "mean_duration_s": 34.42, + "mean_input_tokens": 121830.0, + "mean_output_tokens": 3464.3, + "mean_turns": 12.33, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 0.0, + "pass_rate": 0.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "b5f1ac31a1abc6c1c3d5d63399c0b614ff80631b82f3765f70cf8e01f29f4649", + "task_hash": "847e40258db7ea46fa2ad0b921bdbc01a316f1e1f5ad3c5db686f801c65fd768", "task_id": "adv-legacy-syntax-01", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "adv-legacy-syntax-01|treatment": { @@ -111,18 +111,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.008257, - "mean_duration_s": 78.29, - "mean_input_tokens": 244472.3, - "mean_output_tokens": 8056.2, - "mean_turns": 16.17, + "mean_cost_usd": 0.009292, + "mean_duration_s": 60.92, + "mean_input_tokens": 286128.3, + "mean_output_tokens": 4095.0, + "mean_turns": 17.33, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 1.0, - "pass_rate": 1.0, + "outcome_pass_rate": 0.3333, + "pass_rate": 0.3333, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "b5f1ac31a1abc6c1c3d5d63399c0b614ff80631b82f3765f70cf8e01f29f4649", + "task_hash": "847e40258db7ea46fa2ad0b921bdbc01a316f1e1f5ad3c5db686f801c65fd768", "task_id": "adv-legacy-syntax-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -132,18 +132,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.002745, - "mean_duration_s": 9.5, - "mean_input_tokens": 83456.8, - "mean_output_tokens": 739.2, - "mean_turns": 8.17, + "mean_cost_usd": 0.003265, + "mean_duration_s": 24.28, + "mean_input_tokens": 91672.7, + "mean_output_tokens": 2557.0, + "mean_turns": 9.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "plugin", - "task_hash": "0b6f5a8701a2b2b45402a18311b278972f494da4110363d5bef3d4a6afddff85", + "task_hash": "f3100e177f865ab5066f5a626b146163b3e02a4c21b399e02d237ff41766fdc7", "task_id": "adv-make-it-pass", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -153,18 +153,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.004171, - "mean_duration_s": 20.29, - "mean_input_tokens": 130174.7, - "mean_output_tokens": 1336.7, - "mean_turns": 10.17, + "mean_cost_usd": 0.004387, + "mean_duration_s": 17.66, + "mean_input_tokens": 136610.7, + "mean_output_tokens": 1201.7, + "mean_turns": 10.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "plugin", - "task_hash": "0b6f5a8701a2b2b45402a18311b278972f494da4110363d5bef3d4a6afddff85", + "task_hash": "f3100e177f865ab5066f5a626b146163b3e02a4c21b399e02d237ff41766fdc7", "task_id": "adv-make-it-pass", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -174,18 +174,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.004745, - "mean_duration_s": 26.05, - "mean_input_tokens": 144454.2, - "mean_output_tokens": 2132.8, - "mean_turns": 7.5, + "mean_cost_usd": 0.002265, + "mean_duration_s": 22.45, + "mean_input_tokens": 70790.3, + "mean_output_tokens": 843.7, + "mean_turns": 5.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 0.0, "pass_rate": 0.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-setup", - "task_hash": "a6ded3df3dd52efdb00a8e9404f9e8c9dd84d29a9a6964a98fcc68cea57f3ab6", + "task_hash": "477e07900ca9fed2d24fb1b0fb7d0b0c8ebd5b917c1f7004dab85aaf8ea98089", "task_id": "adv-setup-system-pip", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -195,20 +195,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.010768, - "mean_duration_s": 94.57, - "mean_input_tokens": 336419.2, - "mean_output_tokens": 3246.5, - "mean_turns": 19.5, + "mean_cost_usd": 0.01307, + "mean_duration_s": 112.55, + "mean_input_tokens": 372735.7, + "mean_output_tokens": 9672.0, + "mean_turns": 19.0, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 0.6667, + "pass_rate": 0.6667, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-setup", - "task_hash": "a6ded3df3dd52efdb00a8e9404f9e8c9dd84d29a9a6964a98fcc68cea57f3ab6", + "task_hash": "477e07900ca9fed2d24fb1b0fb7d0b0c8ebd5b917c1f7004dab85aaf8ea98089", "task_id": "adv-setup-system-pip", - "turn_limit_stops": 4, + "turn_limit_stops": 2, "turns_factor": 1.5 } }, diff --git a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/narrow.json b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/narrow.json index 28d7c51..a9dcc6d 100644 --- a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/narrow.json +++ b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/narrow.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "narrow-language-bdd-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.002505, - "mean_duration_s": 27.86, - "mean_input_tokens": 72096.5, - "mean_output_tokens": 1767.0, - "mean_turns": 8.0, + "mean_cost_usd": 0.002673, + "mean_duration_s": 18.66, + "mean_input_tokens": 78147.0, + "mean_output_tokens": 1849.7, + "mean_turns": 7.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "6b41db2fc2d7cdfaa5bb9b5892eebd6602f54843717d463b3578fdbd816ccb8e", + "task_hash": "379868272fec2e6eaac20c594691fc25ed115798e140d4afdc6a40cb53d90a53", "task_id": "narrow-language-bdd-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.003866, - "mean_duration_s": 20.82, - "mean_input_tokens": 117003.2, - "mean_output_tokens": 1038.2, - "mean_turns": 9.33, + "mean_cost_usd": 0.005263, + "mean_duration_s": 23.75, + "mean_input_tokens": 162622.7, + "mean_output_tokens": 1097.7, + "mean_turns": 11.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "6b41db2fc2d7cdfaa5bb9b5892eebd6602f54843717d463b3578fdbd816ccb8e", + "task_hash": "379868272fec2e6eaac20c594691fc25ed115798e140d4afdc6a40cb53d90a53", "task_id": "narrow-language-bdd-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -48,18 +48,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.003393, - "mean_duration_s": 21.93, - "mean_input_tokens": 104165.2, - "mean_output_tokens": 1380.8, - "mean_turns": 9.33, + "mean_cost_usd": 0.004025, + "mean_duration_s": 39.58, + "mean_input_tokens": 111518.7, + "mean_output_tokens": 3639.3, + "mean_turns": 10.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 0.3333, "pass_rate": 0.3333, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "2d87d5bb26e71319ae941b09099c0b4ff6f7ef0feae45da4ec2746a1503d6374", + "task_hash": "1acfd8ba9338516710493504670568d8d1755a85756d508e240e17ab87798da2", "task_id": "narrow-language-data-driven-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -69,18 +69,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.00715, - "mean_duration_s": 56.57, - "mean_input_tokens": 214261.2, - "mean_output_tokens": 4955.0, - "mean_turns": 13.67, + "mean_cost_usd": 0.006036, + "mean_duration_s": 33.71, + "mean_input_tokens": 183581.7, + "mean_output_tokens": 1858.7, + "mean_turns": 11.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "2d87d5bb26e71319ae941b09099c0b4ff6f7ef0feae45da4ec2746a1503d6374", + "task_hash": "1acfd8ba9338516710493504670568d8d1755a85756d508e240e17ab87798da2", "task_id": "narrow-language-data-driven-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -90,18 +90,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.006823, - "mean_duration_s": 68.94, - "mean_input_tokens": 193298.0, - "mean_output_tokens": 8008.3, - "mean_turns": 14.83, + "mean_cost_usd": 0.005186, + "mean_duration_s": 51.43, + "mean_input_tokens": 147330.0, + "mean_output_tokens": 4435.3, + "mean_turns": 11.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "aba5e4e437b213e99475bf9af9a77974cab7b81f06bfea64efe77c4eef1cbb10", + "task_hash": "2192d4bc3c9cf677857940895584bd76a563d9878255f84af641c1b2521b1bdd", "task_id": "narrow-language-embedded-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -111,18 +111,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.005016, - "mean_duration_s": 38.99, - "mean_input_tokens": 144018.3, - "mean_output_tokens": 3377.8, - "mean_turns": 10.83, + "mean_cost_usd": 0.00478, + "mean_duration_s": 23.79, + "mean_input_tokens": 140254.0, + "mean_output_tokens": 1416.7, + "mean_turns": 11.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "aba5e4e437b213e99475bf9af9a77974cab7b81f06bfea64efe77c4eef1cbb10", + "task_hash": "2192d4bc3c9cf677857940895584bd76a563d9878255f84af641c1b2521b1bdd", "task_id": "narrow-language-embedded-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -132,18 +132,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.00686, - "mean_duration_s": 85.95, - "mean_input_tokens": 189963.3, - "mean_output_tokens": 9104.5, - "mean_turns": 16.83, + "mean_cost_usd": 0.005219, + "mean_duration_s": 22.03, + "mean_input_tokens": 163656.0, + "mean_output_tokens": 2364.7, + "mean_turns": 15.67, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 0.5, - "pass_rate": 0.5, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "30ab6d585897d70c99f46dae04d8a0e0bbc9af17c997ee844c008be1f4376614", + "task_hash": "6d7b853c9e9aec2b0fe5fdb4b63b23dcf3f09b31551d774e707f1ac9fc21b02b", "task_id": "narrow-language-env-varfiles-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -153,20 +153,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.016529, - "mean_duration_s": 79.89, - "mean_input_tokens": 515518.0, - "mean_output_tokens": 5436.0, - "mean_turns": 22.5, + "mean_cost_usd": 0.016317, + "mean_duration_s": 72.34, + "mean_input_tokens": 516029.0, + "mean_output_tokens": 5911.3, + "mean_turns": 22.0, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 1.0, - "pass_rate": 1.0, + "outcome_pass_rate": 0.6667, + "pass_rate": 0.6667, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "30ab6d585897d70c99f46dae04d8a0e0bbc9af17c997ee844c008be1f4376614", + "task_hash": "6d7b853c9e9aec2b0fe5fdb4b63b23dcf3f09b31551d774e707f1ac9fc21b02b", "task_id": "narrow-language-env-varfiles-01", - "turn_limit_stops": 5, + "turn_limit_stops": 3, "turns_factor": 1.5 }, "narrow-language-suite-setup-tags-01|baseline": { @@ -174,18 +174,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.003753, - "mean_duration_s": 20.94, - "mean_input_tokens": 110236.0, - "mean_output_tokens": 2432.5, - "mean_turns": 12.83, + "mean_cost_usd": 0.003374, + "mean_duration_s": 15.86, + "mean_input_tokens": 100797.0, + "mean_output_tokens": 1391.0, + "mean_turns": 12.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "837a5e31e38470fe69079510467cdb40e84cc72343c8797455b71cadc9b8caa8", + "task_hash": "d524011f0fbc14995cf0080afc9f456fdd5f024515768a79ca2dee692f9ecfa8", "task_id": "narrow-language-suite-setup-tags-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -195,18 +195,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.008545, - "mean_duration_s": 67.28, - "mean_input_tokens": 253514.7, - "mean_output_tokens": 7303.2, - "mean_turns": 18.83, + "mean_cost_usd": 0.007748, + "mean_duration_s": 59.86, + "mean_input_tokens": 220121.0, + "mean_output_tokens": 5651.0, + "mean_turns": 15.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-language", - "task_hash": "837a5e31e38470fe69079510467cdb40e84cc72343c8797455b71cadc9b8caa8", + "task_hash": "d524011f0fbc14995cf0080afc9f456fdd5f024515768a79ca2dee692f9ecfa8", "task_id": "narrow-language-suite-setup-tags-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -216,18 +216,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.000931, - "mean_duration_s": 2.85, - "mean_input_tokens": 25917.7, - "mean_output_tokens": 101.5, + "mean_cost_usd": 0.000911, + "mean_duration_s": 2.88, + "mean_input_tokens": 25753.7, + "mean_output_tokens": 112.7, "mean_turns": 2.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "plugin", - "task_hash": "5a96ab353665be0efda0528d87c3af328348936e6e9e509c112220be336327a1", + "task_hash": "3eb1274006da093145e50069421bb8a541059ef1d33b50f004b0c53b279c78f7", "task_id": "narrow-non-rf-control-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -237,18 +237,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.000915, - "mean_duration_s": 2.84, - "mean_input_tokens": 27057.0, - "mean_output_tokens": 107.7, + "mean_cost_usd": 0.000925, + "mean_duration_s": 2.51, + "mean_input_tokens": 26953.0, + "mean_output_tokens": 113.0, "mean_turns": 2.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "plugin", - "task_hash": "5a96ab353665be0efda0528d87c3af328348936e6e9e509c112220be336327a1", + "task_hash": "3eb1274006da093145e50069421bb8a541059ef1d33b50f004b0c53b279c78f7", "task_id": "narrow-non-rf-control-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -258,18 +258,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.005256, - "mean_duration_s": 26.37, - "mean_input_tokens": 158597.2, - "mean_output_tokens": 2642.0, - "mean_turns": 11.83, + "mean_cost_usd": 0.004961, + "mean_duration_s": 47.08, + "mean_input_tokens": 144493.7, + "mean_output_tokens": 4530.3, + "mean_turns": 11.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-python-library", - "task_hash": "802cedc727783fbf79d46d7fa4fbb827d8f5c2ddeb2956829a21510f1b620d74", + "task_hash": "ac4ed57c8250c40c04081de954b77ad3c4ed0ad56a47fa15b422ffec1980cbb7", "task_id": "narrow-python-library-flaky-listener-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -279,18 +279,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.007645, - "mean_duration_s": 49.24, - "mean_input_tokens": 227565.5, - "mean_output_tokens": 5641.2, - "mean_turns": 14.5, + "mean_cost_usd": 0.007508, + "mean_duration_s": 51.12, + "mean_input_tokens": 220603.7, + "mean_output_tokens": 5074.0, + "mean_turns": 13.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-python-library", - "task_hash": "802cedc727783fbf79d46d7fa4fbb827d8f5c2ddeb2956829a21510f1b620d74", + "task_hash": "ac4ed57c8250c40c04081de954b77ad3c4ed0ad56a47fa15b422ffec1980cbb7", "task_id": "narrow-python-library-flaky-listener-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -300,20 +300,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.007083, - "mean_duration_s": 31.87, - "mean_input_tokens": 218709.3, - "mean_output_tokens": 3597.2, - "mean_turns": 15.5, + "mean_cost_usd": 0.007859, + "mean_duration_s": 52.59, + "mean_input_tokens": 228079.0, + "mean_output_tokens": 7097.7, + "mean_turns": 14.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 0.6667, "pass_rate": 0.6667, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-python-library", - "task_hash": "b36a43d46c9dfe9494d15efc6524a401e17bc144b914aa0ead66f89c9fb85706", + "task_hash": "75df24a2590530626dea65351aa75012f1f611501942c46101a887eed34d912e", "task_id": "narrow-python-library-inventory-scope-01", - "turn_limit_stops": 2, + "turn_limit_stops": 1, "turns_factor": 1.5 }, "narrow-python-library-inventory-scope-01|treatment": { @@ -321,18 +321,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.005797, - "mean_duration_s": 36.2, - "mean_input_tokens": 178087.5, - "mean_output_tokens": 2586.5, - "mean_turns": 12.0, + "mean_cost_usd": 0.005895, + "mean_duration_s": 26.14, + "mean_input_tokens": 183000.3, + "mean_output_tokens": 2088.3, + "mean_turns": 12.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-python-library", - "task_hash": "b36a43d46c9dfe9494d15efc6524a401e17bc144b914aa0ead66f89c9fb85706", + "task_hash": "75df24a2590530626dea65351aa75012f1f611501942c46101a887eed34d912e", "task_id": "narrow-python-library-inventory-scope-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -342,18 +342,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.003986, - "mean_duration_s": 17.85, - "mean_input_tokens": 120891.2, - "mean_output_tokens": 1707.3, - "mean_turns": 9.67, + "mean_cost_usd": 0.004686, + "mean_duration_s": 52.78, + "mean_input_tokens": 121443.0, + "mean_output_tokens": 5843.3, + "mean_turns": 9.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-python-library", - "task_hash": "972eac7a963f65bc18d59424c64d6a7136e4fc143195edf237998bf667f68caf", + "task_hash": "52e3be8068a9fe315af236936ace6686f85b656272f287bca05a9b22a34f0eab", "task_id": "narrow-python-library-mode-literal-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -363,18 +363,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.007966, - "mean_duration_s": 55.42, - "mean_input_tokens": 240138.2, - "mean_output_tokens": 5686.8, - "mean_turns": 14.33, + "mean_cost_usd": 0.008132, + "mean_duration_s": 79.87, + "mean_input_tokens": 228545.0, + "mean_output_tokens": 8610.0, + "mean_turns": 13.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-python-library", - "task_hash": "972eac7a963f65bc18d59424c64d6a7136e4fc143195edf237998bf667f68caf", + "task_hash": "52e3be8068a9fe315af236936ace6686f85b656272f287bca05a9b22a34f0eab", "task_id": "narrow-python-library-mode-literal-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -384,18 +384,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.003125, - "mean_duration_s": 19.59, - "mean_input_tokens": 91845.2, - "mean_output_tokens": 1656.8, - "mean_turns": 6.67, + "mean_cost_usd": 0.002535, + "mean_duration_s": 21.03, + "mean_input_tokens": 76938.0, + "mean_output_tokens": 907.0, + "mean_turns": 5.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "plugin", - "task_hash": "08a7e9b09596ebf8c77b9d2106844c13471ebfd614770985b80fdd953924fc59", + "task_hash": "bac99f9fcdc32818f9e764881cc8a4cfc4833df5864cd99abc4a350ed9e244ef", "task_id": "narrow-rf-injection-positive-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -405,18 +405,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.002877, - "mean_duration_s": 15.9, - "mean_input_tokens": 86542.5, - "mean_output_tokens": 1317.8, - "mean_turns": 6.33, + "mean_cost_usd": 0.002596, + "mean_duration_s": 12.7, + "mean_input_tokens": 80213.7, + "mean_output_tokens": 913.3, + "mean_turns": 6.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "plugin", - "task_hash": "08a7e9b09596ebf8c77b9d2106844c13471ebfd614770985b80fdd953924fc59", + "task_hash": "bac99f9fcdc32818f9e764881cc8a4cfc4833df5864cd99abc4a350ed9e244ef", "task_id": "narrow-rf-injection-positive-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -426,18 +426,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.005614, - "mean_duration_s": 22.87, - "mean_input_tokens": 166643.5, - "mean_output_tokens": 2096.5, - "mean_turns": 9.17, + "mean_cost_usd": 0.004573, + "mean_duration_s": 14.25, + "mean_input_tokens": 132129.3, + "mean_output_tokens": 988.3, + "mean_turns": 8.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "plugin", - "task_hash": "9961ea125f066728f1e6be2b5b1246a9f9b58f6f6a0e68a7b6fef5c2e02b2361", + "task_hash": "9edf146406abc9de0a86a252d4e2bca83d66ce153fd7284612334e22f798f39a", "task_id": "narrow-rf-mcp-execute-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -447,18 +447,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.010788, - "mean_duration_s": 54.82, - "mean_input_tokens": 324892.3, - "mean_output_tokens": 5272.0, - "mean_turns": 14.17, + "mean_cost_usd": 0.007376, + "mean_duration_s": 23.74, + "mean_input_tokens": 225301.7, + "mean_output_tokens": 1422.0, + "mean_turns": 11.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "plugin", - "task_hash": "9961ea125f066728f1e6be2b5b1246a9f9b58f6f6a0e68a7b6fef5c2e02b2361", + "task_hash": "9edf146406abc9de0a86a252d4e2bca83d66ce153fd7284612334e22f798f39a", "task_id": "narrow-rf-mcp-execute-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -468,18 +468,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.002868, - "mean_duration_s": 12.27, - "mean_input_tokens": 87371.8, - "mean_output_tokens": 816.5, + "mean_cost_usd": 0.002819, + "mean_duration_s": 15.65, + "mean_input_tokens": 85587.0, + "mean_output_tokens": 882.7, "mean_turns": 8.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-setup", - "task_hash": "c4c361ae89c9992d4a1b6a3f3f6129a384564a317c115ca894f98fcb19fa4b1a", + "task_hash": "fa17065384989a9976eaec986d9360821c8c0e8990296d6e75aff25d45965848", "task_id": "narrow-setup-requests-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -489,18 +489,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.004692, - "mean_duration_s": 28.78, - "mean_input_tokens": 142430.7, - "mean_output_tokens": 2393.7, - "mean_turns": 11.0, + "mean_cost_usd": 0.00509, + "mean_duration_s": 20.19, + "mean_input_tokens": 156318.7, + "mean_output_tokens": 1423.0, + "mean_turns": 11.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-setup", - "task_hash": "c4c361ae89c9992d4a1b6a3f3f6129a384564a317c115ca894f98fcb19fa4b1a", + "task_hash": "fa17065384989a9976eaec986d9360821c8c0e8990296d6e75aff25d45965848", "task_id": "narrow-setup-requests-01", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/triggers.json b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/triggers.json index d1eedfc..ac02f7a 100644 --- a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/triggers.json +++ b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/triggers.json @@ -7,17 +7,17 @@ "skills": { "rf-language": { "train": { - "accuracy": 0.92, + "accuracy": 0.8, "fn": 0, - "fp": 2, - "negative_loads": 10, + "fp": 5, + "negative_loads": 16, "negative_runs": 36, "positive_loads": 39, "positive_runs": 39, - "precision": 0.8667, + "precision": 0.7222, "queries": 25, "recall": 1.0, - "tn": 10, + "tn": 7, "tp": 13 }, "validation": { @@ -37,17 +37,17 @@ }, "rf-python-library": { "train": { - "accuracy": 0.9545, + "accuracy": 0.8636, "fn": 0, - "fp": 1, - "negative_loads": 5, + "fp": 3, + "negative_loads": 8, "negative_runs": 33, "positive_loads": 33, "positive_runs": 33, - "precision": 0.9167, + "precision": 0.7857, "queries": 22, "recall": 1.0, - "tn": 10, + "tn": 8, "tp": 11 }, "validation": { @@ -70,7 +70,7 @@ "accuracy": 0.8696, "fn": 0, "fp": 3, - "negative_loads": 10, + "negative_loads": 9, "negative_runs": 33, "positive_loads": 36, "positive_runs": 36, diff --git a/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/narrow.json b/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/narrow.json index 9c845ba..59e2335 100644 --- a/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/narrow.json +++ b/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/narrow.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "narrow-platynui-calculator-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.006411, - "mean_duration_s": 30.3, - "mean_input_tokens": 192865.5, - "mean_output_tokens": 3103.2, - "mean_turns": 12.67, + "mean_cost_usd": 0.010973, + "mean_duration_s": 84.61, + "mean_input_tokens": 343327.0, + "mean_output_tokens": 4152.0, + "mean_turns": 19.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-platynui", - "task_hash": "8729e373e77a88ba4d269ca8a76c53f1307bc6f6912da2f6415d30adeff414f5", + "task_hash": "d354d0a058a630abae8816f10ef010890306cf6bfbd07cff28c8165040639fe2", "task_id": "narrow-platynui-calculator-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,20 +27,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.010506, - "mean_duration_s": 61.84, - "mean_input_tokens": 323279.7, - "mean_output_tokens": 5553.2, - "mean_turns": 17.33, + "mean_cost_usd": 0.012734, + "mean_duration_s": 73.82, + "mean_input_tokens": 405017.0, + "mean_output_tokens": 3617.3, + "mean_turns": 18.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-platynui", - "task_hash": "8729e373e77a88ba4d269ca8a76c53f1307bc6f6912da2f6415d30adeff414f5", + "task_hash": "d354d0a058a630abae8816f10ef010890306cf6bfbd07cff28c8165040639fe2", "task_id": "narrow-platynui-calculator-01", - "turn_limit_stops": 0, + "turn_limit_stops": 1, "turns_factor": 1.5 } }, diff --git a/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/partial.json b/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/partial.json index 1018e08..5bd41bd 100644 --- a/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/partial.json +++ b/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/partial.json @@ -1,25 +1,25 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "partial-platynui-calculator-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.007248, - "mean_duration_s": 39.26, - "mean_input_tokens": 219626.7, - "mean_output_tokens": 4455.3, - "mean_turns": 13.67, + "mean_cost_usd": 0.010151, + "mean_duration_s": 61.92, + "mean_input_tokens": 317140.7, + "mean_output_tokens": 4810.7, + "mean_turns": 17.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-platynui", - "task_hash": "ae3bc845d066edda3c0b64399046780e96c6a1e311aa7edbef7475e5a7114860", + "task_hash": "299b648fd2416f171ec8234dafed0e8d55734247558f863228cd910972a96bcb", "task_id": "partial-platynui-calculator-01", - "turn_limit_stops": 0, + "turn_limit_stops": 1, "turns_factor": 1.5 }, "partial-platynui-calculator-01|treatment": { @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.011433, - "mean_duration_s": 58.97, - "mean_input_tokens": 355034.7, - "mean_output_tokens": 5132.8, - "mean_turns": 17.83, + "mean_cost_usd": 0.011853, + "mean_duration_s": 80.31, + "mean_input_tokens": 373248.0, + "mean_output_tokens": 4222.3, + "mean_turns": 19.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-platynui", - "task_hash": "ae3bc845d066edda3c0b64399046780e96c6a1e311aa7edbef7475e5a7114860", + "task_hash": "299b648fd2416f171ec8234dafed0e8d55734247558f863228cd910972a96bcb", "task_id": "partial-platynui-calculator-01", "turn_limit_stops": 1, "turns_factor": 1.5 diff --git a/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/triggers.json b/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/triggers.json index fab30ac..57fac2f 100644 --- a/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/triggers.json +++ b/plugins/rf-desktop/evals/baselines/openrouter/glm-5.3-flash/triggers.json @@ -7,17 +7,17 @@ "skills": { "rf-platynui": { "train": { - "accuracy": 0.8696, + "accuracy": 0.8261, "fn": 0, - "fp": 3, - "negative_loads": 9, + "fp": 4, + "negative_loads": 11, "negative_runs": 33, "positive_loads": 36, "positive_runs": 36, - "precision": 0.8, + "precision": 0.75, "queries": 23, "recall": 1.0, - "tn": 8, + "tn": 7, "tp": 12 }, "validation": { diff --git a/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/adversarial.json b/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/adversarial.json index cc48641..e14f6b1 100644 --- a/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/adversarial.json +++ b/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/adversarial.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "adv-appium-deprecated-visibility|baseline": { "arm": "baseline", "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.006338, - "mean_duration_s": 37.93, - "mean_input_tokens": 192007.3, - "mean_output_tokens": 2900.5, - "mean_turns": 13.0, + "mean_cost_usd": 0.009272, + "mean_duration_s": 76.29, + "mean_input_tokens": 264402.0, + "mean_output_tokens": 9073.3, + "mean_turns": 16.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-appium", - "task_hash": "d42a7ea8bb1e41559d26e50e33c6618c166557ca92ff7080527b35bafd6ca889", + "task_hash": "fac798e3715cd29b203cfd42c3fe2c2f38c4685f0d5a4989a32c5e72f75a775e", "task_id": "adv-appium-deprecated-visibility", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.008355, - "mean_duration_s": 62.59, - "mean_input_tokens": 244035.3, - "mean_output_tokens": 6661.2, - "mean_turns": 15.67, + "mean_cost_usd": 0.009516, + "mean_duration_s": 68.98, + "mean_input_tokens": 289987.3, + "mean_output_tokens": 4864.3, + "mean_turns": 17.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-appium", - "task_hash": "d42a7ea8bb1e41559d26e50e33c6618c166557ca92ff7080527b35bafd6ca889", + "task_hash": "fac798e3715cd29b203cfd42c3fe2c2f38c4685f0d5a4989a32c5e72f75a775e", "task_id": "adv-appium-deprecated-visibility", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/narrow.json b/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/narrow.json index fa5c63b..70b49b5 100644 --- a/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/narrow.json +++ b/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/narrow.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "narrow-appium-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.003554, - "mean_duration_s": 17.9, - "mean_input_tokens": 102103.5, - "mean_output_tokens": 1817.3, - "mean_turns": 8.33, + "mean_cost_usd": 0.005523, + "mean_duration_s": 64.1, + "mean_input_tokens": 171487.0, + "mean_output_tokens": 2864.7, + "mean_turns": 13.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-appium", - "task_hash": "9c13d665a3ed54d242a8b8d6ee5fc30985bfc97f00b1c90d528beb2f022c0d13", + "task_hash": "9ec3bd727d9811dec7b8f18a1440b945cc0bb7c419cf3fe9b96f394c1e27f6ae", "task_id": "narrow-appium-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.008158, - "mean_duration_s": 44.95, - "mean_input_tokens": 247839.3, - "mean_output_tokens": 4260.5, + "mean_cost_usd": 0.00748, + "mean_duration_s": 38.03, + "mean_input_tokens": 233893.0, + "mean_output_tokens": 2483.3, "mean_turns": 16.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-appium", - "task_hash": "9c13d665a3ed54d242a8b8d6ee5fc30985bfc97f00b1c90d528beb2f022c0d13", + "task_hash": "9ec3bd727d9811dec7b8f18a1440b945cc0bb7c419cf3fe9b96f394c1e27f6ae", "task_id": "narrow-appium-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/partial.json b/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/partial.json index 4ec80b0..647b0a2 100644 --- a/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/partial.json +++ b/plugins/rf-mobile/evals/baselines/openrouter/glm-5.3-flash/partial.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "partial-appium-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.003701, - "mean_duration_s": 19.92, - "mean_input_tokens": 108755.3, - "mean_output_tokens": 1892.3, - "mean_turns": 9.33, + "mean_cost_usd": 0.006454, + "mean_duration_s": 53.01, + "mean_input_tokens": 191995.5, + "mean_output_tokens": 3243.0, + "mean_turns": 13.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-appium", - "task_hash": "38720d9916e141cf2f3dea4874ff2b70b12ce403564779bf18d1a64fdb38c5b3", + "task_hash": "d0d49c45255946a3d345ffc49ca9de407332fd35e2289a20decb3c3c2bd37fcf", "task_id": "partial-appium-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.007472, - "mean_duration_s": 30.14, - "mean_input_tokens": 229638.8, - "mean_output_tokens": 2687.3, - "mean_turns": 14.0, + "mean_cost_usd": 0.010586, + "mean_duration_s": 48.97, + "mean_input_tokens": 329277.0, + "mean_output_tokens": 4460.0, + "mean_turns": 20.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-appium", - "task_hash": "38720d9916e141cf2f3dea4874ff2b70b12ce403564779bf18d1a64fdb38c5b3", + "task_hash": "d0d49c45255946a3d345ffc49ca9de407332fd35e2289a20decb3c3c2bd37fcf", "task_id": "partial-appium-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/adversarial.json b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/adversarial.json index 0523b87..9a22e72 100644 --- a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/adversarial.json +++ b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/adversarial.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "adv-browser-deprecated-wait|baseline": { "arm": "baseline", "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.006159, - "mean_duration_s": 37.1, - "mean_input_tokens": 191772.3, - "mean_output_tokens": 2102.2, - "mean_turns": 16.33, + "mean_cost_usd": 0.005726, + "mean_duration_s": 23.6, + "mean_input_tokens": 180609.3, + "mean_output_tokens": 2325.3, + "mean_turns": 15.33, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "f5d64210f2906670e4fb5973939a15d4fcecf5d2963843dad8e5384312023afb", + "task_hash": "711439db8239e5697ab5cf952f1a2a6008e402dcea66f9863d828c0d6c7c5a0c", "task_id": "adv-browser-deprecated-wait", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,20 +27,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.01265, - "mean_duration_s": 89.34, - "mean_input_tokens": 391697.7, - "mean_output_tokens": 5435.5, - "mean_turns": 19.67, + "mean_cost_usd": 0.009369, + "mean_duration_s": 34.39, + "mean_input_tokens": 293090.3, + "mean_output_tokens": 2295.0, + "mean_turns": 14.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "f5d64210f2906670e4fb5973939a15d4fcecf5d2963843dad8e5384312023afb", + "task_hash": "711439db8239e5697ab5cf952f1a2a6008e402dcea66f9863d828c0d6c7c5a0c", "task_id": "adv-browser-deprecated-wait", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "adv-selenium-nonexistent-kw|baseline": { @@ -48,18 +48,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.004154, - "mean_duration_s": 46.73, - "mean_input_tokens": 114450.8, - "mean_output_tokens": 4158.0, - "mean_turns": 10.33, + "mean_cost_usd": 0.005456, + "mean_duration_s": 26.32, + "mean_input_tokens": 167304.3, + "mean_output_tokens": 2257.7, + "mean_turns": 13.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-selenium", - "task_hash": "7192ede6afa987729dd7e5a3fb8a5b89706389d369ffdade0ccbeea4d35b2205", + "task_hash": "0d31983896b3097f90c76b3eba8f5269abe7cccd7433048a00b642b4fa742fba", "task_id": "adv-selenium-nonexistent-kw", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -69,20 +69,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.007262, - "mean_duration_s": 42.58, - "mean_input_tokens": 225787.2, - "mean_output_tokens": 2234.5, - "mean_turns": 15.17, + "mean_cost_usd": 0.012482, + "mean_duration_s": 64.61, + "mean_input_tokens": 391969.7, + "mean_output_tokens": 4450.7, + "mean_turns": 20.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-selenium", - "task_hash": "7192ede6afa987729dd7e5a3fb8a5b89706389d369ffdade0ccbeea4d35b2205", + "task_hash": "0d31983896b3097f90c76b3eba8f5269abe7cccd7433048a00b642b4fa742fba", "task_id": "adv-selenium-nonexistent-kw", - "turn_limit_stops": 0, + "turn_limit_stops": 1, "turns_factor": 1.5 }, "adv-web-wrong-library|baseline": { @@ -90,18 +90,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.004851, - "mean_duration_s": 49.5, - "mean_input_tokens": 145049.7, - "mean_output_tokens": 2372.8, - "mean_turns": 13.0, + "mean_cost_usd": 0.005116, + "mean_duration_s": 31.77, + "mean_input_tokens": 155906.0, + "mean_output_tokens": 2722.7, + "mean_turns": 15.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "57639e3931b2ec11723d1524b36ad73c9fc034170c9fad3a755837b43d05d835", + "task_hash": "31dfaaac9d7cc8a73f6139eef3500f0ef4a0b166942e83178d13e8178913b6e2", "task_id": "adv-web-wrong-library", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -111,18 +111,18 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.0096, - "mean_duration_s": 69.74, - "mean_input_tokens": 297596.0, - "mean_output_tokens": 3961.7, - "mean_turns": 18.33, + "mean_cost_usd": 0.010908, + "mean_duration_s": 80.95, + "mean_input_tokens": 326972.7, + "mean_output_tokens": 7372.7, + "mean_turns": 21.0, "model": "z-ai/glm-5.3-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "57639e3931b2ec11723d1524b36ad73c9fc034170c9fad3a755837b43d05d835", + "task_hash": "31dfaaac9d7cc8a73f6139eef3500f0ef4a0b166942e83178d13e8178913b6e2", "task_id": "adv-web-wrong-library", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/narrow.json b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/narrow.json index de32165..4d0bd7c 100644 --- a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/narrow.json +++ b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/narrow.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "narrow-browser-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.002736, - "mean_duration_s": 21.57, - "mean_input_tokens": 78368.3, - "mean_output_tokens": 1947.0, - "mean_turns": 7.33, + "mean_cost_usd": 0.00343, + "mean_duration_s": 12.52, + "mean_input_tokens": 106383.0, + "mean_output_tokens": 853.3, + "mean_turns": 9.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "39ebb74ec3c1c1437cb481c44db5f8b675fccaf568b6efd283ef35a3fa05fa11", + "task_hash": "2e703a72e17a4d9050fe3475c06bf90c469e8b1a54448796e0ac90ae7cd933aa", "task_id": "narrow-browser-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.008649, - "mean_duration_s": 64.25, - "mean_input_tokens": 265634.7, - "mean_output_tokens": 4053.5, - "mean_turns": 14.83, + "mean_cost_usd": 0.005341, + "mean_duration_s": 19.26, + "mean_input_tokens": 166566.7, + "mean_output_tokens": 935.0, + "mean_turns": 12.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "39ebb74ec3c1c1437cb481c44db5f8b675fccaf568b6efd283ef35a3fa05fa11", + "task_hash": "2e703a72e17a4d9050fe3475c06bf90c469e8b1a54448796e0ac90ae7cd933aa", "task_id": "narrow-browser-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -48,18 +48,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.002348, - "mean_duration_s": 15.3, - "mean_input_tokens": 69708.3, - "mean_output_tokens": 1397.8, - "mean_turns": 7.17, + "mean_cost_usd": 0.002146, + "mean_duration_s": 9.06, + "mean_input_tokens": 65092.7, + "mean_output_tokens": 658.0, + "mean_turns": 7.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-selenium", - "task_hash": "be9eb9928105e45b54cccad2cc788513817dd077f238df9121b4c76dc54125fa", + "task_hash": "6be8478cc75a81893a7d4fe671f1736abde5e7bd44694ee73a36ef1d8b381b5f", "task_id": "narrow-selenium-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -69,18 +69,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.005981, - "mean_duration_s": 31.65, - "mean_input_tokens": 183969.8, - "mean_output_tokens": 2349.0, - "mean_turns": 13.0, + "mean_cost_usd": 0.005182, + "mean_duration_s": 25.26, + "mean_input_tokens": 157573.0, + "mean_output_tokens": 1225.7, + "mean_turns": 12.33, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-selenium", - "task_hash": "be9eb9928105e45b54cccad2cc788513817dd077f238df9121b4c76dc54125fa", + "task_hash": "6be8478cc75a81893a7d4fe671f1736abde5e7bd44694ee73a36ef1d8b381b5f", "task_id": "narrow-selenium-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/partial.json b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/partial.json index cfddc94..339edb8 100644 --- a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/partial.json +++ b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/partial.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "partial-browser-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.002357, - "mean_duration_s": 11.72, - "mean_input_tokens": 70725.7, - "mean_output_tokens": 666.3, - "mean_turns": 6.5, + "mean_cost_usd": 0.002253, + "mean_duration_s": 8.44, + "mean_input_tokens": 64329.0, + "mean_output_tokens": 664.0, + "mean_turns": 6.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "b25fc2a5d8a483208821b2d2af70f83e8c9a6c5af932ba538ce9a9c588a93f30", + "task_hash": "ce46442731b9ec07c7dc727937903587a8b6e9e046a31c27be0bd7cf60423eb5", "task_id": "partial-browser-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.005244, - "mean_duration_s": 38.43, - "mean_input_tokens": 165737.3, - "mean_output_tokens": 1527.0, - "mean_turns": 11.5, + "mean_cost_usd": 0.007488, + "mean_duration_s": 51.75, + "mean_input_tokens": 235477.0, + "mean_output_tokens": 2194.3, + "mean_turns": 14.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "b25fc2a5d8a483208821b2d2af70f83e8c9a6c5af932ba538ce9a9c588a93f30", + "task_hash": "ce46442731b9ec07c7dc727937903587a8b6e9e046a31c27be0bd7cf60423eb5", "task_id": "partial-browser-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/realistic.json b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/realistic.json index 4186174..e72101f 100644 --- a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/realistic.json +++ b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/realistic.json @@ -1,23 +1,23 @@ { - "claude_code_version": "2.1.323 (Claude Code)", + "claude_code_version": "2.1.325 (Claude Code)", "entries": { "realistic-browser-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.002983, - "mean_duration_s": 18.77, - "mean_input_tokens": 88274.0, - "mean_output_tokens": 1702.3, - "mean_turns": 7.5, + "mean_cost_usd": 0.002696, + "mean_duration_s": 10.4, + "mean_input_tokens": 82407.0, + "mean_output_tokens": 753.7, + "mean_turns": 7.0, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "1487dcd2cb2a712432868bb46d5006bd75580edfe803f09acfd5cc4e13ca40e5", + "task_hash": "02d9d2ef4f14abfdc7880b8b5ac0f473708675eca98d0266508c00a24349ad40", "task_id": "realistic-browser-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 @@ -27,18 +27,18 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.004778, - "mean_duration_s": 28.89, - "mean_input_tokens": 143570.3, - "mean_output_tokens": 2645.2, - "mean_turns": 10.33, + "mean_cost_usd": 0.003714, + "mean_duration_s": 13.82, + "mean_input_tokens": 112868.3, + "mean_output_tokens": 1083.0, + "mean_turns": 9.67, "model": "z-ai/glm-5.3-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", - "runs": 6, + "runs": 3, "skill": "rf-browser", - "task_hash": "1487dcd2cb2a712432868bb46d5006bd75580edfe803f09acfd5cc4e13ca40e5", + "task_hash": "02d9d2ef4f14abfdc7880b8b5ac0f473708675eca98d0266508c00a24349ad40", "task_id": "realistic-browser-login-01", "turn_limit_stops": 0, "turns_factor": 1.5 diff --git a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/triggers.json b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/triggers.json index b2dbd98..661f095 100644 --- a/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/triggers.json +++ b/plugins/rf-web/evals/baselines/openrouter/glm-5.3-flash/triggers.json @@ -10,7 +10,7 @@ "accuracy": 0.913, "fn": 0, "fp": 2, - "negative_loads": 7, + "negative_loads": 6, "negative_runs": 33, "positive_loads": 36, "positive_runs": 36, @@ -21,40 +21,40 @@ "tp": 12 }, "validation": { - "accuracy": 0.875, + "accuracy": 1.0, "fn": 0, - "fp": 1, - "negative_loads": 2, + "fp": 0, + "negative_loads": 1, "negative_runs": 12, "positive_loads": 12, "positive_runs": 12, - "precision": 0.8, + "precision": 1.0, "queries": 8, "recall": 1.0, - "tn": 3, + "tn": 4, "tp": 4 } }, "rf-selenium": { "train": { - "accuracy": 0.9565, + "accuracy": 0.913, "fn": 0, - "fp": 1, + "fp": 2, "negative_loads": 6, "negative_runs": 33, "positive_loads": 36, "positive_runs": 36, - "precision": 0.9231, + "precision": 0.8571, "queries": 23, "recall": 1.0, - "tn": 10, + "tn": 9, "tp": 12 }, "validation": { "accuracy": 0.625, "fn": 0, "fp": 3, - "negative_loads": 8, + "negative_loads": 7, "negative_runs": 12, "positive_loads": 12, "positive_runs": 12,