From bba998d42dfd98404c6f907937aaeb893356ee8d Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Sun, 15 Nov 2026 09:50:52 +0300 Subject: [PATCH 01/15] feat: add rf-test-quality mutant checks skill Adds a skill, a Mutant Must Fail resource and a stdlib kill ledger so a new or changed test is shown to fail when its check is broken. An empty ledger reports rate 0.0, never a full kill. Verified: pytest plugins/rf-core/tests/test_test_quality_skill.py, 4 passed; with the ledger broken, 3 failed. build-catalog --check, governance, validate-skills, plugin references and docs voice pass. --- .../skills/rf-test-quality/SKILL.md | 44 +++++++++++++ .../rf-test-quality/assets/mutants.resource | 22 +++++++ .../scripts/mutation_ledger.py | 64 +++++++++++++++++++ .../narrow/narrow-test-quality-mutant-01.yaml | 33 ++++++++++ .../evals/triggers/rf-test-quality.yaml | 20 ++++++ .../rf-core/skills/rf-test-quality/SKILL.md | 44 +++++++++++++ .../rf-test-quality/assets/mutants.resource | 22 +++++++ .../scripts/mutation_ledger.py | 64 +++++++++++++++++++ .../rf-core/tests/test_test_quality_skill.py | 37 +++++++++++ 9 files changed, 350 insertions(+) create mode 100644 bundles/rf-agentskills/skills/rf-test-quality/SKILL.md create mode 100644 bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource create mode 100644 bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py create mode 100644 plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml create mode 100644 plugins/rf-core/evals/triggers/rf-test-quality.yaml create mode 100644 plugins/rf-core/skills/rf-test-quality/SKILL.md create mode 100644 plugins/rf-core/skills/rf-test-quality/assets/mutants.resource create mode 100644 plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py create mode 100644 plugins/rf-core/tests/test_test_quality_skill.py diff --git a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md new file mode 100644 index 0000000..578eb3a --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md @@ -0,0 +1,44 @@ +--- +name: rf-test-quality +description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." +license: Apache-2.0 +compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +metadata: + author: manykarim + version: "1.0.0" +--- + +# Robot Framework Test Quality Skill + +## When to use + +Load this skill after you add or change a test, before you call it done. A test that passes against broken code proves nothing. + +## The rule + +Every new or changed test is seen failing once. Break the thing it checks, run it, and confirm it goes red. Then restore the code and confirm it goes green. + +## Mutant checks in a suite + +Import `assets/mutants.resource` and wrap a check in `Mutant Must Fail` with a mutated input or expectation: + +```robotframework +*** Settings *** +Resource mutants.resource + +*** Test Cases *** +Status Check Catches A Wrong Code + ${killed}= Mutant Must Fail T1 flip-status Should Be Equal As Integers 200 500 + Should Be True ${killed} +``` + +Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. + +## Reading the ledger + +```bash +python scripts/mutation_ledger.py rate results/mutation-ledger.jsonl +python scripts/mutation_ledger.py selftest +``` + +An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource b/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource new file mode 100644 index 0000000..3ea4ec6 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource @@ -0,0 +1,22 @@ +*** Settings *** +Documentation Mutant checks: a new or changed test must fail when its code or data is broken. +Library ../scripts/mutation_ledger.py + +*** Variables *** +${MUTATION_LOG} ${OUTPUT_DIR}${/}mutation-ledger.jsonl + +*** Keywords *** +Mutant Must Fail + [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail. + ... Writes one ledger row and returns True when the mutant was killed. + ... A survivor is a finding about the check, never a pass. + [Arguments] ${test_id} ${operator} ${keyword} @{args} + ${passed}= Run Keyword And Return Status ${keyword} @{args} + ${killed}= Evaluate not $passed + Record Mutation ${MUTATION_LOG} ${test_id} ${operator} ${killed} + RETURN ${killed} + +Mutation Kill Rate + [Documentation] Returns ``{killed, total, rate, survivors}`` read from the ledger. + ${r}= Kill Rate ${MUTATION_LOG} + RETURN ${r} diff --git a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py new file mode 100644 index 0000000..75a40e4 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Kill ledger for hand-made mutants. Stdlib only. + +A check is worth keeping only when it fails once its code or data is broken. +Each mutant run appends one JSON line; the kill rate is read back from the file. + + python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} + python scripts/mutation_ledger.py selftest + +Also importable as a Robot Framework library (see assets/mutants.resource). +""" +import json +import os +import sys +import tempfile + + +def record_mutation(path, test_id, operator, killed): + """Append one row. ``killed`` is True when the mutated run failed.""" + row = {"test": str(test_id), "operator": str(operator), "killed": bool(killed)} + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(row) + "\n") + return row + + +def kill_rate(path): + """Summarise the ledger. A missing or empty ledger has rate 0.0, never 1.0.""" + rows = [] + if os.path.exists(path): + with open(path, encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] + killed = sum(1 for r in rows if r["killed"]) + survivors = [f'{r["test"]}:{r["operator"]}' for r in rows if not r["killed"]] + total = len(rows) + return {"killed": killed, "total": total, + "rate": round(killed / total, 3) if total else 0.0, + "survivors": survivors} + + +def _selftest(): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "kills.jsonl") + assert kill_rate(p)["rate"] == 0.0 + record_mutation(p, "T1", "flip-status", True) + record_mutation(p, "T1", "drop-field", False) + r = kill_rate(p) + assert r == {"killed": 1, "total": 2, "rate": 0.5, + "survivors": ["T1:drop-field"]}, r + print(json.dumps({"selftest": "ok"})) + + +def main(argv): + if argv[:1] == ["selftest"]: + _selftest() + return 0 + if len(argv) == 2 and argv[0] == "rate": + print(json.dumps(kill_rate(argv[1]))) + return 0 + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml b/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml new file mode 100644 index 0000000..dab0517 --- /dev/null +++ b/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml @@ -0,0 +1,33 @@ +id: narrow-test-quality-mutant-01 +skill: rf-test-quality +tier: narrow +description: | + Add a mutant check for an existing equality test and run it. Gated by a + Mutant Must Fail call in the suite and a mutation ledger row written by + the run. +prompt: | + tests/smoke.robot checks that a value equals 200. Prove the check fails + when the expected value is wrong: add a mutant check with a wrong value, + run the suite into results/, and report the kill rate. +model: claude-haiku-5-5 +max_turns: 14 +timeout_seconds: 300 +allowed_tools: + - Read + - Write + - Edit + - Bash + - Glob + - Grep + - Skill +grader_checks: + - type: file_contains + name: suite_uses_mutant_check + path: tests/smoke.robot + regex: 'Mutant Must Fail' + - type: file_contains + name: ledger_row_written + path: results/mutation-ledger.jsonl + regex: '"killed": true' +fixture: sut-minimal +primary_metric: file_contains diff --git a/plugins/rf-core/evals/triggers/rf-test-quality.yaml b/plugins/rf-core/evals/triggers/rf-test-quality.yaml new file mode 100644 index 0000000..5d113db --- /dev/null +++ b/plugins/rf-core/evals/triggers/rf-test-quality.yaml @@ -0,0 +1,20 @@ +# Trigger query set for rf-test-quality (format: eval/triggers/README.md). +# Tune descriptions on the train split only; never edit validation queries +# to make a description pass (rotate them only when the skill's intent changes). +skill: rf-test-quality +model: claude-haiku-5-5 +runs: 3 +threshold: 0.5 +queries: + # should trigger (train) + - {id: tq-t01, split: train, should_trigger: true, query: "I added a Robot Framework test, how do I prove it fails when the code is broken?"} + - {id: tq-t02, split: train, should_trigger: true, query: "Add mutant checks to my Robot suite and report the kill rate"} + - {id: tq-t03, split: train, should_trigger: true, note: "user phrasing", query: "my robot test always passes even when the endpoint is wrong, is the check too weak?"} + # should trigger (validation) + - {id: tq-v01, split: validation, should_trigger: true, query: "Show that the new status code check in tests/api.robot goes red on a wrong value"} + - {id: tq-v02, split: validation, should_trigger: true, query: "Which of my changed Robot tests survive a mutated expectation?"} + # should NOT trigger (train) + - {id: tq-n01, split: train, should_trigger: false, note: "sibling rf-setup", query: "Install robotframework-browser into our uv project"} + - {id: tq-n02, split: train, should_trigger: false, note: "sibling rf-language", query: "Add a suite setup in an __init__.robot file"} + # should NOT trigger (validation) + - {id: tq-nv01, split: validation, should_trigger: false, note: "non-RF look-alike", query: "Run mutmut on my Python package"} diff --git a/plugins/rf-core/skills/rf-test-quality/SKILL.md b/plugins/rf-core/skills/rf-test-quality/SKILL.md new file mode 100644 index 0000000..578eb3a --- /dev/null +++ b/plugins/rf-core/skills/rf-test-quality/SKILL.md @@ -0,0 +1,44 @@ +--- +name: rf-test-quality +description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." +license: Apache-2.0 +compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +metadata: + author: manykarim + version: "1.0.0" +--- + +# Robot Framework Test Quality Skill + +## When to use + +Load this skill after you add or change a test, before you call it done. A test that passes against broken code proves nothing. + +## The rule + +Every new or changed test is seen failing once. Break the thing it checks, run it, and confirm it goes red. Then restore the code and confirm it goes green. + +## Mutant checks in a suite + +Import `assets/mutants.resource` and wrap a check in `Mutant Must Fail` with a mutated input or expectation: + +```robotframework +*** Settings *** +Resource mutants.resource + +*** Test Cases *** +Status Check Catches A Wrong Code + ${killed}= Mutant Must Fail T1 flip-status Should Be Equal As Integers 200 500 + Should Be True ${killed} +``` + +Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. + +## Reading the ledger + +```bash +python scripts/mutation_ledger.py rate results/mutation-ledger.jsonl +python scripts/mutation_ledger.py selftest +``` + +An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource b/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource new file mode 100644 index 0000000..3ea4ec6 --- /dev/null +++ b/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource @@ -0,0 +1,22 @@ +*** Settings *** +Documentation Mutant checks: a new or changed test must fail when its code or data is broken. +Library ../scripts/mutation_ledger.py + +*** Variables *** +${MUTATION_LOG} ${OUTPUT_DIR}${/}mutation-ledger.jsonl + +*** Keywords *** +Mutant Must Fail + [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail. + ... Writes one ledger row and returns True when the mutant was killed. + ... A survivor is a finding about the check, never a pass. + [Arguments] ${test_id} ${operator} ${keyword} @{args} + ${passed}= Run Keyword And Return Status ${keyword} @{args} + ${killed}= Evaluate not $passed + Record Mutation ${MUTATION_LOG} ${test_id} ${operator} ${killed} + RETURN ${killed} + +Mutation Kill Rate + [Documentation] Returns ``{killed, total, rate, survivors}`` read from the ledger. + ${r}= Kill Rate ${MUTATION_LOG} + RETURN ${r} diff --git a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py new file mode 100644 index 0000000..75a40e4 --- /dev/null +++ b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Kill ledger for hand-made mutants. Stdlib only. + +A check is worth keeping only when it fails once its code or data is broken. +Each mutant run appends one JSON line; the kill rate is read back from the file. + + python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} + python scripts/mutation_ledger.py selftest + +Also importable as a Robot Framework library (see assets/mutants.resource). +""" +import json +import os +import sys +import tempfile + + +def record_mutation(path, test_id, operator, killed): + """Append one row. ``killed`` is True when the mutated run failed.""" + row = {"test": str(test_id), "operator": str(operator), "killed": bool(killed)} + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(row) + "\n") + return row + + +def kill_rate(path): + """Summarise the ledger. A missing or empty ledger has rate 0.0, never 1.0.""" + rows = [] + if os.path.exists(path): + with open(path, encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] + killed = sum(1 for r in rows if r["killed"]) + survivors = [f'{r["test"]}:{r["operator"]}' for r in rows if not r["killed"]] + total = len(rows) + return {"killed": killed, "total": total, + "rate": round(killed / total, 3) if total else 0.0, + "survivors": survivors} + + +def _selftest(): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "kills.jsonl") + assert kill_rate(p)["rate"] == 0.0 + record_mutation(p, "T1", "flip-status", True) + record_mutation(p, "T1", "drop-field", False) + r = kill_rate(p) + assert r == {"killed": 1, "total": 2, "rate": 0.5, + "survivors": ["T1:drop-field"]}, r + print(json.dumps({"selftest": "ok"})) + + +def main(argv): + if argv[:1] == ["selftest"]: + _selftest() + return 0 + if len(argv) == 2 and argv[0] == "rate": + print(json.dumps(kill_rate(argv[1]))) + return 0 + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/plugins/rf-core/tests/test_test_quality_skill.py b/plugins/rf-core/tests/test_test_quality_skill.py new file mode 100644 index 0000000..cd947a1 --- /dev/null +++ b/plugins/rf-core/tests/test_test_quality_skill.py @@ -0,0 +1,37 @@ +"""rf-test-quality: the kill ledger counts survivors and never reports an empty run as killed.""" +import importlib.util +import subprocess +import sys +from pathlib import Path + +SKILL = Path(__file__).resolve().parents[1] / "skills" / "rf-test-quality" +SCRIPT = SKILL / "scripts" / "mutation_ledger.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("mutation_ledger", SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def test_survivor_is_counted(tmp_path): + m = _load() + p = tmp_path / "k.jsonl" + m.record_mutation(str(p), "T1", "flip", True) + m.record_mutation(str(p), "T1", "drop", False) + assert m.kill_rate(str(p)) == {"killed": 1, "total": 2, "rate": 0.5, "survivors": ["T1:drop"]} + + +def test_empty_ledger_is_not_a_full_kill(tmp_path): + assert _load().kill_rate(str(tmp_path / "none.jsonl"))["rate"] == 0.0 + + +def test_selftest_passes(): + r = subprocess.run([sys.executable, str(SCRIPT), "selftest"], capture_output=True, text=True) + assert r.returncode == 0 and '"ok"' in r.stdout + + +def test_resource_defines_keywords(): + text = (SKILL / "assets" / "mutants.resource").read_text(encoding="utf-8") + assert "Mutant Must Fail" in text and "Mutation Kill Rate" in text From bf368ff4fa1e8b92dbcb5a41be1f817da556195c Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Sun, 15 Nov 2026 09:57:02 +0300 Subject: [PATCH 02/15] feat: add rf-run-digest skill Add the rf-core skill rf-run-digest: read a short digest of a Robot Framework run (summary, failures grouped by signature) before output.xml. Ships digest_listener.py and run_digest.py (stdlib only), a pytest that runs the selftest and checks summary and grouped failures on a tiny output.xml, a trigger query set and one narrow eval task. Bundle regenerated with build-catalog.py. Verified: rf-core pytest 332 passed; the new summary test failed with run_digest.py broken; build-catalog --check, check-governance, validate-skills --channel all, check-plugin-references, check-docs-voice and check_content_rules all exit 0. --- .../skills/rf-run-digest/SKILL.md | 45 +++ .../rf-run-digest/scripts/digest_listener.py | 64 ++++ .../rf-run-digest/scripts/run_digest.py | 318 ++++++++++++++++++ .../tasks/narrow/narrow-run-digest-01.yaml | 27 ++ .../rf-core/evals/triggers/rf-run-digest.yaml | 20 ++ plugins/rf-core/skills/rf-run-digest/SKILL.md | 45 +++ .../rf-run-digest/scripts/digest_listener.py | 64 ++++ .../rf-run-digest/scripts/run_digest.py | 318 ++++++++++++++++++ .../rf-core/tests/test_run_digest_skill.py | 36 ++ 9 files changed, 937 insertions(+) create mode 100644 bundles/rf-agentskills/skills/rf-run-digest/SKILL.md create mode 100644 bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py create mode 100644 bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py create mode 100644 plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml create mode 100644 plugins/rf-core/evals/triggers/rf-run-digest.yaml create mode 100644 plugins/rf-core/skills/rf-run-digest/SKILL.md create mode 100644 plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py create mode 100644 plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py create mode 100644 plugins/rf-core/tests/test_run_digest_skill.py diff --git a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md new file mode 100644 index 0000000..0326289 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md @@ -0,0 +1,45 @@ +--- +name: rf-run-digest +description: "Use after a Robot Framework run, to read its result from a short digest (summary, grouped failures) instead of output.xml or log.html." +license: Apache-2.0 +compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +metadata: + author: manykarim + version: "1.0.0" +--- + +# Robot Framework Run Digest Skill + +## When to use + +Load this skill after a `robot` or `pabot` run, before you look at its result. output.xml and log.html are large; reading them wastes time and hides the few failure groups that matter. + +## The rule + +Read the digest first, not output.xml. Open output.xml only for one named test, with `trace`. + +## Write a digest on every run + +Attach `scripts/digest_listener.py` as a listener. When the run ends, it writes `digest.txt` next to output.xml: + +```bash +robot --listener scripts/digest_listener.py -d results/run tests/ +cat results/run/digest.txt +``` + +## Read an existing output.xml + +```bash +python scripts/run_digest.py summary results/output.xml # totals and per-suite counts +python scripts/run_digest.py failures results/output.xml --max 3 # failures grouped by signature +python scripts/run_digest.py diff before.xml after.xml # what changed between two runs +python scripts/run_digest.py trace results/output.xml --test "Name" +python scripts/run_digest.py selftest +``` + +Failures that differ only by ids, hosts or timestamps fall into one group, so a run with many failures reads as a few groups. Quote the summary line and the group messages when you report a run. + +## Companion Skills + +- `rf-setup`: install Robot Framework before the first run. +- `rf-language`: fix the suite once the digest names the failing test. diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py new file mode 100644 index 0000000..1535392 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +"""Robot listener: shrink output.xml, write digest.txt, point the human at report.html. + +Attach it to any `robot` or `pabot` command; it fires when the run tears down. + + robot --listener tools/digest_listener.py -d results/run \ + --output output.xml --log NONE --report NONE tests/ > /dev/null 2>&1 + cat results/run/digest.txt + +Always-on for a shell: + + # PowerShell + $env:ROBOT_OPTIONS = '--listener "C:\\path\\to\\tools\\digest_listener.py"' + # bash + export ROBOT_OPTIONS='--listener /abs/path/tools/digest_listener.py' + +Read digest.txt first. A person opens robot's own report.html (the listener prints its path +when a run fails and never opens it). Neither one ever reads output.xml or log.html. + +Under pabot each process writes its own digest.txt; read the merged run's instead +(`uv run python tools/run_digest.py failures `). + +Errors are swallowed: a reporting tool must never fail a run. +""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import run_digest # noqa: E402 (sibling file) + +ROBOT_LISTENER_API_VERSION = 3 + +MAX_PER_GROUP = int(os.environ.get("DIGEST_MAX_PER_GROUP", "3")) + + +class digest_listener: # noqa: N801 (robot resolves the listener by file/class name) + + def __init__(self): + self.output_xml = None + self.report = None + + def output_file(self, path): + self.output_xml = str(path) + + def report_file(self, path): + self.report = str(path) if path and str(path) != "NONE" else None + + def close(self): + try: + self._write_digest() + except Exception as e: # never raise into the run + print("[digest_listener] skipped: %s" % e) + + def _write_digest(self): + if not self.output_xml or not os.path.exists(self.output_xml): + return + run_digest.shrink(self.output_xml) # keywords of PASSED tests dropped + text, failed = run_digest.digest(self.output_xml, MAX_PER_GROUP) + path = os.path.join(os.path.dirname(self.output_xml), "digest.txt") + with open(path, "w", encoding="utf-8") as f: + f.write(text) + print("\n[digest_listener] %d failed -> %s" % (failed, path)) + if failed and self.report and os.path.exists(self.report): + print("[digest_listener] report: %s" % os.path.abspath(self.report)) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py new file mode 100644 index 0000000..851cdd9 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Read a robot run without reading output.xml. Stdlib only. + + uv run python tools/run_digest.py summary OUT.xml # totals + per-suite + uv run python tools/run_digest.py failures OUT.xml [--max 3] # grouped by signature + uv run python tools/run_digest.py diff BEFORE.xml AFTER.xml # status vs message changes + uv run python tools/run_digest.py trace OUT.xml --test NAME # keyword path to the failure + uv run python tools/run_digest.py selftest + +A 94-failure run reads as ~2 groups. Never `cat` output.xml, never open log.html. +""" +import hashlib +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +UUID = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.I) +URL = re.compile(r"https?://\S+|/api/\S*") +TS = re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}\S*") +PATH_URL = re.compile(r"path_url=(\S+)") +BODY = re.compile(r"\bbody=(.*?)(?:\n|$)", re.S) +STATUS = re.compile(r"\bstatus=(\d+)") + + +def _path_template(m): + """Keep WHICH endpoint failed, drop what varies: host, query string, ids -> {id}.""" + u = re.sub(r"^https?://[^/\s]+", "", m.group(0)).split("?")[0] + u = UUID.sub("{id}", u) + return re.sub(r"/\d+(?=/|$)", "/{id}", u) or "/" + + +def norm(msg): + """Signature for grouping: drop what varies per test, keep numbers (400 != 409) and the + endpoint's path template (a 504 on /orders is not the same finding as a 504 on /stock).""" + s = URL.sub(_path_template, msg or "") + s = UUID.sub("", s) + s = TS.sub("", s) + return " ".join(s.split())[:200] + + +def fingerprint(sig): + """Stable id of a failure group across runs. Put fp- on the BUGS.md entry and look it up + there before filing a new issue.""" + return "fp-" + hashlib.sha256(sig.encode("utf-8")).hexdigest()[:12] + + +def clip(s, n=160): + s = " ".join((s or "").split()) + return s[:n] + (" ..." if len(s) > n else "") + + +def unwrap(body): + """Requests logs payloads as a bytes repr; GETs and empty responses log None. + + Strip the b'...' wrapper and drop None on both sides, or every GET (and every 401 with an + empty body) grows a junk line. + """ + body = (body or "").strip() + if body in ("None", ""): + return None + if body[:2] in ("b'", 'b"'): + body = body[2:-1] + return body + + +def _grab(rx, text): + m = rx.search(text) + return m.group(1) if m else None + + +def parse(path): + """-> one dict per test, in run order. Request context is kept for every test. + + The payload is reset at each new request: scanning for "the last body= on any request" + made a failing GET inherit an earlier POST's payload in the same test, and a sent: line + from a different endpoint than the last request: line above it is worse than no line. + The label is "last request", not "the failing request": in a multi-request test the + assertion can be about an earlier call; `trace` gives the failing keyword path. + """ + tests, suites, cur = [], [], None + for event, el in ET.iterparse(path, events=("start", "end")): + tag = el.tag + if event == "start": + if tag == "suite": + suites.append(el.get("name")) + elif tag == "test": + cur = {"name": el.get("name"), "suite": suites[-1], + "status": "", "msg": "", "sent": None, "req": None, "res": None, + "body": None, "kws": []} + continue + if cur is not None and tag == "msg": + text = el.text or "" + m = PATH_URL.search(text) + if m: # a new request: reset the payload + cur["req"], cur["sent"] = m.group(1), unwrap(_grab(BODY, text)) + elif STATUS.search(text) and "body=" in text: + cur["res"] = STATUS.search(text).group(1) + cur["body"] = unwrap(_grab(BODY, text)) + elif cur is not None and tag == "kw": + st = el.find("status") + cur["kws"].append((el.get("name") or "", st.get("status", "") if st is not None else "")) + elif tag == "test": + st = el.find("status") + cur["status"] = st.get("status", "") if st is not None else "" + cur["msg"] = (st.text or "").strip() if st is not None else "" + tests.append(cur) + cur = None + el.clear() + elif tag == "suite": + suites.pop() + el.clear() + return tests + + +def cmd_summary(tests): + fail = [t for t in tests if t["status"] == "FAIL"] + skip = sum(t["status"] == "SKIP" for t in tests) + # SKIP is neither: counting it as passed hid a chained test that never ran. + line = "%d tests, %d passed, %d failed" % (len(tests), len(tests) - len(fail) - skip, len(fail)) + print(line + (", %d skipped" % skip if skip else "")) + per = {} + for t in tests: + p, f = per.get(t["suite"], (0, 0)) + per[t["suite"]] = (p + (t["status"] == "PASS"), f + (t["status"] == "FAIL")) + for suite, (p, f) in sorted(per.items(), key=lambda kv: -kv[1][1]): + if f: + print(" %3d failed / %-4d %s" % (f, p + f, suite)) + + +def cmd_failures(tests, max_per_group=3): + groups = {} + for t in tests: + if t["status"] == "FAIL": + groups.setdefault(norm(t["msg"]), []).append(t) + if not groups: + return print("no failures") + for sig, ts in sorted(groups.items(), key=lambda kv: -len(kv[1])): + print("\n %dx %s [%s]" % (len(ts), sig, fingerprint(sig))) + for t in ts[:max_per_group]: + print(" %s :: %s" % (t["suite"], t["name"])) + if t["req"]: + print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) + if t["sent"]: + print(" sent: %s" % clip(t["sent"])) + if t["body"]: + print(" body: %s" % clip(t["body"])) + if len(ts) > max_per_group: + print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) + + +def cmd_diff(before, after): + b = dict(((t["suite"], t["name"]), t) for t in before) + a = dict(((t["suite"], t["name"]), t) for t in after) + status, message = [], 0 + for k in sorted(set(a) & set(b)): + if a[k]["status"] != b[k]["status"]: + status.append((k, b[k]["status"], a[k]["status"])) + elif norm(a[k]["msg"]) != norm(b[k]["msg"]): + message += 1 + print("%d status changes, %d message changes, +%d new, -%d gone" + % (len(status), message, len(set(a) - set(b)), len(set(b) - set(a)))) + for (suite, name), old, new in status[:40]: + print(" %s -> %s %s :: %s" % (old, new, suite, name)) + + +def cmd_trace(tests, name): + hits = [t for t in tests if t["name"] == name] + if not hits: + sys.exit("no test named %r" % name) + for t in hits: + print("%s :: %s [%s]" % (t["suite"], t["name"], t["status"])) + for kw, st in t["kws"]: + if st == "FAIL": + print(" FAIL %s" % kw) + print(" message: %s" % clip(t["msg"], 400)) + if t["req"]: + print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) + + +def shrink(output_xml): + """rebot --removekeywords PASSED, out of process. 25.7 MB -> 2.7 MB, same failures. + + Out of process on purpose: robot's LOGGER is a global singleton, so calling rebot() + in-process from a listener re-fires that same listener's close() forever. + `--removekeywords` on robot's own command line does not shrink output.xml: it only + filters the HTML (649,732 vs 649,806 bytes measured). Only a rebot pass removes anything. + """ + tmp = output_xml + ".small" + r = subprocess.run([sys.executable, "-m", "robot.rebot", "--removekeywords", "PASSED", + "--log", "NONE", "--report", "NONE", "--output", tmp, output_xml], + capture_output=True) + # rebot's exit code is the FAILED TEST COUNT (0-249), not a success flag. + if r.returncode < 250 and os.path.exists(tmp): + os.replace(tmp, output_xml) + return True + return False + + +def digest(output_xml, max_per_group=3): + """summary + failures as one string. Used by digest_listener.py.""" + import io + tests = parse(output_xml) + buf, real = io.StringIO(), sys.stdout + try: + sys.stdout = buf + cmd_summary(tests) + cmd_failures(tests, max_per_group) + finally: + sys.stdout = real + return buf.getvalue(), sum(1 for t in tests if t["status"] == "FAIL") + + +def main(argv): + cmd = argv[0] + if cmd == "diff": + return cmd_diff(parse(argv[1]), parse(argv[2])) + tests = parse(argv[1]) + if cmd == "summary": + cmd_summary(tests) + elif cmd == "failures": + n = int(argv[argv.index("--max") + 1]) if "--max" in argv else 3 + cmd_summary(tests) + cmd_failures(tests, n) + elif cmd == "trace": + cmd_trace(tests, argv[argv.index("--test") + 1]) + else: + sys.exit(__doc__) + + +SAMPLE = """ + + + + + POST Request : url=http://h/api/orders +path_url=/api/orders/{ID} +body=b'{{"amount": 10}}' + POST Response : url=http://h/api/orders +status=400, reason=Bad Request +body=b'{{"error":"Validation failed"}}' + + 400.0 != 200.0 + + + POST Request : url=http://h/api/orders +path_url=/api/orders/{ID2} +body=b'{{"amount": 11}}' + 400.0 != 200.0 + + + GET Request : url=http://h/api/orders +path_url=/api/orders +body=None + + + + + + + +""" + + +def selftest(): + import tempfile + d = tempfile.mkdtemp() + uid = "11111111-2222-3333-4444-555555555555" + uid2 = "99999999-2222-3333-4444-555555555555" + p = os.path.join(d, "out.xml") + open(p, "w", encoding="utf-8").write(SAMPLE.format(ID=uid, ID2=uid2)) + tests = parse(p) + assert len(tests) == 3, tests + assert [t["status"] for t in tests] == ["FAIL", "FAIL", "PASS"] + t = tests[0] + assert t["suite"] == "01 Orders" and t["req"] == "/api/orders/" + uid, t + assert t["sent"] == '{"amount": 10}' and t["res"] == "400", t + assert "Validation failed" in t["body"] + assert tests[2]["sent"] is None, "a GET body=None must not become a junk sent: line" + # two failures differing only by UUID must collapse into ONE group + sigs = set(norm(x["msg"]) for x in tests if x["status"] == "FAIL") + assert len(sigs) == 1, sigs + assert norm("Url: /api/x/%s Expected 504 != 200" % uid) == "Url: /api/x/{id} Expected 504 != 200" + assert norm("Url: https://h.dev/api/x/42?p=1 504") == "Url: /api/x/{id} 504" + # fingerprint: stable across ids/hosts/timestamps, distinct across status and endpoint + fp = lambda m: fingerprint(norm(m)) + assert fp("Url: /api/o/%s 504 != 200" % uid) == fp("Url: https://x/api/o/%s 504 != 200" % uid2) + assert fp("Url: /api/o/1 400 != 200") != fp("Url: /api/o/1 409 != 200") + assert fp("Url: /api/orders/1 504 != 200") != fp("Url: /api/stock/1 504 != 200") + assert unwrap("None") is None and unwrap("b'{}'") == "{}" + q = os.path.join(d, "after.xml") + open(q, "w", encoding="utf-8").write( + SAMPLE.format(ID=uid, ID2=uid2).replace('status="FAIL">400.0 != 200.0', 'status="PASS">', 1)) + text, nfail = digest(p) + assert nfail == 2 and "2x 400.0 != 200.0" in text, text + import contextlib, io + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + cmd_summary(tests + [dict(tests[2], name="Skipped", status="SKIP")]) + assert buf.getvalue().startswith("4 tests, 1 passed, 2 failed, 1 skipped\n"), buf.getvalue() + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + cmd_summary(tests) + assert buf.getvalue().startswith("3 tests, 1 passed, 2 failed\n"), buf.getvalue() + cmd_diff(tests, parse(q)) + cmd_trace(tests, "Create Order Basic") + print(text) + print("selftest ok") + + +if __name__ == "__main__": + if len(sys.argv) == 2 and sys.argv[1] == "selftest": + selftest() + elif len(sys.argv) < 3: + sys.exit(__doc__) + else: + main(sys.argv[1:]) diff --git a/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml b/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml new file mode 100644 index 0000000..ace273c --- /dev/null +++ b/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml @@ -0,0 +1,27 @@ +id: narrow-run-digest-01 +skill: rf-run-digest +tier: narrow +description: | + Run a suite with the digest listener and report from the digest. Gated by + a digest.txt written next to output.xml that holds the summary line. +prompt: | + Run tests/smoke.robot into results/ so that a digest.txt is written next + to output.xml, then report the pass and fail counts from that digest. +model: claude-haiku-5-5 +max_turns: 14 +timeout_seconds: 300 +allowed_tools: + - Read + - Write + - Edit + - Bash + - Glob + - Grep + - Skill +grader_checks: + - type: file_contains + name: digest_written + path: results/digest.txt + regex: '\d+ tests, \d+ passed' +fixture: sut-minimal +primary_metric: file_contains diff --git a/plugins/rf-core/evals/triggers/rf-run-digest.yaml b/plugins/rf-core/evals/triggers/rf-run-digest.yaml new file mode 100644 index 0000000..67fd508 --- /dev/null +++ b/plugins/rf-core/evals/triggers/rf-run-digest.yaml @@ -0,0 +1,20 @@ +# Trigger query set for rf-run-digest (format: eval/triggers/README.md). +# Tune descriptions on the train split only; never edit validation queries +# to make a description pass (rotate them only when the skill's intent changes). +skill: rf-run-digest +model: claude-haiku-5-5 +runs: 3 +threshold: 0.5 +queries: + # should trigger (train) + - {id: rd-t01, split: train, should_trigger: true, query: "My robot run just finished with 80 failures, what went wrong?"} + - {id: rd-t02, split: train, should_trigger: true, query: "Summarize results/output.xml from the last Robot Framework run"} + - {id: rd-t03, split: train, should_trigger: true, note: "user phrasing", query: "log.html is huge, how do I see which robot tests failed and why?"} + # should trigger (validation) + - {id: rd-v01, split: validation, should_trigger: true, query: "Group the failures in this Robot output.xml by error message"} + - {id: rd-v02, split: validation, should_trigger: true, query: "Compare two Robot Framework runs and tell me which tests changed status"} + # should NOT trigger (train) + - {id: rd-n01, split: train, should_trigger: false, note: "sibling rf-setup", query: "Install robotframework-requests into our uv project"} + - {id: rd-n02, split: train, should_trigger: false, note: "sibling rf-language", query: "Add a suite teardown in an __init__.robot file"} + # should NOT trigger (validation) + - {id: rd-nv01, split: validation, should_trigger: false, note: "non-RF look-alike", query: "Parse this JUnit XML report from pytest"} diff --git a/plugins/rf-core/skills/rf-run-digest/SKILL.md b/plugins/rf-core/skills/rf-run-digest/SKILL.md new file mode 100644 index 0000000..0326289 --- /dev/null +++ b/plugins/rf-core/skills/rf-run-digest/SKILL.md @@ -0,0 +1,45 @@ +--- +name: rf-run-digest +description: "Use after a Robot Framework run, to read its result from a short digest (summary, grouped failures) instead of output.xml or log.html." +license: Apache-2.0 +compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +metadata: + author: manykarim + version: "1.0.0" +--- + +# Robot Framework Run Digest Skill + +## When to use + +Load this skill after a `robot` or `pabot` run, before you look at its result. output.xml and log.html are large; reading them wastes time and hides the few failure groups that matter. + +## The rule + +Read the digest first, not output.xml. Open output.xml only for one named test, with `trace`. + +## Write a digest on every run + +Attach `scripts/digest_listener.py` as a listener. When the run ends, it writes `digest.txt` next to output.xml: + +```bash +robot --listener scripts/digest_listener.py -d results/run tests/ +cat results/run/digest.txt +``` + +## Read an existing output.xml + +```bash +python scripts/run_digest.py summary results/output.xml # totals and per-suite counts +python scripts/run_digest.py failures results/output.xml --max 3 # failures grouped by signature +python scripts/run_digest.py diff before.xml after.xml # what changed between two runs +python scripts/run_digest.py trace results/output.xml --test "Name" +python scripts/run_digest.py selftest +``` + +Failures that differ only by ids, hosts or timestamps fall into one group, so a run with many failures reads as a few groups. Quote the summary line and the group messages when you report a run. + +## Companion Skills + +- `rf-setup`: install Robot Framework before the first run. +- `rf-language`: fix the suite once the digest names the failing test. diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py b/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py new file mode 100644 index 0000000..1535392 --- /dev/null +++ b/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +"""Robot listener: shrink output.xml, write digest.txt, point the human at report.html. + +Attach it to any `robot` or `pabot` command; it fires when the run tears down. + + robot --listener tools/digest_listener.py -d results/run \ + --output output.xml --log NONE --report NONE tests/ > /dev/null 2>&1 + cat results/run/digest.txt + +Always-on for a shell: + + # PowerShell + $env:ROBOT_OPTIONS = '--listener "C:\\path\\to\\tools\\digest_listener.py"' + # bash + export ROBOT_OPTIONS='--listener /abs/path/tools/digest_listener.py' + +Read digest.txt first. A person opens robot's own report.html (the listener prints its path +when a run fails and never opens it). Neither one ever reads output.xml or log.html. + +Under pabot each process writes its own digest.txt; read the merged run's instead +(`uv run python tools/run_digest.py failures `). + +Errors are swallowed: a reporting tool must never fail a run. +""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import run_digest # noqa: E402 (sibling file) + +ROBOT_LISTENER_API_VERSION = 3 + +MAX_PER_GROUP = int(os.environ.get("DIGEST_MAX_PER_GROUP", "3")) + + +class digest_listener: # noqa: N801 (robot resolves the listener by file/class name) + + def __init__(self): + self.output_xml = None + self.report = None + + def output_file(self, path): + self.output_xml = str(path) + + def report_file(self, path): + self.report = str(path) if path and str(path) != "NONE" else None + + def close(self): + try: + self._write_digest() + except Exception as e: # never raise into the run + print("[digest_listener] skipped: %s" % e) + + def _write_digest(self): + if not self.output_xml or not os.path.exists(self.output_xml): + return + run_digest.shrink(self.output_xml) # keywords of PASSED tests dropped + text, failed = run_digest.digest(self.output_xml, MAX_PER_GROUP) + path = os.path.join(os.path.dirname(self.output_xml), "digest.txt") + with open(path, "w", encoding="utf-8") as f: + f.write(text) + print("\n[digest_listener] %d failed -> %s" % (failed, path)) + if failed and self.report and os.path.exists(self.report): + print("[digest_listener] report: %s" % os.path.abspath(self.report)) diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py new file mode 100644 index 0000000..851cdd9 --- /dev/null +++ b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Read a robot run without reading output.xml. Stdlib only. + + uv run python tools/run_digest.py summary OUT.xml # totals + per-suite + uv run python tools/run_digest.py failures OUT.xml [--max 3] # grouped by signature + uv run python tools/run_digest.py diff BEFORE.xml AFTER.xml # status vs message changes + uv run python tools/run_digest.py trace OUT.xml --test NAME # keyword path to the failure + uv run python tools/run_digest.py selftest + +A 94-failure run reads as ~2 groups. Never `cat` output.xml, never open log.html. +""" +import hashlib +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +UUID = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.I) +URL = re.compile(r"https?://\S+|/api/\S*") +TS = re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}\S*") +PATH_URL = re.compile(r"path_url=(\S+)") +BODY = re.compile(r"\bbody=(.*?)(?:\n|$)", re.S) +STATUS = re.compile(r"\bstatus=(\d+)") + + +def _path_template(m): + """Keep WHICH endpoint failed, drop what varies: host, query string, ids -> {id}.""" + u = re.sub(r"^https?://[^/\s]+", "", m.group(0)).split("?")[0] + u = UUID.sub("{id}", u) + return re.sub(r"/\d+(?=/|$)", "/{id}", u) or "/" + + +def norm(msg): + """Signature for grouping: drop what varies per test, keep numbers (400 != 409) and the + endpoint's path template (a 504 on /orders is not the same finding as a 504 on /stock).""" + s = URL.sub(_path_template, msg or "") + s = UUID.sub("", s) + s = TS.sub("", s) + return " ".join(s.split())[:200] + + +def fingerprint(sig): + """Stable id of a failure group across runs. Put fp- on the BUGS.md entry and look it up + there before filing a new issue.""" + return "fp-" + hashlib.sha256(sig.encode("utf-8")).hexdigest()[:12] + + +def clip(s, n=160): + s = " ".join((s or "").split()) + return s[:n] + (" ..." if len(s) > n else "") + + +def unwrap(body): + """Requests logs payloads as a bytes repr; GETs and empty responses log None. + + Strip the b'...' wrapper and drop None on both sides, or every GET (and every 401 with an + empty body) grows a junk line. + """ + body = (body or "").strip() + if body in ("None", ""): + return None + if body[:2] in ("b'", 'b"'): + body = body[2:-1] + return body + + +def _grab(rx, text): + m = rx.search(text) + return m.group(1) if m else None + + +def parse(path): + """-> one dict per test, in run order. Request context is kept for every test. + + The payload is reset at each new request: scanning for "the last body= on any request" + made a failing GET inherit an earlier POST's payload in the same test, and a sent: line + from a different endpoint than the last request: line above it is worse than no line. + The label is "last request", not "the failing request": in a multi-request test the + assertion can be about an earlier call; `trace` gives the failing keyword path. + """ + tests, suites, cur = [], [], None + for event, el in ET.iterparse(path, events=("start", "end")): + tag = el.tag + if event == "start": + if tag == "suite": + suites.append(el.get("name")) + elif tag == "test": + cur = {"name": el.get("name"), "suite": suites[-1], + "status": "", "msg": "", "sent": None, "req": None, "res": None, + "body": None, "kws": []} + continue + if cur is not None and tag == "msg": + text = el.text or "" + m = PATH_URL.search(text) + if m: # a new request: reset the payload + cur["req"], cur["sent"] = m.group(1), unwrap(_grab(BODY, text)) + elif STATUS.search(text) and "body=" in text: + cur["res"] = STATUS.search(text).group(1) + cur["body"] = unwrap(_grab(BODY, text)) + elif cur is not None and tag == "kw": + st = el.find("status") + cur["kws"].append((el.get("name") or "", st.get("status", "") if st is not None else "")) + elif tag == "test": + st = el.find("status") + cur["status"] = st.get("status", "") if st is not None else "" + cur["msg"] = (st.text or "").strip() if st is not None else "" + tests.append(cur) + cur = None + el.clear() + elif tag == "suite": + suites.pop() + el.clear() + return tests + + +def cmd_summary(tests): + fail = [t for t in tests if t["status"] == "FAIL"] + skip = sum(t["status"] == "SKIP" for t in tests) + # SKIP is neither: counting it as passed hid a chained test that never ran. + line = "%d tests, %d passed, %d failed" % (len(tests), len(tests) - len(fail) - skip, len(fail)) + print(line + (", %d skipped" % skip if skip else "")) + per = {} + for t in tests: + p, f = per.get(t["suite"], (0, 0)) + per[t["suite"]] = (p + (t["status"] == "PASS"), f + (t["status"] == "FAIL")) + for suite, (p, f) in sorted(per.items(), key=lambda kv: -kv[1][1]): + if f: + print(" %3d failed / %-4d %s" % (f, p + f, suite)) + + +def cmd_failures(tests, max_per_group=3): + groups = {} + for t in tests: + if t["status"] == "FAIL": + groups.setdefault(norm(t["msg"]), []).append(t) + if not groups: + return print("no failures") + for sig, ts in sorted(groups.items(), key=lambda kv: -len(kv[1])): + print("\n %dx %s [%s]" % (len(ts), sig, fingerprint(sig))) + for t in ts[:max_per_group]: + print(" %s :: %s" % (t["suite"], t["name"])) + if t["req"]: + print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) + if t["sent"]: + print(" sent: %s" % clip(t["sent"])) + if t["body"]: + print(" body: %s" % clip(t["body"])) + if len(ts) > max_per_group: + print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) + + +def cmd_diff(before, after): + b = dict(((t["suite"], t["name"]), t) for t in before) + a = dict(((t["suite"], t["name"]), t) for t in after) + status, message = [], 0 + for k in sorted(set(a) & set(b)): + if a[k]["status"] != b[k]["status"]: + status.append((k, b[k]["status"], a[k]["status"])) + elif norm(a[k]["msg"]) != norm(b[k]["msg"]): + message += 1 + print("%d status changes, %d message changes, +%d new, -%d gone" + % (len(status), message, len(set(a) - set(b)), len(set(b) - set(a)))) + for (suite, name), old, new in status[:40]: + print(" %s -> %s %s :: %s" % (old, new, suite, name)) + + +def cmd_trace(tests, name): + hits = [t for t in tests if t["name"] == name] + if not hits: + sys.exit("no test named %r" % name) + for t in hits: + print("%s :: %s [%s]" % (t["suite"], t["name"], t["status"])) + for kw, st in t["kws"]: + if st == "FAIL": + print(" FAIL %s" % kw) + print(" message: %s" % clip(t["msg"], 400)) + if t["req"]: + print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) + + +def shrink(output_xml): + """rebot --removekeywords PASSED, out of process. 25.7 MB -> 2.7 MB, same failures. + + Out of process on purpose: robot's LOGGER is a global singleton, so calling rebot() + in-process from a listener re-fires that same listener's close() forever. + `--removekeywords` on robot's own command line does not shrink output.xml: it only + filters the HTML (649,732 vs 649,806 bytes measured). Only a rebot pass removes anything. + """ + tmp = output_xml + ".small" + r = subprocess.run([sys.executable, "-m", "robot.rebot", "--removekeywords", "PASSED", + "--log", "NONE", "--report", "NONE", "--output", tmp, output_xml], + capture_output=True) + # rebot's exit code is the FAILED TEST COUNT (0-249), not a success flag. + if r.returncode < 250 and os.path.exists(tmp): + os.replace(tmp, output_xml) + return True + return False + + +def digest(output_xml, max_per_group=3): + """summary + failures as one string. Used by digest_listener.py.""" + import io + tests = parse(output_xml) + buf, real = io.StringIO(), sys.stdout + try: + sys.stdout = buf + cmd_summary(tests) + cmd_failures(tests, max_per_group) + finally: + sys.stdout = real + return buf.getvalue(), sum(1 for t in tests if t["status"] == "FAIL") + + +def main(argv): + cmd = argv[0] + if cmd == "diff": + return cmd_diff(parse(argv[1]), parse(argv[2])) + tests = parse(argv[1]) + if cmd == "summary": + cmd_summary(tests) + elif cmd == "failures": + n = int(argv[argv.index("--max") + 1]) if "--max" in argv else 3 + cmd_summary(tests) + cmd_failures(tests, n) + elif cmd == "trace": + cmd_trace(tests, argv[argv.index("--test") + 1]) + else: + sys.exit(__doc__) + + +SAMPLE = """ + + + + + POST Request : url=http://h/api/orders +path_url=/api/orders/{ID} +body=b'{{"amount": 10}}' + POST Response : url=http://h/api/orders +status=400, reason=Bad Request +body=b'{{"error":"Validation failed"}}' + + 400.0 != 200.0 + + + POST Request : url=http://h/api/orders +path_url=/api/orders/{ID2} +body=b'{{"amount": 11}}' + 400.0 != 200.0 + + + GET Request : url=http://h/api/orders +path_url=/api/orders +body=None + + + + + + + +""" + + +def selftest(): + import tempfile + d = tempfile.mkdtemp() + uid = "11111111-2222-3333-4444-555555555555" + uid2 = "99999999-2222-3333-4444-555555555555" + p = os.path.join(d, "out.xml") + open(p, "w", encoding="utf-8").write(SAMPLE.format(ID=uid, ID2=uid2)) + tests = parse(p) + assert len(tests) == 3, tests + assert [t["status"] for t in tests] == ["FAIL", "FAIL", "PASS"] + t = tests[0] + assert t["suite"] == "01 Orders" and t["req"] == "/api/orders/" + uid, t + assert t["sent"] == '{"amount": 10}' and t["res"] == "400", t + assert "Validation failed" in t["body"] + assert tests[2]["sent"] is None, "a GET body=None must not become a junk sent: line" + # two failures differing only by UUID must collapse into ONE group + sigs = set(norm(x["msg"]) for x in tests if x["status"] == "FAIL") + assert len(sigs) == 1, sigs + assert norm("Url: /api/x/%s Expected 504 != 200" % uid) == "Url: /api/x/{id} Expected 504 != 200" + assert norm("Url: https://h.dev/api/x/42?p=1 504") == "Url: /api/x/{id} 504" + # fingerprint: stable across ids/hosts/timestamps, distinct across status and endpoint + fp = lambda m: fingerprint(norm(m)) + assert fp("Url: /api/o/%s 504 != 200" % uid) == fp("Url: https://x/api/o/%s 504 != 200" % uid2) + assert fp("Url: /api/o/1 400 != 200") != fp("Url: /api/o/1 409 != 200") + assert fp("Url: /api/orders/1 504 != 200") != fp("Url: /api/stock/1 504 != 200") + assert unwrap("None") is None and unwrap("b'{}'") == "{}" + q = os.path.join(d, "after.xml") + open(q, "w", encoding="utf-8").write( + SAMPLE.format(ID=uid, ID2=uid2).replace('status="FAIL">400.0 != 200.0', 'status="PASS">', 1)) + text, nfail = digest(p) + assert nfail == 2 and "2x 400.0 != 200.0" in text, text + import contextlib, io + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + cmd_summary(tests + [dict(tests[2], name="Skipped", status="SKIP")]) + assert buf.getvalue().startswith("4 tests, 1 passed, 2 failed, 1 skipped\n"), buf.getvalue() + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + cmd_summary(tests) + assert buf.getvalue().startswith("3 tests, 1 passed, 2 failed\n"), buf.getvalue() + cmd_diff(tests, parse(q)) + cmd_trace(tests, "Create Order Basic") + print(text) + print("selftest ok") + + +if __name__ == "__main__": + if len(sys.argv) == 2 and sys.argv[1] == "selftest": + selftest() + elif len(sys.argv) < 3: + sys.exit(__doc__) + else: + main(sys.argv[1:]) diff --git a/plugins/rf-core/tests/test_run_digest_skill.py b/plugins/rf-core/tests/test_run_digest_skill.py new file mode 100644 index 0000000..fd420bc --- /dev/null +++ b/plugins/rf-core/tests/test_run_digest_skill.py @@ -0,0 +1,36 @@ +"""rf-run-digest: the digest reports the right totals and groups failures by signature.""" +import subprocess +import sys +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "skills" / "rf-run-digest" / "scripts" / "run_digest.py" + +OUTPUT_XML = """ + + + + Expected 404 but got 200 + Expected 404 but got 200 + + + +""" + + +def _run(*args): + return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True) + + +def test_selftest_passes(): + r = _run("selftest") + assert r.returncode == 0, r.stdout + r.stderr + assert "selftest ok" in r.stdout + + +def test_summary_and_failures_on_tiny_output(tmp_path): + xml = tmp_path / "output.xml" + xml.write_text(OUTPUT_XML, encoding="utf-8") + r = _run("failures", str(xml)) + assert r.returncode == 0, r.stderr + assert r.stdout.startswith("3 tests, 1 passed, 2 failed\n"), r.stdout + assert "2x Expected 404 but got 200" in r.stdout, r.stdout From 65c705a325516b88bcac0203d6f0616e9193c885 Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Sun, 15 Nov 2026 10:20:45 +0300 Subject: [PATCH 03/15] fix: make output.xml shrink opt-in in listener The digest listener rewrote output.xml in place with rebot --removekeywords PASSED on every run, undocumented. It now does so only when given the shrink listener argument (digest_listener.py:shrink). SKILL.md documents the argument and what it removes. Both skill copies are identical. Verified: pytest -q plugins/rf-core/tests/test_run_digest_skill.py (log pytest-blocker1.log in the session scratchpad): 4 passed, 1 failed. The failing test is the body-redaction test fixed in the next commit; the two listener tests failed before this change. --- .../skills/rf-run-digest/SKILL.md | 5 ++ .../rf-run-digest/scripts/digest_listener.py | 11 +++- plugins/rf-core/skills/rf-run-digest/SKILL.md | 5 ++ .../rf-run-digest/scripts/digest_listener.py | 11 +++- .../rf-core/tests/test_run_digest_skill.py | 66 +++++++++++++++++++ 5 files changed, 92 insertions(+), 6 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md index 0326289..67c00e2 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md @@ -27,6 +27,11 @@ robot --listener scripts/digest_listener.py -d results/run tests/ cat results/run/digest.txt ``` +The listener leaves output.xml as robot wrote it. To shrink it, pass the `shrink` +argument: `--listener scripts/digest_listener.py:shrink`. This rewrites output.xml in +place with `rebot --removekeywords PASSED`, so the keywords of passed tests are gone +from it and from any log built from it later. + ## Read an existing output.xml ```bash diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py index 1535392..de082c3 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py +++ b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py @@ -1,5 +1,8 @@ #!/usr/bin/env python -"""Robot listener: shrink output.xml, write digest.txt, point the human at report.html. +"""Robot listener: write digest.txt, point the human at report.html. + +With the `shrink` argument (`--listener digest_listener.py:shrink`) it also rewrites +output.xml in place with the keywords of PASSED tests removed. Off by default. Attach it to any `robot` or `pabot` command; it fires when the run tears down. @@ -35,9 +38,10 @@ class digest_listener: # noqa: N801 (robot resolves the listener by file/class name) - def __init__(self): + def __init__(self, *args): self.output_xml = None self.report = None + self.shrink = "shrink" in args def output_file(self, path): self.output_xml = str(path) @@ -54,7 +58,8 @@ def close(self): def _write_digest(self): if not self.output_xml or not os.path.exists(self.output_xml): return - run_digest.shrink(self.output_xml) # keywords of PASSED tests dropped + if self.shrink: + run_digest.shrink(self.output_xml) # rewrites output.xml without PASSED keywords text, failed = run_digest.digest(self.output_xml, MAX_PER_GROUP) path = os.path.join(os.path.dirname(self.output_xml), "digest.txt") with open(path, "w", encoding="utf-8") as f: diff --git a/plugins/rf-core/skills/rf-run-digest/SKILL.md b/plugins/rf-core/skills/rf-run-digest/SKILL.md index 0326289..67c00e2 100644 --- a/plugins/rf-core/skills/rf-run-digest/SKILL.md +++ b/plugins/rf-core/skills/rf-run-digest/SKILL.md @@ -27,6 +27,11 @@ robot --listener scripts/digest_listener.py -d results/run tests/ cat results/run/digest.txt ``` +The listener leaves output.xml as robot wrote it. To shrink it, pass the `shrink` +argument: `--listener scripts/digest_listener.py:shrink`. This rewrites output.xml in +place with `rebot --removekeywords PASSED`, so the keywords of passed tests are gone +from it and from any log built from it later. + ## Read an existing output.xml ```bash diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py b/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py index 1535392..de082c3 100644 --- a/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py +++ b/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py @@ -1,5 +1,8 @@ #!/usr/bin/env python -"""Robot listener: shrink output.xml, write digest.txt, point the human at report.html. +"""Robot listener: write digest.txt, point the human at report.html. + +With the `shrink` argument (`--listener digest_listener.py:shrink`) it also rewrites +output.xml in place with the keywords of PASSED tests removed. Off by default. Attach it to any `robot` or `pabot` command; it fires when the run tears down. @@ -35,9 +38,10 @@ class digest_listener: # noqa: N801 (robot resolves the listener by file/class name) - def __init__(self): + def __init__(self, *args): self.output_xml = None self.report = None + self.shrink = "shrink" in args def output_file(self, path): self.output_xml = str(path) @@ -54,7 +58,8 @@ def close(self): def _write_digest(self): if not self.output_xml or not os.path.exists(self.output_xml): return - run_digest.shrink(self.output_xml) # keywords of PASSED tests dropped + if self.shrink: + run_digest.shrink(self.output_xml) # rewrites output.xml without PASSED keywords text, failed = run_digest.digest(self.output_xml, MAX_PER_GROUP) path = os.path.join(os.path.dirname(self.output_xml), "digest.txt") with open(path, "w", encoding="utf-8") as f: diff --git a/plugins/rf-core/tests/test_run_digest_skill.py b/plugins/rf-core/tests/test_run_digest_skill.py index fd420bc..7fe892b 100644 --- a/plugins/rf-core/tests/test_run_digest_skill.py +++ b/plugins/rf-core/tests/test_run_digest_skill.py @@ -34,3 +34,69 @@ def test_summary_and_failures_on_tiny_output(tmp_path): assert r.returncode == 0, r.stderr assert r.stdout.startswith("3 tests, 1 passed, 2 failed\n"), r.stdout assert "2x Expected 404 but got 200" in r.stdout, r.stdout + + +LISTENER = SCRIPT.parent / "digest_listener.py" + + +def _load_listener(): + import importlib.util + spec = importlib.util.spec_from_file_location("digest_listener", LISTENER) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def test_listener_leaves_output_xml_alone_by_default(tmp_path, monkeypatch): + mod = _load_listener() + calls = [] + monkeypatch.setattr(mod.run_digest, "shrink", lambda path: calls.append(path)) + xml = tmp_path / "output.xml" + xml.write_text(OUTPUT_XML, encoding="utf-8") + listener = mod.digest_listener() + listener.output_file(str(xml)) + listener.close() + assert calls == [] + assert xml.read_text(encoding="utf-8") == OUTPUT_XML + assert (tmp_path / "digest.txt").exists() + + +def test_listener_shrinks_only_when_asked(tmp_path, monkeypatch): + mod = _load_listener() + calls = [] + monkeypatch.setattr(mod.run_digest, "shrink", lambda path: calls.append(path)) + xml = tmp_path / "output.xml" + xml.write_text(OUTPUT_XML, encoding="utf-8") + listener = mod.digest_listener("shrink") + listener.output_file(str(xml)) + listener.close() + assert calls == [str(xml)] + + +SECRET_XML = """ + + + + POST Request : url=http://h/api/login +path_url=/api/login +body=b'{"password": "hunter2"}' + POST Response : url=http://h/api/login +status=500, reason=Server Error +body=b'{"token": "sekret-tok"}' + + 500 != 200 + + + + +""" + + +def test_failures_never_print_request_or_response_bodies(tmp_path): + xml = tmp_path / "output.xml" + xml.write_text(SECRET_XML, encoding="utf-8") + r = _run("failures", str(xml)) + assert r.returncode == 0, r.stderr + assert "hunter2" not in r.stdout, r.stdout + assert "sekret-tok" not in r.stdout, r.stdout + assert "last request: /api/login" in r.stdout, r.stdout From aa10add8cdac3d5bf4789083e70c1f70cb789992 Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Sun, 15 Nov 2026 10:21:08 +0300 Subject: [PATCH 04/15] fix: drop request and response bodies from digest The failures view printed the last request body (sent:) and response body (body:) of each failed test, so tokens and passwords landed in digest.txt. Bodies are no longer printed; the request line and status stay. Both skill copies are identical. Verified: pytest -q plugins/rf-core/tests/test_run_digest_skill.py (log pytest-blocker2.log in the session scratchpad): 5 passed. The body test failed before this change with the token in the output. --- .../skills/rf-run-digest/scripts/run_digest.py | 5 +---- plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py index 851cdd9..075288e 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py +++ b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py @@ -142,10 +142,7 @@ def cmd_failures(tests, max_per_group=3): print(" %s :: %s" % (t["suite"], t["name"])) if t["req"]: print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) - if t["sent"]: - print(" sent: %s" % clip(t["sent"])) - if t["body"]: - print(" body: %s" % clip(t["body"])) + # Bodies are never printed: they carry tokens and passwords. if len(ts) > max_per_group: print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py index 851cdd9..075288e 100644 --- a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py +++ b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py @@ -142,10 +142,7 @@ def cmd_failures(tests, max_per_group=3): print(" %s :: %s" % (t["suite"], t["name"])) if t["req"]: print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) - if t["sent"]: - print(" sent: %s" % clip(t["sent"])) - if t["body"]: - print(" body: %s" % clip(t["body"])) + # Bodies are never printed: they carry tokens and passwords. if len(ts) > max_per_group: print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) From b51ae239391a90fc3670ffb53bcf0fe8e15f49fa Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Sun, 15 Nov 2026 10:30:56 +0300 Subject: [PATCH 05/15] feat: read UI library failures in digest failures and trace now print a screenshot: line for failed tests, taken from the last the test logged (Browser, SeleniumLibrary, AppiumLibrary, MaestroLibrary). norm() drops the Playwright "Call log:" tail, cuts absolute file paths to their basename and replaces 32+ hex session ids with before grouping. SKILL.md names the libraries. The bundles/rf-agentskills copies match the plugin files (cmp). Verified (log: scratchpad/digest-pytest.log, written this run): - pytest plugins/rf-core/tests/test_run_digest_skill.py: 6 new tests red before the change, then 12 passed - run_digest.py selftest: ok - failures on a real Browser run: 1 TimeoutError group with its FAILURE_SCREENSHOT_1.png line. The Appium fixture is built from the library's message format, not a live run. --- .../skills/rf-run-digest/SKILL.md | 2 + .../rf-run-digest/scripts/run_digest.py | 20 +++- plugins/rf-core/skills/rf-run-digest/SKILL.md | 2 + .../rf-run-digest/scripts/run_digest.py | 20 +++- .../rf-core/tests/test_run_digest_skill.py | 93 +++++++++++++++++++ 5 files changed, 131 insertions(+), 6 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md index 67c00e2..33f0ffe 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md @@ -44,6 +44,8 @@ python scripts/run_digest.py selftest Failures that differ only by ids, hosts or timestamps fall into one group, so a run with many failures reads as a few groups. Quote the summary line and the group messages when you report a run. +It reads Browser, SeleniumLibrary, AppiumLibrary, MaestroLibrary and RequestsLibrary failures; the `screenshot:` line on a failed test comes from the last `` that test logged. + ## Companion Skills - `rf-setup`: install Robot Framework before the first run. diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py index 075288e..cba28f1 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py +++ b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py @@ -22,6 +22,9 @@ PATH_URL = re.compile(r"path_url=(\S+)") BODY = re.compile(r"\bbody=(.*?)(?:\n|$)", re.S) STATUS = re.compile(r"\bstatus=(\d+)") +IMG = re.compile(r"]*?src=\"([^\"]+)\"", re.I) +FILE_PATH = re.compile(r"(?:\b[A-Za-z]:[\\/]|(?", s) + s = HEX_ID.sub("", s) s = TS.sub("", s) return " ".join(s.split())[:200] @@ -88,10 +95,13 @@ def parse(path): elif tag == "test": cur = {"name": el.get("name"), "suite": suites[-1], "status": "", "msg": "", "sent": None, "req": None, "res": None, - "body": None, "kws": []} + "body": None, "shot": None, "kws": []} continue if cur is not None and tag == "msg": text = el.text or "" + shot = IMG.search(text) + if shot: # UI libraries log screenshots as + cur["shot"] = shot.group(1) m = PATH_URL.search(text) if m: # a new request: reset the payload cur["req"], cur["sent"] = m.group(1), unwrap(_grab(BODY, text)) @@ -142,6 +152,8 @@ def cmd_failures(tests, max_per_group=3): print(" %s :: %s" % (t["suite"], t["name"])) if t["req"]: print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) + if t["shot"]: + print(" screenshot: %s" % t["shot"]) # Bodies are never printed: they carry tokens and passwords. if len(ts) > max_per_group: print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) @@ -174,6 +186,8 @@ def cmd_trace(tests, name): print(" message: %s" % clip(t["msg"], 400)) if t["req"]: print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) + if t["shot"] and t["status"] == "FAIL": + print(" screenshot: %s" % t["shot"]) def shrink(output_xml): diff --git a/plugins/rf-core/skills/rf-run-digest/SKILL.md b/plugins/rf-core/skills/rf-run-digest/SKILL.md index 67c00e2..33f0ffe 100644 --- a/plugins/rf-core/skills/rf-run-digest/SKILL.md +++ b/plugins/rf-core/skills/rf-run-digest/SKILL.md @@ -44,6 +44,8 @@ python scripts/run_digest.py selftest Failures that differ only by ids, hosts or timestamps fall into one group, so a run with many failures reads as a few groups. Quote the summary line and the group messages when you report a run. +It reads Browser, SeleniumLibrary, AppiumLibrary, MaestroLibrary and RequestsLibrary failures; the `screenshot:` line on a failed test comes from the last `` that test logged. + ## Companion Skills - `rf-setup`: install Robot Framework before the first run. diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py index 075288e..cba28f1 100644 --- a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py +++ b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py @@ -22,6 +22,9 @@ PATH_URL = re.compile(r"path_url=(\S+)") BODY = re.compile(r"\bbody=(.*?)(?:\n|$)", re.S) STATUS = re.compile(r"\bstatus=(\d+)") +IMG = re.compile(r"]*?src=\"([^\"]+)\"", re.I) +FILE_PATH = re.compile(r"(?:\b[A-Za-z]:[\\/]|(?", s) + s = HEX_ID.sub("", s) s = TS.sub("", s) return " ".join(s.split())[:200] @@ -88,10 +95,13 @@ def parse(path): elif tag == "test": cur = {"name": el.get("name"), "suite": suites[-1], "status": "", "msg": "", "sent": None, "req": None, "res": None, - "body": None, "kws": []} + "body": None, "shot": None, "kws": []} continue if cur is not None and tag == "msg": text = el.text or "" + shot = IMG.search(text) + if shot: # UI libraries log screenshots as + cur["shot"] = shot.group(1) m = PATH_URL.search(text) if m: # a new request: reset the payload cur["req"], cur["sent"] = m.group(1), unwrap(_grab(BODY, text)) @@ -142,6 +152,8 @@ def cmd_failures(tests, max_per_group=3): print(" %s :: %s" % (t["suite"], t["name"])) if t["req"]: print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) + if t["shot"]: + print(" screenshot: %s" % t["shot"]) # Bodies are never printed: they carry tokens and passwords. if len(ts) > max_per_group: print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) @@ -174,6 +186,8 @@ def cmd_trace(tests, name): print(" message: %s" % clip(t["msg"], 400)) if t["req"]: print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) + if t["shot"] and t["status"] == "FAIL": + print(" screenshot: %s" % t["shot"]) def shrink(output_xml): diff --git a/plugins/rf-core/tests/test_run_digest_skill.py b/plugins/rf-core/tests/test_run_digest_skill.py index 7fe892b..b134365 100644 --- a/plugins/rf-core/tests/test_run_digest_skill.py +++ b/plugins/rf-core/tests/test_run_digest_skill.py @@ -100,3 +100,96 @@ def test_failures_never_print_request_or_response_bodies(tmp_path): assert "hunter2" not in r.stdout, r.stdout assert "sekret-tok" not in r.stdout, r.stdout assert "last request: /api/login" in r.stdout, r.stdout + + +def _ui_xml(tests): + """tests: (name, status, screenshot_html_or_None, message). Message text is XML-escaped.""" + from xml.sax.saxutils import escape + rows = [] + for i, (name, status, shot, msg) in enumerate(tests, 1): + kw = '' % status + if shot: + kw = ('%s' + '' % (escape(shot), status)) + rows.append('%s%s' + % (i, name, kw, status, escape(msg))) + return ('\n%s' + '' % "".join(rows)) + + +def _failures(tmp_path, tests): + xml = tmp_path / "output.xml" + xml.write_text(_ui_xml(tests), encoding="utf-8") + r = _run("failures", str(xml)) + assert r.returncode == 0, r.stderr + return r.stdout + + +BROWSER_SHOT = '' +BROWSER_MSG = ("TimeoutError: locator.waitFor: Timeout 60000ms exceeded.\nCall log:\n" + " - waiting for locator('role=main') to be visible\n - %d x locator resolved to hidden") + + +def test_browser_failures_group_across_call_log_and_show_screenshot(tmp_path): + out = _failures(tmp_path, [ + ("Passes", "PASS", '', ""), + ("One", "FAIL", BROWSER_SHOT, BROWSER_MSG % 12), + ("Two", "FAIL", BROWSER_SHOT, BROWSER_MSG % 31), + ]) + assert "2x TimeoutError: locator.waitFor: Timeout 60000ms exceeded. [" in out, out + assert "screenshot: Direct_Calls_With_IAM_Token_FAILURE_SCREENSHOT_1.png" in out, out + assert "pass-shot.png" not in out, out + + +def test_selenium_failure_shows_screenshot(tmp_path): + out = _failures(tmp_path, [("Login", "FAIL", '', + "Element 'id=login' not visible after 5 seconds.")]) + assert "1x Element 'id=login' not visible after 5 seconds." in out, out + assert "screenshot: selenium-screenshot-1.png" in out, out + + +def test_appium_failure_shows_screenshot(tmp_path): + # Built from AppiumLibrary's message format, not from a live run. + shot = '' + out = _failures(tmp_path, [("Tap", "FAIL", shot, + "Element locator 'accessibility_id=Login' did not match any elements.")]) + assert "1x Element locator 'accessibility_id=Login' did not match any elements." in out, out + assert "screenshot: appium-screenshot-1.png" in out, out + + +def test_maestro_failures_group_across_absolute_paths(tmp_path): + msg = ("Screenshot size mismatch: Assert screenshot matches %s (threshold: 100%%)" + " - expected 900x2000") + shot = '' + out = _failures(tmp_path, [ + ("A", "FAIL", shot, msg % "C:/Users/x/scratchpad/diff-out/probe-full.png"), + ("B", "FAIL", shot, msg % "/home/ci/run2/probe-full.png"), + ("C", "FAIL", shot, "Element not found: Text matching regex: Network.*"), + ]) + assert "2x Screenshot size mismatch: Assert screenshot matches probe-full.png" in out, out + assert "1x Element not found: Text matching regex: Network.*" in out, out + assert "screenshot: maestro-screenshot-1.jpg" in out, out + assert "C:/Users" not in out, out + + +def test_requests_failure_keeps_endpoint_and_no_screenshot(tmp_path): + xml = tmp_path / "output.xml" + xml.write_text(SECRET_XML, encoding="utf-8") + out = _run("failures", str(xml)).stdout + assert "last request: /api/login -> 500" in out, out + assert "screenshot:" not in out, out + + +def test_session_ids_collapse(tmp_path): + out = _failures(tmp_path, [ + ("A", "FAIL", None, "invalid session id: " + "a" * 32), + ("B", "FAIL", None, "invalid session id: " + "b1" * 20), + ]) + assert "2x invalid session id: " in out, out + + +def test_trace_shows_screenshot(tmp_path): + xml = tmp_path / "output.xml" + xml.write_text(_ui_xml([("One", "FAIL", BROWSER_SHOT, BROWSER_MSG % 3)]), encoding="utf-8") + out = _run("trace", str(xml), "--test", "One").stdout + assert "screenshot: Direct_Calls_With_IAM_Token_FAILURE_SCREENSHOT_1.png" in out, out From 73da3625ba45094d52ed1814afc4f568adbe4ea6 Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Sun, 15 Nov 2026 10:51:35 +0300 Subject: [PATCH 06/15] fix: score only real kills, latest row per mutant Mutant Must Fail now counts a kill only on an assertion failure; a missing keyword, missing variable or wrong argument count is a survivor. kill_rate keeps the latest ledger row per test and operator. SKILL.md mutates the value the test checks, imports the resource through ${CLAUDE_SKILL_DIR} and runs scripts with uv run. Trigger set grows to 8 positive and 8 negative queries with the sibling and look-alike quotas; the narrow task targets tests/example.robot and adds robot_pass. Tests use 3-of-4 data and assert the exact rate. Verified: uv run --frozen python -m pytest -q plugins/rf-core/tests/test_test_quality_skill.py tests/eval/test_trigger_sets.py: red 6 failed, 29 passed (../red.log), green 35 passed (../green.log). selftest ok. Plugin and bundle copies identical (cmp). --- .../skills/rf-test-quality/SKILL.md | 17 +++++--- .../rf-test-quality/assets/mutants.resource | 7 ++-- .../scripts/mutation_ledger.py | 27 +++++++++--- .../narrow/narrow-test-quality-mutant-01.yaml | 14 ++++--- .../evals/triggers/rf-test-quality.yaml | 10 ++++- .../rf-core/skills/rf-test-quality/SKILL.md | 17 +++++--- .../rf-test-quality/assets/mutants.resource | 7 ++-- .../scripts/mutation_ledger.py | 27 +++++++++--- .../rf-core/tests/test_test_quality_skill.py | 42 +++++++++++++++---- 9 files changed, 125 insertions(+), 43 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md index 578eb3a..8f4cb56 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md @@ -20,25 +20,30 @@ Every new or changed test is seen failing once. Break the thing it checks, run i ## Mutant checks in a suite -Import `assets/mutants.resource` and wrap a check in `Mutant Must Fail` with a mutated input or expectation: +Import `assets/mutants.resource` and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: ```robotframework *** Settings *** -Resource mutants.resource +Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code - ${killed}= Mutant Must Fail T1 flip-status Should Be Equal As Integers 200 500 + ${resp}= GET On Session api /health + Should Be Equal As Integers ${resp.status_code} 200 + ${killed}= Mutant Must Fail T1 flip-status + ... Should Be Equal As Integers ${resp.status_code} 500 Should Be True ${killed} ``` -Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. +Only an assertion failure counts as a kill. A mutant that fails because of a typo, a missing keyword, a missing variable or a wrong argument count is recorded as a survivor: fix the mutant call. + +Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. A rerun of the same test and operator replaces its earlier row, so the rate uses the latest result per mutant. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. ## Reading the ledger ```bash -python scripts/mutation_ledger.py rate results/mutation-ledger.jsonl -python scripts/mutation_ledger.py selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate results/mutation-ledger.jsonl +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest ``` An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource b/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource index 3ea4ec6..d025da0 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource +++ b/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource @@ -7,12 +7,13 @@ ${MUTATION_LOG} ${OUTPUT_DIR}${/}mutation-ledger.jsonl *** Keywords *** Mutant Must Fail - [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail. + [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail + ... on its assertion. A typo or a missing keyword is not a kill. ... Writes one ledger row and returns True when the mutant was killed. ... A survivor is a finding about the check, never a pass. [Arguments] ${test_id} ${operator} ${keyword} @{args} - ${passed}= Run Keyword And Return Status ${keyword} @{args} - ${killed}= Evaluate not $passed + ${status} ${message}= Run Keyword And Ignore Error ${keyword} @{args} + ${killed}= Is Kill ${{$status == 'PASS'}} ${message} Record Mutation ${MUTATION_LOG} ${test_id} ${operator} ${killed} RETURN ${killed} diff --git a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py index 75a40e4..c35eb5b 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py +++ b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py @@ -4,16 +4,28 @@ A check is worth keeping only when it fails once its code or data is broken. Each mutant run appends one JSON line; the kill rate is read back from the file. - python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} - python scripts/mutation_ledger.py selftest + uv run python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} + uv run python scripts/mutation_ledger.py selftest Also importable as a Robot Framework library (see assets/mutants.resource). """ import json import os +import re import sys import tempfile +# Failures that mean the mutant never ran its check (typo, bad call), not a kill. +_NOT_A_KILL = re.compile( + r"No keyword with name|Variable '.*' not found|expected \d+(?: to \d+)? arguments?, got" + r"|Importing .* failed|Resource file .* does not exist" +) + + +def is_kill(passed, message): + """True only when the mutated check failed on its assertion.""" + return not passed and not _NOT_A_KILL.search(str(message)) + def record_mutation(path, test_id, operator, killed): """Append one row. ``killed`` is True when the mutated run failed.""" @@ -29,6 +41,8 @@ def kill_rate(path): if os.path.exists(path): with open(path, encoding="utf-8") as fh: rows = [json.loads(line) for line in fh if line.strip()] + # The ledger is append-only: a rerun of a mutant replaces its earlier row. + rows = list({(r["test"], r["operator"]): r for r in rows}.values()) killed = sum(1 for r in rows if r["killed"]) survivors = [f'{r["test"]}:{r["operator"]}' for r in rows if not r["killed"]] total = len(rows) @@ -41,11 +55,12 @@ def _selftest(): with tempfile.TemporaryDirectory() as d: p = os.path.join(d, "kills.jsonl") assert kill_rate(p)["rate"] == 0.0 - record_mutation(p, "T1", "flip-status", True) - record_mutation(p, "T1", "drop-field", False) + for op, killed in (("a", False), ("a", True), ("b", True), ("c", True), ("d", False)): + record_mutation(p, "T1", op, killed) r = kill_rate(p) - assert r == {"killed": 1, "total": 2, "rate": 0.5, - "survivors": ["T1:drop-field"]}, r + assert r == {"killed": 3, "total": 4, "rate": 0.75, + "survivors": ["T1:d"]}, r + assert not is_kill(False, "No keyword with name 'X' found.") print(json.dumps({"selftest": "ok"})) diff --git a/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml b/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml index dab0517..4c5c09c 100644 --- a/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml +++ b/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml @@ -6,9 +6,10 @@ description: | Mutant Must Fail call in the suite and a mutation ledger row written by the run. prompt: | - tests/smoke.robot checks that a value equals 200. Prove the check fails - when the expected value is wrong: add a mutant check with a wrong value, - run the suite into results/, and report the kill rate. + tests/example.robot checks that Build Greeting returns "Hello, World". + Prove the check fails when the expected value is wrong: add a mutant + check with a wrong value, run the suite into results/, and report the + kill rate. model: claude-haiku-5-5 max_turns: 14 timeout_seconds: 300 @@ -23,8 +24,11 @@ allowed_tools: grader_checks: - type: file_contains name: suite_uses_mutant_check - path: tests/smoke.robot - regex: 'Mutant Must Fail' + path: tests/example.robot + regex: 'Mutant Must Fail\s+\S+\s+\S+\s+Should Be Equal\s+\$\{greeting\}' + - type: robot_pass + name: suite_still_passes + path: tests/example.robot - type: file_contains name: ledger_row_written path: results/mutation-ledger.jsonl diff --git a/plugins/rf-core/evals/triggers/rf-test-quality.yaml b/plugins/rf-core/evals/triggers/rf-test-quality.yaml index 5d113db..1a5d7d6 100644 --- a/plugins/rf-core/evals/triggers/rf-test-quality.yaml +++ b/plugins/rf-core/evals/triggers/rf-test-quality.yaml @@ -9,12 +9,20 @@ queries: # should trigger (train) - {id: tq-t01, split: train, should_trigger: true, query: "I added a Robot Framework test, how do I prove it fails when the code is broken?"} - {id: tq-t02, split: train, should_trigger: true, query: "Add mutant checks to my Robot suite and report the kill rate"} - - {id: tq-t03, split: train, should_trigger: true, note: "user phrasing", query: "my robot test always passes even when the endpoint is wrong, is the check too weak?"} + - {id: tq-t03, split: train, should_trigger: true, note: "indirect", query: "my robot test always passes even when the endpoint is wrong, is the check too weak?"} + - {id: tq-t04, split: train, should_trigger: true, note: "edit existing file", query: "Edit tests/login.robot so each new check is wrapped in Mutant Must Fail with a wrong expected value"} + - {id: tq-t05, split: train, should_trigger: true, note: "indirect", query: "Before I merge, I want proof my new Robot keywords actually catch regressions"} # should trigger (validation) - {id: tq-v01, split: validation, should_trigger: true, query: "Show that the new status code check in tests/api.robot goes red on a wrong value"} - {id: tq-v02, split: validation, should_trigger: true, query: "Which of my changed Robot tests survive a mutated expectation?"} + - {id: tq-v03, split: validation, should_trigger: true, query: "Read mutation-ledger.jsonl from my Robot run and tell me the kill rate and survivors"} # should NOT trigger (train) - {id: tq-n01, split: train, should_trigger: false, note: "sibling rf-setup", query: "Install robotframework-browser into our uv project"} - {id: tq-n02, split: train, should_trigger: false, note: "sibling rf-language", query: "Add a suite setup in an __init__.robot file"} + - {id: tq-n03, split: train, should_trigger: false, note: "sibling rf-requests", query: "Write a Robot test that POSTs a user with RequestsLibrary and checks the 201"} + - {id: tq-n04, split: train, should_trigger: false, note: "non-RF look-alike", query: "Write a pytest test that checks my Flask health endpoint returns 200"} + - {id: tq-n05, split: train, should_trigger: false, note: "sibling rf-browser", query: "Click the login button and wait for the dashboard in a Browser library test"} # should NOT trigger (validation) - {id: tq-nv01, split: validation, should_trigger: false, note: "non-RF look-alike", query: "Run mutmut on my Python package"} + - {id: tq-nv02, split: validation, should_trigger: false, note: "sibling rf-language", query: "Rename a keyword across my Robot suites and fix the callers"} + - {id: tq-nv03, split: validation, should_trigger: false, note: "non-RF look-alike", query: "Set up Stryker mutation testing for my TypeScript project"} diff --git a/plugins/rf-core/skills/rf-test-quality/SKILL.md b/plugins/rf-core/skills/rf-test-quality/SKILL.md index 578eb3a..8f4cb56 100644 --- a/plugins/rf-core/skills/rf-test-quality/SKILL.md +++ b/plugins/rf-core/skills/rf-test-quality/SKILL.md @@ -20,25 +20,30 @@ Every new or changed test is seen failing once. Break the thing it checks, run i ## Mutant checks in a suite -Import `assets/mutants.resource` and wrap a check in `Mutant Must Fail` with a mutated input or expectation: +Import `assets/mutants.resource` and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: ```robotframework *** Settings *** -Resource mutants.resource +Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code - ${killed}= Mutant Must Fail T1 flip-status Should Be Equal As Integers 200 500 + ${resp}= GET On Session api /health + Should Be Equal As Integers ${resp.status_code} 200 + ${killed}= Mutant Must Fail T1 flip-status + ... Should Be Equal As Integers ${resp.status_code} 500 Should Be True ${killed} ``` -Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. +Only an assertion failure counts as a kill. A mutant that fails because of a typo, a missing keyword, a missing variable or a wrong argument count is recorded as a survivor: fix the mutant call. + +Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. A rerun of the same test and operator replaces its earlier row, so the rate uses the latest result per mutant. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. ## Reading the ledger ```bash -python scripts/mutation_ledger.py rate results/mutation-ledger.jsonl -python scripts/mutation_ledger.py selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate results/mutation-ledger.jsonl +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest ``` An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource b/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource index 3ea4ec6..d025da0 100644 --- a/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource +++ b/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource @@ -7,12 +7,13 @@ ${MUTATION_LOG} ${OUTPUT_DIR}${/}mutation-ledger.jsonl *** Keywords *** Mutant Must Fail - [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail. + [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail + ... on its assertion. A typo or a missing keyword is not a kill. ... Writes one ledger row and returns True when the mutant was killed. ... A survivor is a finding about the check, never a pass. [Arguments] ${test_id} ${operator} ${keyword} @{args} - ${passed}= Run Keyword And Return Status ${keyword} @{args} - ${killed}= Evaluate not $passed + ${status} ${message}= Run Keyword And Ignore Error ${keyword} @{args} + ${killed}= Is Kill ${{$status == 'PASS'}} ${message} Record Mutation ${MUTATION_LOG} ${test_id} ${operator} ${killed} RETURN ${killed} diff --git a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py index 75a40e4..c35eb5b 100644 --- a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py +++ b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py @@ -4,16 +4,28 @@ A check is worth keeping only when it fails once its code or data is broken. Each mutant run appends one JSON line; the kill rate is read back from the file. - python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} - python scripts/mutation_ledger.py selftest + uv run python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} + uv run python scripts/mutation_ledger.py selftest Also importable as a Robot Framework library (see assets/mutants.resource). """ import json import os +import re import sys import tempfile +# Failures that mean the mutant never ran its check (typo, bad call), not a kill. +_NOT_A_KILL = re.compile( + r"No keyword with name|Variable '.*' not found|expected \d+(?: to \d+)? arguments?, got" + r"|Importing .* failed|Resource file .* does not exist" +) + + +def is_kill(passed, message): + """True only when the mutated check failed on its assertion.""" + return not passed and not _NOT_A_KILL.search(str(message)) + def record_mutation(path, test_id, operator, killed): """Append one row. ``killed`` is True when the mutated run failed.""" @@ -29,6 +41,8 @@ def kill_rate(path): if os.path.exists(path): with open(path, encoding="utf-8") as fh: rows = [json.loads(line) for line in fh if line.strip()] + # The ledger is append-only: a rerun of a mutant replaces its earlier row. + rows = list({(r["test"], r["operator"]): r for r in rows}.values()) killed = sum(1 for r in rows if r["killed"]) survivors = [f'{r["test"]}:{r["operator"]}' for r in rows if not r["killed"]] total = len(rows) @@ -41,11 +55,12 @@ def _selftest(): with tempfile.TemporaryDirectory() as d: p = os.path.join(d, "kills.jsonl") assert kill_rate(p)["rate"] == 0.0 - record_mutation(p, "T1", "flip-status", True) - record_mutation(p, "T1", "drop-field", False) + for op, killed in (("a", False), ("a", True), ("b", True), ("c", True), ("d", False)): + record_mutation(p, "T1", op, killed) r = kill_rate(p) - assert r == {"killed": 1, "total": 2, "rate": 0.5, - "survivors": ["T1:drop-field"]}, r + assert r == {"killed": 3, "total": 4, "rate": 0.75, + "survivors": ["T1:d"]}, r + assert not is_kill(False, "No keyword with name 'X' found.") print(json.dumps({"selftest": "ok"})) diff --git a/plugins/rf-core/tests/test_test_quality_skill.py b/plugins/rf-core/tests/test_test_quality_skill.py index cd947a1..810496c 100644 --- a/plugins/rf-core/tests/test_test_quality_skill.py +++ b/plugins/rf-core/tests/test_test_quality_skill.py @@ -1,5 +1,6 @@ """rf-test-quality: the kill ledger counts survivors and never reports an empty run as killed.""" import importlib.util +import re import subprocess import sys from pathlib import Path @@ -15,12 +16,30 @@ def _load(): return mod -def test_survivor_is_counted(tmp_path): +def test_rate_is_killed_over_total_with_asymmetric_data(tmp_path): m = _load() - p = tmp_path / "k.jsonl" - m.record_mutation(str(p), "T1", "flip", True) - m.record_mutation(str(p), "T1", "drop", False) - assert m.kill_rate(str(p)) == {"killed": 1, "total": 2, "rate": 0.5, "survivors": ["T1:drop"]} + p = str(tmp_path / "k.jsonl") + for op, killed in (("a", True), ("b", True), ("c", True), ("d", False)): + m.record_mutation(p, "T1", op, killed) + assert m.kill_rate(p) == {"killed": 3, "total": 4, "rate": 0.75, "survivors": ["T1:d"]} + + +def test_latest_row_per_mutant_wins(tmp_path): + m = _load() + p = str(tmp_path / "k.jsonl") + m.record_mutation(p, "T1", "a", False) + m.record_mutation(p, "T1", "b", True) + m.record_mutation(p, "T1", "a", True) + assert m.kill_rate(p) == {"killed": 2, "total": 2, "rate": 1.0, "survivors": []} + + +def test_only_assertion_failures_kill(): + m = _load() + assert m.is_kill(False, "500 != 200") is True + assert m.is_kill(True, "") is False + assert m.is_kill(False, "No keyword with name 'Shuold Be Equal' found.") is False + assert m.is_kill(False, "Variable '${nope}' not found.") is False + assert m.is_kill(False, "Keyword 'BuiltIn.Should Be Equal' expected 2 to 8 arguments, got 1.") is False def test_empty_ledger_is_not_a_full_kill(tmp_path): @@ -32,6 +51,15 @@ def test_selftest_passes(): assert r.returncode == 0 and '"ok"' in r.stdout -def test_resource_defines_keywords(): +def test_resource_classifies_with_is_kill(): text = (SKILL / "assets" / "mutants.resource").read_text(encoding="utf-8") - assert "Mutant Must Fail" in text and "Mutation Kill Rate" in text + body = text.split("Mutant Must Fail", 1)[1] + assert "Run Keyword And Ignore Error" in body + assert re.search(r"Is Kill\s+\$\{\{\$status == 'PASS'\}\}\s+\$\{message\}", body) + + +def test_skill_example_uses_skill_dir_and_a_real_value(): + text = (SKILL / "SKILL.md").read_text(encoding="utf-8") + assert "${CLAUDE_SKILL_DIR}/assets/mutants.resource" in text + assert not re.search(r"(?m)^python ", text) + assert "200 500" not in text From 7d21f502c45bfb85689eb517c52dcda6bd7c92a4 Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Sun, 15 Nov 2026 10:56:22 +0300 Subject: [PATCH 07/15] fix: address run-digest review findings SKILL.md calls the scripts through uv run and ${CLAUDE_SKILL_DIR}. run_digest.py drops the fingerprint, body capture and unwrap helpers, uses argparse, and moves the selftest to digest_selftest.py. The listener reads DIGEST_MAX_PER_GROUP inside a try. Trigger set meets the 8/8 and sibling quotas; the narrow task targets tests/example.robot and adds a robot_pass grader. Verified (logs in scratchpad mksq/digest-logs): red.txt 3 failed, 11 passed before the fix; green.txt 42 passed for test_run_digest_skill.py and tests/eval/test_trigger_sets.py; selftest ok; failures output on cs2369-r1 output.xml matches before.txt apart from the removed fp tag. --- .../skills/rf-run-digest/SKILL.md | 14 +- .../rf-run-digest/scripts/digest_listener.py | 16 +- .../rf-run-digest/scripts/digest_selftest.py | 73 +++++++ .../rf-run-digest/scripts/run_digest.py | 197 ++++-------------- .../tasks/narrow/narrow-run-digest-01.yaml | 2 +- .../rf-core/evals/triggers/rf-run-digest.yaml | 10 +- plugins/rf-core/skills/rf-run-digest/SKILL.md | 14 +- .../rf-run-digest/scripts/digest_listener.py | 16 +- .../rf-run-digest/scripts/digest_selftest.py | 73 +++++++ .../rf-run-digest/scripts/run_digest.py | 197 ++++-------------- .../rf-core/tests/test_run_digest_skill.py | 24 ++- 11 files changed, 293 insertions(+), 343 deletions(-) create mode 100644 bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_selftest.py create mode 100644 plugins/rf-core/skills/rf-run-digest/scripts/digest_selftest.py diff --git a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md index 33f0ffe..dc54bec 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md @@ -23,23 +23,23 @@ Read the digest first, not output.xml. Open output.xml only for one named test, Attach `scripts/digest_listener.py` as a listener. When the run ends, it writes `digest.txt` next to output.xml: ```bash -robot --listener scripts/digest_listener.py -d results/run tests/ +robot --listener "${CLAUDE_SKILL_DIR}/scripts/digest_listener.py" -d results/run tests/ cat results/run/digest.txt ``` The listener leaves output.xml as robot wrote it. To shrink it, pass the `shrink` -argument: `--listener scripts/digest_listener.py:shrink`. This rewrites output.xml in +argument: `--listener "${CLAUDE_SKILL_DIR}/scripts/digest_listener.py:shrink"`. This rewrites output.xml in place with `rebot --removekeywords PASSED`, so the keywords of passed tests are gone from it and from any log built from it later. ## Read an existing output.xml ```bash -python scripts/run_digest.py summary results/output.xml # totals and per-suite counts -python scripts/run_digest.py failures results/output.xml --max 3 # failures grouped by signature -python scripts/run_digest.py diff before.xml after.xml # what changed between two runs -python scripts/run_digest.py trace results/output.xml --test "Name" -python scripts/run_digest.py selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" summary results/output.xml # totals and per-suite counts +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" failures results/output.xml --max 3 # failures grouped by signature +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" diff before.xml after.xml # what changed between two runs +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" trace results/output.xml --test "Name" +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" selftest ``` Failures that differ only by ids, hosts or timestamps fall into one group, so a run with many failures reads as a few groups. Quote the summary line and the group messages when you report a run. diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py index de082c3..a80b741 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py +++ b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py @@ -6,7 +6,7 @@ Attach it to any `robot` or `pabot` command; it fires when the run tears down. - robot --listener tools/digest_listener.py -d results/run \ + robot --listener scripts/digest_listener.py -d results/run \ --output output.xml --log NONE --report NONE tests/ > /dev/null 2>&1 cat results/run/digest.txt @@ -15,13 +15,13 @@ # PowerShell $env:ROBOT_OPTIONS = '--listener "C:\\path\\to\\tools\\digest_listener.py"' # bash - export ROBOT_OPTIONS='--listener /abs/path/tools/digest_listener.py' + export ROBOT_OPTIONS='--listener /abs/path/scripts/digest_listener.py' Read digest.txt first. A person opens robot's own report.html (the listener prints its path when a run fails and never opens it). Neither one ever reads output.xml or log.html. Under pabot each process writes its own digest.txt; read the merged run's instead -(`uv run python tools/run_digest.py failures `). +(`uv run python scripts/run_digest.py failures `). Errors are swallowed: a reporting tool must never fail a run. """ @@ -33,7 +33,13 @@ ROBOT_LISTENER_API_VERSION = 3 -MAX_PER_GROUP = int(os.environ.get("DIGEST_MAX_PER_GROUP", "3")) + + +def _max_per_group(): + try: + return max(1, int(os.environ.get("DIGEST_MAX_PER_GROUP", "3"))) + except ValueError: + return 3 class digest_listener: # noqa: N801 (robot resolves the listener by file/class name) @@ -60,7 +66,7 @@ def _write_digest(self): return if self.shrink: run_digest.shrink(self.output_xml) # rewrites output.xml without PASSED keywords - text, failed = run_digest.digest(self.output_xml, MAX_PER_GROUP) + text, failed = run_digest.digest(self.output_xml, _max_per_group()) path = os.path.join(os.path.dirname(self.output_xml), "digest.txt") with open(path, "w", encoding="utf-8") as f: f.write(text) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_selftest.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_selftest.py new file mode 100644 index 0000000..d3ba122 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_selftest.py @@ -0,0 +1,73 @@ +"""Self-check for run_digest.py on a tiny generated output.xml.""" +import contextlib +import io +import os +import tempfile + +from run_digest import cmd_diff, cmd_summary, cmd_trace, digest, norm, parse + +SAMPLE = """ + + + + + POST Request : url=http://h/api/orders +path_url=/api/orders/{ID} + POST Response : url=http://h/api/orders +status=400, reason=Bad Request +body=b'{{"error":"Validation failed"}}' + + 400.0 != 200.0 + + + POST Request : url=http://h/api/orders +path_url=/api/orders/{ID2} + 400.0 != 200.0 + + + GET Request : url=http://h/api/orders +path_url=/api/orders + + + + + + + +""" + + +def selftest(): + d = tempfile.mkdtemp() + uid = "11111111-2222-3333-4444-555555555555" + uid2 = "99999999-2222-3333-4444-555555555555" + p = os.path.join(d, "out.xml") + open(p, "w", encoding="utf-8").write(SAMPLE.format(ID=uid, ID2=uid2)) + tests = parse(p) + assert len(tests) == 3, tests + assert [t["status"] for t in tests] == ["FAIL", "FAIL", "PASS"] + t = tests[0] + assert t["suite"] == "01 Orders" and t["req"] == "/api/orders/" + uid, t + assert t["res"] == "400", t + # two failures differing only by UUID must collapse into ONE group + sigs = set(norm(x["msg"]) for x in tests if x["status"] == "FAIL") + assert len(sigs) == 1, sigs + assert norm("Url: /api/x/%s Expected 504 != 200" % uid) == "Url: /api/x/{id} Expected 504 != 200" + assert norm("Url: https://h.dev/api/x/42?p=1 504") == "Url: /api/x/{id} 504" + q = os.path.join(d, "after.xml") + open(q, "w", encoding="utf-8").write( + SAMPLE.format(ID=uid, ID2=uid2).replace('status="FAIL">400.0 != 200.0', 'status="PASS">', 1)) + text, nfail = digest(p) + assert nfail == 2 and "2x 400.0 != 200.0" in text, text + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + cmd_summary(tests + [dict(tests[2], name="Skipped", status="SKIP")]) + assert buf.getvalue().startswith("4 tests, 1 passed, 2 failed, 1 skipped\n"), buf.getvalue() + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + cmd_summary(tests) + assert buf.getvalue().startswith("3 tests, 1 passed, 2 failed\n"), buf.getvalue() + cmd_diff(tests, parse(q)) + cmd_trace(tests, "Create Order Basic") + print(text) + print("selftest ok") diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py index cba28f1..8f42206 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py +++ b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 """Read a robot run without reading output.xml. Stdlib only. - uv run python tools/run_digest.py summary OUT.xml # totals + per-suite - uv run python tools/run_digest.py failures OUT.xml [--max 3] # grouped by signature - uv run python tools/run_digest.py diff BEFORE.xml AFTER.xml # status vs message changes - uv run python tools/run_digest.py trace OUT.xml --test NAME # keyword path to the failure - uv run python tools/run_digest.py selftest + run_digest.py summary OUT.xml # totals + per-suite + run_digest.py failures OUT.xml [--max 3] # grouped by signature + run_digest.py diff BEFORE.xml AFTER.xml # status vs message changes + run_digest.py trace OUT.xml --test NAME # keyword path to the failure + run_digest.py selftest -A 94-failure run reads as ~2 groups. Never `cat` output.xml, never open log.html. +Never `cat` output.xml, never open log.html. """ -import hashlib +import argparse +import io import os import re import subprocess @@ -20,7 +21,6 @@ URL = re.compile(r"https?://\S+|/api/\S*") TS = re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}\S*") PATH_URL = re.compile(r"path_url=(\S+)") -BODY = re.compile(r"\bbody=(.*?)(?:\n|$)", re.S) STATUS = re.compile(r"\bstatus=(\d+)") IMG = re.compile(r"]*?src=\"([^\"]+)\"", re.I) FILE_PATH = re.compile(r"(?:\b[A-Za-z]:[\\/]|(? on the BUGS.md entry and look it up - there before filing a new issue.""" - return "fp-" + hashlib.sha256(sig.encode("utf-8")).hexdigest()[:12] - - def clip(s, n=160): s = " ".join((s or "").split()) return s[:n] + (" ..." if len(s) > n else "") -def unwrap(body): - """Requests logs payloads as a bytes repr; GETs and empty responses log None. - - Strip the b'...' wrapper and drop None on both sides, or every GET (and every 401 with an - empty body) grows a junk line. - """ - body = (body or "").strip() - if body in ("None", ""): - return None - if body[:2] in ("b'", 'b"'): - body = body[2:-1] - return body - - -def _grab(rx, text): - m = rx.search(text) - return m.group(1) if m else None - - def parse(path): - """-> one dict per test, in run order. Request context is kept for every test. + """-> one dict per test, in run order, with the last request path and response status. - The payload is reset at each new request: scanning for "the last body= on any request" - made a failing GET inherit an earlier POST's payload in the same test, and a sent: line - from a different endpoint than the last request: line above it is worse than no line. - The label is "last request", not "the failing request": in a multi-request test the - assertion can be about an earlier call; `trace` gives the failing keyword path. + "last request" is not always the failing one; `trace` gives the failing keyword path. """ tests, suites, cur = [], [], None for event, el in ET.iterparse(path, events=("start", "end")): @@ -94,8 +63,8 @@ def parse(path): suites.append(el.get("name")) elif tag == "test": cur = {"name": el.get("name"), "suite": suites[-1], - "status": "", "msg": "", "sent": None, "req": None, "res": None, - "body": None, "shot": None, "kws": []} + "status": "", "msg": "", "req": None, "res": None, + "shot": None, "kws": []} continue if cur is not None and tag == "msg": text = el.text or "" @@ -103,11 +72,10 @@ def parse(path): if shot: # UI libraries log screenshots as cur["shot"] = shot.group(1) m = PATH_URL.search(text) - if m: # a new request: reset the payload - cur["req"], cur["sent"] = m.group(1), unwrap(_grab(BODY, text)) + if m: + cur["req"] = m.group(1) elif STATUS.search(text) and "body=" in text: cur["res"] = STATUS.search(text).group(1) - cur["body"] = unwrap(_grab(BODY, text)) elif cur is not None and tag == "kw": st = el.find("status") cur["kws"].append((el.get("name") or "", st.get("status", "") if st is not None else "")) @@ -127,7 +95,7 @@ def parse(path): def cmd_summary(tests): fail = [t for t in tests if t["status"] == "FAIL"] skip = sum(t["status"] == "SKIP" for t in tests) - # SKIP is neither: counting it as passed hid a chained test that never ran. + # SKIP counts as neither passed nor failed. line = "%d tests, %d passed, %d failed" % (len(tests), len(tests) - len(fail) - skip, len(fail)) print(line + (", %d skipped" % skip if skip else "")) per = {} @@ -147,14 +115,13 @@ def cmd_failures(tests, max_per_group=3): if not groups: return print("no failures") for sig, ts in sorted(groups.items(), key=lambda kv: -len(kv[1])): - print("\n %dx %s [%s]" % (len(ts), sig, fingerprint(sig))) + print("\n %dx %s" % (len(ts), sig)) for t in ts[:max_per_group]: print(" %s :: %s" % (t["suite"], t["name"])) if t["req"]: print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) if t["shot"]: print(" screenshot: %s" % t["shot"]) - # Bodies are never printed: they carry tokens and passwords. if len(ts) > max_per_group: print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) @@ -191,18 +158,15 @@ def cmd_trace(tests, name): def shrink(output_xml): - """rebot --removekeywords PASSED, out of process. 25.7 MB -> 2.7 MB, same failures. + """Rewrite output.xml without PASSED keywords, via rebot in a child process. - Out of process on purpose: robot's LOGGER is a global singleton, so calling rebot() - in-process from a listener re-fires that same listener's close() forever. - `--removekeywords` on robot's own command line does not shrink output.xml: it only - filters the HTML (649,732 vs 649,806 bytes measured). Only a rebot pass removes anything. + In process, rebot() would re-fire the calling listener's close(): robot's LOGGER is global. """ tmp = output_xml + ".small" r = subprocess.run([sys.executable, "-m", "robot.rebot", "--removekeywords", "PASSED", "--log", "NONE", "--report", "NONE", "--output", tmp, output_xml], capture_output=True) - # rebot's exit code is the FAILED TEST COUNT (0-249), not a success flag. + # rebot exits with the failed test count (0-249). if r.returncode < 250 and os.path.exists(tmp): os.replace(tmp, output_xml) return True @@ -210,8 +174,7 @@ def shrink(output_xml): def digest(output_xml, max_per_group=3): - """summary + failures as one string. Used by digest_listener.py.""" - import io + """summary + failures as one string, for digest_listener.py.""" tests = parse(output_xml) buf, real = io.StringIO(), sys.stdout try: @@ -224,106 +187,24 @@ def digest(output_xml, max_per_group=3): def main(argv): - cmd = argv[0] - if cmd == "diff": - return cmd_diff(parse(argv[1]), parse(argv[2])) - tests = parse(argv[1]) - if cmd == "summary": - cmd_summary(tests) - elif cmd == "failures": - n = int(argv[argv.index("--max") + 1]) if "--max" in argv else 3 - cmd_summary(tests) - cmd_failures(tests, n) - elif cmd == "trace": - cmd_trace(tests, argv[argv.index("--test") + 1]) - else: - sys.exit(__doc__) - - -SAMPLE = """ - - - - - POST Request : url=http://h/api/orders -path_url=/api/orders/{ID} -body=b'{{"amount": 10}}' - POST Response : url=http://h/api/orders -status=400, reason=Bad Request -body=b'{{"error":"Validation failed"}}' - - 400.0 != 200.0 - - - POST Request : url=http://h/api/orders -path_url=/api/orders/{ID2} -body=b'{{"amount": 11}}' - 400.0 != 200.0 - - - GET Request : url=http://h/api/orders -path_url=/api/orders -body=None - - - - - - - -""" - - -def selftest(): - import tempfile - d = tempfile.mkdtemp() - uid = "11111111-2222-3333-4444-555555555555" - uid2 = "99999999-2222-3333-4444-555555555555" - p = os.path.join(d, "out.xml") - open(p, "w", encoding="utf-8").write(SAMPLE.format(ID=uid, ID2=uid2)) - tests = parse(p) - assert len(tests) == 3, tests - assert [t["status"] for t in tests] == ["FAIL", "FAIL", "PASS"] - t = tests[0] - assert t["suite"] == "01 Orders" and t["req"] == "/api/orders/" + uid, t - assert t["sent"] == '{"amount": 10}' and t["res"] == "400", t - assert "Validation failed" in t["body"] - assert tests[2]["sent"] is None, "a GET body=None must not become a junk sent: line" - # two failures differing only by UUID must collapse into ONE group - sigs = set(norm(x["msg"]) for x in tests if x["status"] == "FAIL") - assert len(sigs) == 1, sigs - assert norm("Url: /api/x/%s Expected 504 != 200" % uid) == "Url: /api/x/{id} Expected 504 != 200" - assert norm("Url: https://h.dev/api/x/42?p=1 504") == "Url: /api/x/{id} 504" - # fingerprint: stable across ids/hosts/timestamps, distinct across status and endpoint - fp = lambda m: fingerprint(norm(m)) - assert fp("Url: /api/o/%s 504 != 200" % uid) == fp("Url: https://x/api/o/%s 504 != 200" % uid2) - assert fp("Url: /api/o/1 400 != 200") != fp("Url: /api/o/1 409 != 200") - assert fp("Url: /api/orders/1 504 != 200") != fp("Url: /api/stock/1 504 != 200") - assert unwrap("None") is None and unwrap("b'{}'") == "{}" - q = os.path.join(d, "after.xml") - open(q, "w", encoding="utf-8").write( - SAMPLE.format(ID=uid, ID2=uid2).replace('status="FAIL">400.0 != 200.0', 'status="PASS">', 1)) - text, nfail = digest(p) - assert nfail == 2 and "2x 400.0 != 200.0" in text, text - import contextlib, io - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - cmd_summary(tests + [dict(tests[2], name="Skipped", status="SKIP")]) - assert buf.getvalue().startswith("4 tests, 1 passed, 2 failed, 1 skipped\n"), buf.getvalue() - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - cmd_summary(tests) - assert buf.getvalue().startswith("3 tests, 1 passed, 2 failed\n"), buf.getvalue() - cmd_diff(tests, parse(q)) - cmd_trace(tests, "Create Order Basic") - print(text) - print("selftest ok") + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("cmd", choices=["summary", "failures", "diff", "trace", "selftest"]) + ap.add_argument("xml", nargs="*") + ap.add_argument("--max", type=int, default=3) + ap.add_argument("--test") + a = ap.parse_args(argv) + if a.cmd == "selftest": + from digest_selftest import selftest + return selftest() + if a.cmd == "diff": + return cmd_diff(parse(a.xml[0]), parse(a.xml[1])) + tests = parse(a.xml[0]) + if a.cmd == "trace": + return cmd_trace(tests, a.test) + cmd_summary(tests) + if a.cmd == "failures": + cmd_failures(tests, a.max) if __name__ == "__main__": - if len(sys.argv) == 2 and sys.argv[1] == "selftest": - selftest() - elif len(sys.argv) < 3: - sys.exit(__doc__) - else: - main(sys.argv[1:]) + main(sys.argv[1:]) diff --git a/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml b/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml index ace273c..d4a3e60 100644 --- a/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml +++ b/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml @@ -5,7 +5,7 @@ description: | Run a suite with the digest listener and report from the digest. Gated by a digest.txt written next to output.xml that holds the summary line. prompt: | - Run tests/smoke.robot into results/ so that a digest.txt is written next + Run tests/example.robot into results/ so that a digest.txt is written next to output.xml, then report the pass and fail counts from that digest. model: claude-haiku-5-5 max_turns: 14 diff --git a/plugins/rf-core/evals/triggers/rf-run-digest.yaml b/plugins/rf-core/evals/triggers/rf-run-digest.yaml index 67fd508..ec79f3c 100644 --- a/plugins/rf-core/evals/triggers/rf-run-digest.yaml +++ b/plugins/rf-core/evals/triggers/rf-run-digest.yaml @@ -9,12 +9,20 @@ queries: # should trigger (train) - {id: rd-t01, split: train, should_trigger: true, query: "My robot run just finished with 80 failures, what went wrong?"} - {id: rd-t02, split: train, should_trigger: true, query: "Summarize results/output.xml from the last Robot Framework run"} - - {id: rd-t03, split: train, should_trigger: true, note: "user phrasing", query: "log.html is huge, how do I see which robot tests failed and why?"} + - {id: rd-t03, split: train, should_trigger: true, note: "indirect", query: "log.html is huge, how do I see which robot tests failed and why?"} + - {id: rd-t04, split: train, should_trigger: true, note: "edit existing file", query: "Edit run_tests.sh so every robot call writes a digest.txt next to output.xml"} + - {id: rd-t05, split: train, should_trigger: true, note: "indirect", query: "The nightly robot job went red again, I just need the short story of what broke"} # should trigger (validation) - {id: rd-v01, split: validation, should_trigger: true, query: "Group the failures in this Robot output.xml by error message"} - {id: rd-v02, split: validation, should_trigger: true, query: "Compare two Robot Framework runs and tell me which tests changed status"} + - {id: rd-v03, split: validation, should_trigger: true, query: "Show the failing keyword path for one test in my Robot output.xml"} # should NOT trigger (train) - {id: rd-n01, split: train, should_trigger: false, note: "sibling rf-setup", query: "Install robotframework-requests into our uv project"} - {id: rd-n02, split: train, should_trigger: false, note: "sibling rf-language", query: "Add a suite teardown in an __init__.robot file"} + - {id: rd-n03, split: train, should_trigger: false, note: "sibling rf-requests", query: "Write a Robot test that GETs /health with RequestsLibrary and checks the 200"} + - {id: rd-n04, split: train, should_trigger: false, note: "non-RF look-alike", query: "Summarize the failures in this pytest run output"} + - {id: rd-n05, split: train, should_trigger: false, note: "sibling rf-browser", query: "Click the login button and wait for the dashboard in a Browser library test"} # should NOT trigger (validation) - {id: rd-nv01, split: validation, should_trigger: false, note: "non-RF look-alike", query: "Parse this JUnit XML report from pytest"} + - {id: rd-nv02, split: validation, should_trigger: false, note: "sibling rf-language", query: "Rename a keyword across my Robot suites and fix the callers"} + - {id: rd-nv03, split: validation, should_trigger: false, note: "non-RF look-alike", query: "Why did my GitHub Actions build fail on the npm install step?"} diff --git a/plugins/rf-core/skills/rf-run-digest/SKILL.md b/plugins/rf-core/skills/rf-run-digest/SKILL.md index 33f0ffe..dc54bec 100644 --- a/plugins/rf-core/skills/rf-run-digest/SKILL.md +++ b/plugins/rf-core/skills/rf-run-digest/SKILL.md @@ -23,23 +23,23 @@ Read the digest first, not output.xml. Open output.xml only for one named test, Attach `scripts/digest_listener.py` as a listener. When the run ends, it writes `digest.txt` next to output.xml: ```bash -robot --listener scripts/digest_listener.py -d results/run tests/ +robot --listener "${CLAUDE_SKILL_DIR}/scripts/digest_listener.py" -d results/run tests/ cat results/run/digest.txt ``` The listener leaves output.xml as robot wrote it. To shrink it, pass the `shrink` -argument: `--listener scripts/digest_listener.py:shrink`. This rewrites output.xml in +argument: `--listener "${CLAUDE_SKILL_DIR}/scripts/digest_listener.py:shrink"`. This rewrites output.xml in place with `rebot --removekeywords PASSED`, so the keywords of passed tests are gone from it and from any log built from it later. ## Read an existing output.xml ```bash -python scripts/run_digest.py summary results/output.xml # totals and per-suite counts -python scripts/run_digest.py failures results/output.xml --max 3 # failures grouped by signature -python scripts/run_digest.py diff before.xml after.xml # what changed between two runs -python scripts/run_digest.py trace results/output.xml --test "Name" -python scripts/run_digest.py selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" summary results/output.xml # totals and per-suite counts +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" failures results/output.xml --max 3 # failures grouped by signature +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" diff before.xml after.xml # what changed between two runs +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" trace results/output.xml --test "Name" +uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" selftest ``` Failures that differ only by ids, hosts or timestamps fall into one group, so a run with many failures reads as a few groups. Quote the summary line and the group messages when you report a run. diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py b/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py index de082c3..a80b741 100644 --- a/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py +++ b/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py @@ -6,7 +6,7 @@ Attach it to any `robot` or `pabot` command; it fires when the run tears down. - robot --listener tools/digest_listener.py -d results/run \ + robot --listener scripts/digest_listener.py -d results/run \ --output output.xml --log NONE --report NONE tests/ > /dev/null 2>&1 cat results/run/digest.txt @@ -15,13 +15,13 @@ # PowerShell $env:ROBOT_OPTIONS = '--listener "C:\\path\\to\\tools\\digest_listener.py"' # bash - export ROBOT_OPTIONS='--listener /abs/path/tools/digest_listener.py' + export ROBOT_OPTIONS='--listener /abs/path/scripts/digest_listener.py' Read digest.txt first. A person opens robot's own report.html (the listener prints its path when a run fails and never opens it). Neither one ever reads output.xml or log.html. Under pabot each process writes its own digest.txt; read the merged run's instead -(`uv run python tools/run_digest.py failures `). +(`uv run python scripts/run_digest.py failures `). Errors are swallowed: a reporting tool must never fail a run. """ @@ -33,7 +33,13 @@ ROBOT_LISTENER_API_VERSION = 3 -MAX_PER_GROUP = int(os.environ.get("DIGEST_MAX_PER_GROUP", "3")) + + +def _max_per_group(): + try: + return max(1, int(os.environ.get("DIGEST_MAX_PER_GROUP", "3"))) + except ValueError: + return 3 class digest_listener: # noqa: N801 (robot resolves the listener by file/class name) @@ -60,7 +66,7 @@ def _write_digest(self): return if self.shrink: run_digest.shrink(self.output_xml) # rewrites output.xml without PASSED keywords - text, failed = run_digest.digest(self.output_xml, MAX_PER_GROUP) + text, failed = run_digest.digest(self.output_xml, _max_per_group()) path = os.path.join(os.path.dirname(self.output_xml), "digest.txt") with open(path, "w", encoding="utf-8") as f: f.write(text) diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/digest_selftest.py b/plugins/rf-core/skills/rf-run-digest/scripts/digest_selftest.py new file mode 100644 index 0000000..d3ba122 --- /dev/null +++ b/plugins/rf-core/skills/rf-run-digest/scripts/digest_selftest.py @@ -0,0 +1,73 @@ +"""Self-check for run_digest.py on a tiny generated output.xml.""" +import contextlib +import io +import os +import tempfile + +from run_digest import cmd_diff, cmd_summary, cmd_trace, digest, norm, parse + +SAMPLE = """ + + + + + POST Request : url=http://h/api/orders +path_url=/api/orders/{ID} + POST Response : url=http://h/api/orders +status=400, reason=Bad Request +body=b'{{"error":"Validation failed"}}' + + 400.0 != 200.0 + + + POST Request : url=http://h/api/orders +path_url=/api/orders/{ID2} + 400.0 != 200.0 + + + GET Request : url=http://h/api/orders +path_url=/api/orders + + + + + + + +""" + + +def selftest(): + d = tempfile.mkdtemp() + uid = "11111111-2222-3333-4444-555555555555" + uid2 = "99999999-2222-3333-4444-555555555555" + p = os.path.join(d, "out.xml") + open(p, "w", encoding="utf-8").write(SAMPLE.format(ID=uid, ID2=uid2)) + tests = parse(p) + assert len(tests) == 3, tests + assert [t["status"] for t in tests] == ["FAIL", "FAIL", "PASS"] + t = tests[0] + assert t["suite"] == "01 Orders" and t["req"] == "/api/orders/" + uid, t + assert t["res"] == "400", t + # two failures differing only by UUID must collapse into ONE group + sigs = set(norm(x["msg"]) for x in tests if x["status"] == "FAIL") + assert len(sigs) == 1, sigs + assert norm("Url: /api/x/%s Expected 504 != 200" % uid) == "Url: /api/x/{id} Expected 504 != 200" + assert norm("Url: https://h.dev/api/x/42?p=1 504") == "Url: /api/x/{id} 504" + q = os.path.join(d, "after.xml") + open(q, "w", encoding="utf-8").write( + SAMPLE.format(ID=uid, ID2=uid2).replace('status="FAIL">400.0 != 200.0', 'status="PASS">', 1)) + text, nfail = digest(p) + assert nfail == 2 and "2x 400.0 != 200.0" in text, text + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + cmd_summary(tests + [dict(tests[2], name="Skipped", status="SKIP")]) + assert buf.getvalue().startswith("4 tests, 1 passed, 2 failed, 1 skipped\n"), buf.getvalue() + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + cmd_summary(tests) + assert buf.getvalue().startswith("3 tests, 1 passed, 2 failed\n"), buf.getvalue() + cmd_diff(tests, parse(q)) + cmd_trace(tests, "Create Order Basic") + print(text) + print("selftest ok") diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py index cba28f1..8f42206 100644 --- a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py +++ b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 """Read a robot run without reading output.xml. Stdlib only. - uv run python tools/run_digest.py summary OUT.xml # totals + per-suite - uv run python tools/run_digest.py failures OUT.xml [--max 3] # grouped by signature - uv run python tools/run_digest.py diff BEFORE.xml AFTER.xml # status vs message changes - uv run python tools/run_digest.py trace OUT.xml --test NAME # keyword path to the failure - uv run python tools/run_digest.py selftest + run_digest.py summary OUT.xml # totals + per-suite + run_digest.py failures OUT.xml [--max 3] # grouped by signature + run_digest.py diff BEFORE.xml AFTER.xml # status vs message changes + run_digest.py trace OUT.xml --test NAME # keyword path to the failure + run_digest.py selftest -A 94-failure run reads as ~2 groups. Never `cat` output.xml, never open log.html. +Never `cat` output.xml, never open log.html. """ -import hashlib +import argparse +import io import os import re import subprocess @@ -20,7 +21,6 @@ URL = re.compile(r"https?://\S+|/api/\S*") TS = re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}\S*") PATH_URL = re.compile(r"path_url=(\S+)") -BODY = re.compile(r"\bbody=(.*?)(?:\n|$)", re.S) STATUS = re.compile(r"\bstatus=(\d+)") IMG = re.compile(r"]*?src=\"([^\"]+)\"", re.I) FILE_PATH = re.compile(r"(?:\b[A-Za-z]:[\\/]|(? on the BUGS.md entry and look it up - there before filing a new issue.""" - return "fp-" + hashlib.sha256(sig.encode("utf-8")).hexdigest()[:12] - - def clip(s, n=160): s = " ".join((s or "").split()) return s[:n] + (" ..." if len(s) > n else "") -def unwrap(body): - """Requests logs payloads as a bytes repr; GETs and empty responses log None. - - Strip the b'...' wrapper and drop None on both sides, or every GET (and every 401 with an - empty body) grows a junk line. - """ - body = (body or "").strip() - if body in ("None", ""): - return None - if body[:2] in ("b'", 'b"'): - body = body[2:-1] - return body - - -def _grab(rx, text): - m = rx.search(text) - return m.group(1) if m else None - - def parse(path): - """-> one dict per test, in run order. Request context is kept for every test. + """-> one dict per test, in run order, with the last request path and response status. - The payload is reset at each new request: scanning for "the last body= on any request" - made a failing GET inherit an earlier POST's payload in the same test, and a sent: line - from a different endpoint than the last request: line above it is worse than no line. - The label is "last request", not "the failing request": in a multi-request test the - assertion can be about an earlier call; `trace` gives the failing keyword path. + "last request" is not always the failing one; `trace` gives the failing keyword path. """ tests, suites, cur = [], [], None for event, el in ET.iterparse(path, events=("start", "end")): @@ -94,8 +63,8 @@ def parse(path): suites.append(el.get("name")) elif tag == "test": cur = {"name": el.get("name"), "suite": suites[-1], - "status": "", "msg": "", "sent": None, "req": None, "res": None, - "body": None, "shot": None, "kws": []} + "status": "", "msg": "", "req": None, "res": None, + "shot": None, "kws": []} continue if cur is not None and tag == "msg": text = el.text or "" @@ -103,11 +72,10 @@ def parse(path): if shot: # UI libraries log screenshots as cur["shot"] = shot.group(1) m = PATH_URL.search(text) - if m: # a new request: reset the payload - cur["req"], cur["sent"] = m.group(1), unwrap(_grab(BODY, text)) + if m: + cur["req"] = m.group(1) elif STATUS.search(text) and "body=" in text: cur["res"] = STATUS.search(text).group(1) - cur["body"] = unwrap(_grab(BODY, text)) elif cur is not None and tag == "kw": st = el.find("status") cur["kws"].append((el.get("name") or "", st.get("status", "") if st is not None else "")) @@ -127,7 +95,7 @@ def parse(path): def cmd_summary(tests): fail = [t for t in tests if t["status"] == "FAIL"] skip = sum(t["status"] == "SKIP" for t in tests) - # SKIP is neither: counting it as passed hid a chained test that never ran. + # SKIP counts as neither passed nor failed. line = "%d tests, %d passed, %d failed" % (len(tests), len(tests) - len(fail) - skip, len(fail)) print(line + (", %d skipped" % skip if skip else "")) per = {} @@ -147,14 +115,13 @@ def cmd_failures(tests, max_per_group=3): if not groups: return print("no failures") for sig, ts in sorted(groups.items(), key=lambda kv: -len(kv[1])): - print("\n %dx %s [%s]" % (len(ts), sig, fingerprint(sig))) + print("\n %dx %s" % (len(ts), sig)) for t in ts[:max_per_group]: print(" %s :: %s" % (t["suite"], t["name"])) if t["req"]: print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) if t["shot"]: print(" screenshot: %s" % t["shot"]) - # Bodies are never printed: they carry tokens and passwords. if len(ts) > max_per_group: print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) @@ -191,18 +158,15 @@ def cmd_trace(tests, name): def shrink(output_xml): - """rebot --removekeywords PASSED, out of process. 25.7 MB -> 2.7 MB, same failures. + """Rewrite output.xml without PASSED keywords, via rebot in a child process. - Out of process on purpose: robot's LOGGER is a global singleton, so calling rebot() - in-process from a listener re-fires that same listener's close() forever. - `--removekeywords` on robot's own command line does not shrink output.xml: it only - filters the HTML (649,732 vs 649,806 bytes measured). Only a rebot pass removes anything. + In process, rebot() would re-fire the calling listener's close(): robot's LOGGER is global. """ tmp = output_xml + ".small" r = subprocess.run([sys.executable, "-m", "robot.rebot", "--removekeywords", "PASSED", "--log", "NONE", "--report", "NONE", "--output", tmp, output_xml], capture_output=True) - # rebot's exit code is the FAILED TEST COUNT (0-249), not a success flag. + # rebot exits with the failed test count (0-249). if r.returncode < 250 and os.path.exists(tmp): os.replace(tmp, output_xml) return True @@ -210,8 +174,7 @@ def shrink(output_xml): def digest(output_xml, max_per_group=3): - """summary + failures as one string. Used by digest_listener.py.""" - import io + """summary + failures as one string, for digest_listener.py.""" tests = parse(output_xml) buf, real = io.StringIO(), sys.stdout try: @@ -224,106 +187,24 @@ def digest(output_xml, max_per_group=3): def main(argv): - cmd = argv[0] - if cmd == "diff": - return cmd_diff(parse(argv[1]), parse(argv[2])) - tests = parse(argv[1]) - if cmd == "summary": - cmd_summary(tests) - elif cmd == "failures": - n = int(argv[argv.index("--max") + 1]) if "--max" in argv else 3 - cmd_summary(tests) - cmd_failures(tests, n) - elif cmd == "trace": - cmd_trace(tests, argv[argv.index("--test") + 1]) - else: - sys.exit(__doc__) - - -SAMPLE = """ - - - - - POST Request : url=http://h/api/orders -path_url=/api/orders/{ID} -body=b'{{"amount": 10}}' - POST Response : url=http://h/api/orders -status=400, reason=Bad Request -body=b'{{"error":"Validation failed"}}' - - 400.0 != 200.0 - - - POST Request : url=http://h/api/orders -path_url=/api/orders/{ID2} -body=b'{{"amount": 11}}' - 400.0 != 200.0 - - - GET Request : url=http://h/api/orders -path_url=/api/orders -body=None - - - - - - - -""" - - -def selftest(): - import tempfile - d = tempfile.mkdtemp() - uid = "11111111-2222-3333-4444-555555555555" - uid2 = "99999999-2222-3333-4444-555555555555" - p = os.path.join(d, "out.xml") - open(p, "w", encoding="utf-8").write(SAMPLE.format(ID=uid, ID2=uid2)) - tests = parse(p) - assert len(tests) == 3, tests - assert [t["status"] for t in tests] == ["FAIL", "FAIL", "PASS"] - t = tests[0] - assert t["suite"] == "01 Orders" and t["req"] == "/api/orders/" + uid, t - assert t["sent"] == '{"amount": 10}' and t["res"] == "400", t - assert "Validation failed" in t["body"] - assert tests[2]["sent"] is None, "a GET body=None must not become a junk sent: line" - # two failures differing only by UUID must collapse into ONE group - sigs = set(norm(x["msg"]) for x in tests if x["status"] == "FAIL") - assert len(sigs) == 1, sigs - assert norm("Url: /api/x/%s Expected 504 != 200" % uid) == "Url: /api/x/{id} Expected 504 != 200" - assert norm("Url: https://h.dev/api/x/42?p=1 504") == "Url: /api/x/{id} 504" - # fingerprint: stable across ids/hosts/timestamps, distinct across status and endpoint - fp = lambda m: fingerprint(norm(m)) - assert fp("Url: /api/o/%s 504 != 200" % uid) == fp("Url: https://x/api/o/%s 504 != 200" % uid2) - assert fp("Url: /api/o/1 400 != 200") != fp("Url: /api/o/1 409 != 200") - assert fp("Url: /api/orders/1 504 != 200") != fp("Url: /api/stock/1 504 != 200") - assert unwrap("None") is None and unwrap("b'{}'") == "{}" - q = os.path.join(d, "after.xml") - open(q, "w", encoding="utf-8").write( - SAMPLE.format(ID=uid, ID2=uid2).replace('status="FAIL">400.0 != 200.0', 'status="PASS">', 1)) - text, nfail = digest(p) - assert nfail == 2 and "2x 400.0 != 200.0" in text, text - import contextlib, io - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - cmd_summary(tests + [dict(tests[2], name="Skipped", status="SKIP")]) - assert buf.getvalue().startswith("4 tests, 1 passed, 2 failed, 1 skipped\n"), buf.getvalue() - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - cmd_summary(tests) - assert buf.getvalue().startswith("3 tests, 1 passed, 2 failed\n"), buf.getvalue() - cmd_diff(tests, parse(q)) - cmd_trace(tests, "Create Order Basic") - print(text) - print("selftest ok") + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("cmd", choices=["summary", "failures", "diff", "trace", "selftest"]) + ap.add_argument("xml", nargs="*") + ap.add_argument("--max", type=int, default=3) + ap.add_argument("--test") + a = ap.parse_args(argv) + if a.cmd == "selftest": + from digest_selftest import selftest + return selftest() + if a.cmd == "diff": + return cmd_diff(parse(a.xml[0]), parse(a.xml[1])) + tests = parse(a.xml[0]) + if a.cmd == "trace": + return cmd_trace(tests, a.test) + cmd_summary(tests) + if a.cmd == "failures": + cmd_failures(tests, a.max) if __name__ == "__main__": - if len(sys.argv) == 2 and sys.argv[1] == "selftest": - selftest() - elif len(sys.argv) < 3: - sys.exit(__doc__) - else: - main(sys.argv[1:]) + main(sys.argv[1:]) diff --git a/plugins/rf-core/tests/test_run_digest_skill.py b/plugins/rf-core/tests/test_run_digest_skill.py index b134365..d8226bb 100644 --- a/plugins/rf-core/tests/test_run_digest_skill.py +++ b/plugins/rf-core/tests/test_run_digest_skill.py @@ -136,7 +136,7 @@ def test_browser_failures_group_across_call_log_and_show_screenshot(tmp_path): ("One", "FAIL", BROWSER_SHOT, BROWSER_MSG % 12), ("Two", "FAIL", BROWSER_SHOT, BROWSER_MSG % 31), ]) - assert "2x TimeoutError: locator.waitFor: Timeout 60000ms exceeded. [" in out, out + assert "2x TimeoutError: locator.waitFor: Timeout 60000ms exceeded.\n" in out, out assert "screenshot: Direct_Calls_With_IAM_Token_FAILURE_SCREENSHOT_1.png" in out, out assert "pass-shot.png" not in out, out @@ -193,3 +193,25 @@ def test_trace_shows_screenshot(tmp_path): xml.write_text(_ui_xml([("One", "FAIL", BROWSER_SHOT, BROWSER_MSG % 3)]), encoding="utf-8") out = _run("trace", str(xml), "--test", "One").stdout assert "screenshot: Direct_Calls_With_IAM_Token_FAILURE_SCREENSHOT_1.png" in out, out + + +def test_listener_survives_bad_env_value(tmp_path, monkeypatch): + monkeypatch.setenv("DIGEST_MAX_PER_GROUP", "not-a-number") + mod = _load_listener() + xml = tmp_path / "output.xml" + xml.write_text(SECRET_XML, encoding="utf-8") + listener = mod.digest_listener() + listener.output_file(str(xml)) + listener.close() + assert (tmp_path / "digest.txt").exists() + + +def test_parse_keeps_no_request_or_response_bodies(tmp_path): + import importlib.util + spec = importlib.util.spec_from_file_location("run_digest", SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + xml = tmp_path / "output.xml" + xml.write_text(SECRET_XML, encoding="utf-8") + for test in mod.parse(str(xml)): + assert "sent" not in test and "body" not in test, test From caee41b689fd2d49bbf94caf759d21413db9e57c Mon Sep 17 00:00:00 2001 From: ai94iq <5349989+ai94iq@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:47:34 +0300 Subject: [PATCH 08/15] fix(rf-run-digest): meet skill contract and installer counts --- .../rf-agentskills/skills/rf-run-digest/SKILL.md | 8 +++++--- .../skills/rf-run-digest/scripts/run_digest.py | 4 ++++ plugins/rf-core/skills/rf-run-digest/SKILL.md | 16 ++++++++++------ .../skills/rf-run-digest/scripts/run_digest.py | 4 ++++ tests/installer/test_plugin_selection.py | 6 +++--- tests/installer/test_reinstall_prune.py | 2 +- tests/installer/test_split_upgrade.py | 4 ++-- tests/test_skill_commands.py | 3 ++- tests/test_skill_descriptions.py | 1 + 9 files changed, 32 insertions(+), 16 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md index dc54bec..7d9a2e1 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md @@ -2,7 +2,7 @@ name: rf-run-digest description: "Use after a Robot Framework run, to read its result from a short digest (summary, grouped failures) instead of output.xml or log.html." license: Apache-2.0 -compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled run_digest script. Stdlib only otherwise." metadata: author: manykarim version: "1.0.0" @@ -48,5 +48,7 @@ It reads Browser, SeleniumLibrary, AppiumLibrary, MaestroLibrary and RequestsLib ## Companion Skills -- `rf-setup`: install Robot Framework before the first run. -- `rf-language`: fix the suite once the digest names the failing test. +| Need | Skill | +|------|-------| +| Install Robot Framework or a library, fix the environment | `rf-setup` | +| Write tests, suites, user keywords, resources and variables in Robot Framework syntax | `rf-language` | diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py index 8f42206..0b08829 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py +++ b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["robotframework>=7"] +# /// """Read a robot run without reading output.xml. Stdlib only. run_digest.py summary OUT.xml # totals + per-suite diff --git a/plugins/rf-core/skills/rf-run-digest/SKILL.md b/plugins/rf-core/skills/rf-run-digest/SKILL.md index dc54bec..6061ff1 100644 --- a/plugins/rf-core/skills/rf-run-digest/SKILL.md +++ b/plugins/rf-core/skills/rf-run-digest/SKILL.md @@ -2,7 +2,7 @@ name: rf-run-digest description: "Use after a Robot Framework run, to read its result from a short digest (summary, grouped failures) instead of output.xml or log.html." license: Apache-2.0 -compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled run_digest script. Stdlib only otherwise." metadata: author: manykarim version: "1.0.0" @@ -20,15 +20,15 @@ Read the digest first, not output.xml. Open output.xml only for one named test, ## Write a digest on every run -Attach `scripts/digest_listener.py` as a listener. When the run ends, it writes `digest.txt` next to output.xml: +Attach `scripts/digest_listener.py` as a listener; `` is the folder containing this SKILL.md. When the run ends, it writes `digest.txt` next to output.xml: ```bash -robot --listener "${CLAUDE_SKILL_DIR}/scripts/digest_listener.py" -d results/run tests/ +robot --listener /scripts/digest_listener.py -d results/run tests/ cat results/run/digest.txt ``` The listener leaves output.xml as robot wrote it. To shrink it, pass the `shrink` -argument: `--listener "${CLAUDE_SKILL_DIR}/scripts/digest_listener.py:shrink"`. This rewrites output.xml in +argument: `--listener /scripts/digest_listener.py:shrink`. This rewrites output.xml in place with `rebot --removekeywords PASSED`, so the keywords of passed tests are gone from it and from any log built from it later. @@ -42,11 +42,15 @@ uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" trace results/outpu uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" selftest ``` +The script path is relative to this skill's directory; output.xml paths are relative to the project root. Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. + Failures that differ only by ids, hosts or timestamps fall into one group, so a run with many failures reads as a few groups. Quote the summary line and the group messages when you report a run. It reads Browser, SeleniumLibrary, AppiumLibrary, MaestroLibrary and RequestsLibrary failures; the `screenshot:` line on a failed test comes from the last `` that test logged. ## Companion Skills -- `rf-setup`: install Robot Framework before the first run. -- `rf-language`: fix the suite once the digest names the failing test. +| Need | Skill | +|------|-------| +| Install Robot Framework or a library, fix the environment | `rf-setup` | +| Write tests, suites, user keywords, resources and variables in Robot Framework syntax | `rf-language` | diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py index 8f42206..0b08829 100644 --- a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py +++ b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["robotframework>=7"] +# /// """Read a robot run without reading output.xml. Stdlib only. run_digest.py summary OUT.xml # totals + per-suite diff --git a/tests/installer/test_plugin_selection.py b/tests/installer/test_plugin_selection.py index 5a76e67..3b42e8d 100644 --- a/tests/installer/test_plugin_selection.py +++ b/tests/installer/test_plugin_selection.py @@ -30,7 +30,7 @@ def test_plugins_flag_selects_plugins(two_plugin_bundle: Path, fake_home: Path, prefix = tmp_path / "root" assert main(["install", "--agent", "claude-code", "--prefix", str(prefix), "--yes", "--plugins", "rf-core,rf-web,rf-example"]) == 0 - assert _skills(prefix) == {"rf-language", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium", + assert _skills(prefix) == {"rf-language", "rf-setup", "rf-python-library", "rf-run-digest", "rf-browser", "rf-selenium", "rf-example"} @@ -42,7 +42,7 @@ def test_default_skips_the_bundle_and_installs_every_content_plugin( (path,) = list(tmp_path.rglob("installed.json")) files = [f for ins in json.loads(path.read_text())["installations"] for f in ins["files"]] assert {f["plugin"] for f in files} == {"rf-core", "rf-web", "rf-api", "rf-mobile", "rf-desktop"} - assert len(_skills(prefix)) == 9 + assert len(_skills(prefix)) == 10 def test_bundle_in_file_mode_installs_its_plugins(two_plugin_bundle: Path, fake_home: Path, tmp_path: Path) -> None: @@ -50,7 +50,7 @@ def test_bundle_in_file_mode_installs_its_plugins(two_plugin_bundle: Path, fake_ prefix = tmp_path / "root" assert main(["install", "--agent", "claude-code", "--prefix", str(prefix), "--yes", "--plugins", "rf-agentskills"]) == 0 - assert len(_skills(prefix)) == 9 + assert len(_skills(prefix)) == 10 assert (prefix / "rf-core-files" / "scripts").is_dir() assert not (prefix / "rf-agentskills-files").exists() diff --git a/tests/installer/test_reinstall_prune.py b/tests/installer/test_reinstall_prune.py index 07ac871..72f1c38 100644 --- a/tests/installer/test_reinstall_prune.py +++ b/tests/installer/test_reinstall_prune.py @@ -213,7 +213,7 @@ def test_upgrade_removes_managed_robotcode_overlap_skills_and_agent(fake_home: P assert not path.exists(), path for name in ("rf-libdoc", "rf-results", "rf-robotcode"): assert not (skills / name).exists(), name - assert len([d for d in skills.iterdir() if d.name.startswith("rf-")]) == 9 + assert len([d for d in skills.iterdir() if d.name.startswith("rf-")]) == 10 agents = sorted(p.stem for p in (fake_home / ".claude" / "agents").glob("rf-*.md")) assert agents == ["rf-debug-expert", "rf-migration-guide", "rf-test-architect"] assert "5 removed" in capsys.readouterr().out diff --git a/tests/installer/test_split_upgrade.py b/tests/installer/test_split_upgrade.py index 8a17f89..682c37f 100644 --- a/tests/installer/test_split_upgrade.py +++ b/tests/installer/test_split_upgrade.py @@ -67,7 +67,7 @@ def test_file_mode_upgrade(agent: str, old_bundle: Path, fake_home: Path, tmp_pa project.mkdir() _old_install(project, agent, old_bundle, monkeypatch) skills_before = _skills(project) - assert len(skills_before) == 9 + assert len(skills_before) == 10 assert _install(project, agent) == 0 root = project / ROOTS[agent] @@ -104,7 +104,7 @@ def test_narrowing_is_explicit(old_bundle: Path, fake_home: Path, tmp_path: Path assert _install(project, "claude-code") == 0 capsys.readouterr() assert _install(project, "claude-code", "--plugins", "rf-core,rf-web") == 0 - assert _skills(project) == {"rf-language", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium"} + assert _skills(project) == {"rf-language", "rf-setup", "rf-python-library", "rf-run-digest", "rf-browser", "rf-selenium"} assert set(_record(project)) == {"rf-core", "rf-web"} out = " ".join(capsys.readouterr().out.split()) assert "skills rf-appium, rf-platynui, rf-requests, rf-restinstance" in out diff --git a/tests/test_skill_commands.py b/tests/test_skill_commands.py index cab77af..91e19f5 100644 --- a/tests/test_skill_commands.py +++ b/tests/test_skill_commands.py @@ -21,7 +21,8 @@ "root": standalone_skills(), "plugin": PLUGIN / "skills", } -SCRIPT_SKILLS = {"rf-language": "rf_conventions.py", "rf-python-library": "check_library.py"} +SCRIPT_SKILLS = {"rf-language": "rf_conventions.py", "rf-python-library": "check_library.py", + "rf-run-digest": "run_digest.py"} #: Skills whose content is Robot Framework code, so `${var}` is expected; only #: agent-specific variables such as ${CLAUDE_SKILL_DIR} are forbidden there. RF_CODE_SKILLS = {"rf-language", "rf-python-library"} diff --git a/tests/test_skill_descriptions.py b/tests/test_skill_descriptions.py index 28730df..792b0c2 100644 --- a/tests/test_skill_descriptions.py +++ b/tests/test_skill_descriptions.py @@ -192,6 +192,7 @@ "rf-setup": ("browser", "selenium", "appium", "requests", "restinstance", "platynui", "language"), "rf-language": ("browser", "requests", "setup", "python-library"), "rf-python-library": ("language", "setup"), + "rf-run-digest": ("setup", "language"), } THREE_COLUMN_HEADERS: dict[str, tuple[str, ...]] = {} From c7451e82ff4f352c305ac6ad0f13f7569b28ab4e Mon Sep 17 00:00:00 2001 From: ai94iq <5349989+ai94iq@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:50:09 +0300 Subject: [PATCH 09/15] fix(rf-test-quality): meet skill contract and installer counts --- .../skills/rf-test-quality/SKILL.md | 20 ++++++++++++---- .../scripts/mutation_ledger.py | 4 ++++ .../rf-core/skills/rf-test-quality/SKILL.md | 24 +++++++++++++------ .../scripts/mutation_ledger.py | 4 ++++ tests/installer/test_plugin_selection.py | 6 ++--- tests/installer/test_reinstall_prune.py | 2 +- tests/installer/test_split_upgrade.py | 4 ++-- tests/test_skill_commands.py | 5 ++-- tests/test_skill_descriptions.py | 1 + 9 files changed, 50 insertions(+), 20 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md index 8f4cb56..5fcc33a 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md @@ -2,7 +2,7 @@ name: rf-test-quality description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." license: Apache-2.0 -compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled mutation_ledger script (stdlib only otherwise)." metadata: author: manykarim version: "1.0.0" @@ -28,10 +28,10 @@ Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code - ${resp}= GET On Session api /health - Should Be Equal As Integers ${resp.status_code} 200 - ${killed}= Mutant Must Fail T1 flip-status - ... Should Be Equal As Integers ${resp.status_code} 500 + ${total}= Evaluate 2 + 3 + Should Be Equal As Integers ${total} 5 + ${killed}= Mutant Must Fail T1 flip-total + ... Should Be Equal As Integers ${total} 6 Should Be True ${killed} ``` @@ -44,6 +44,16 @@ Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mu ```bash uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate results/mutation-ledger.jsonl uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate output/mutation-ledger.jsonl ``` +Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. + An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. + +## Companion Skills + +| Need | Skill | +|------|-------| +| Write tests, suites, user keywords, resources and variables in Robot Framework syntax | `rf-language` | +| Install Robot Framework or a library, fix the environment | `rf-setup` | diff --git a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py index c35eb5b..599b7fa 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py +++ b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["robotframework>=7"] +# /// """Kill ledger for hand-made mutants. Stdlib only. A check is worth keeping only when it fails once its code or data is broken. diff --git a/plugins/rf-core/skills/rf-test-quality/SKILL.md b/plugins/rf-core/skills/rf-test-quality/SKILL.md index 8f4cb56..b0c66af 100644 --- a/plugins/rf-core/skills/rf-test-quality/SKILL.md +++ b/plugins/rf-core/skills/rf-test-quality/SKILL.md @@ -2,7 +2,7 @@ name: rf-test-quality description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." license: Apache-2.0 -compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled mutation_ledger script (stdlib only otherwise)." metadata: author: manykarim version: "1.0.0" @@ -20,18 +20,18 @@ Every new or changed test is seen failing once. Break the thing it checks, run i ## Mutant checks in a suite -Import `assets/mutants.resource` and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: +Copy `assets/mutants.resource` from this skill into the project's `resources/` folder, import it, and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: ```robotframework *** Settings *** -Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource +Resource ../resources/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code - ${resp}= GET On Session api /health - Should Be Equal As Integers ${resp.status_code} 200 - ${killed}= Mutant Must Fail T1 flip-status - ... Should Be Equal As Integers ${resp.status_code} 500 + ${total}= Evaluate 2 + 3 + Should Be Equal As Integers ${total} 5 + ${killed}= Mutant Must Fail T1 flip-total + ... Should Be Equal As Integers ${total} 6 Should Be True ${killed} ``` @@ -44,6 +44,16 @@ Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mu ```bash uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate results/mutation-ledger.jsonl uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate output/mutation-ledger.jsonl ``` +The script path is relative to this skill's directory. Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. + An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. + +## Companion Skills + +| Need | Skill | +|------|-------| +| Write tests, suites, user keywords, resources and variables in Robot Framework syntax | `rf-language` | +| Install Robot Framework or a library, fix the environment | `rf-setup` | diff --git a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py index c35eb5b..599b7fa 100644 --- a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py +++ b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["robotframework>=7"] +# /// """Kill ledger for hand-made mutants. Stdlib only. A check is worth keeping only when it fails once its code or data is broken. diff --git a/tests/installer/test_plugin_selection.py b/tests/installer/test_plugin_selection.py index 5a76e67..9bb8257 100644 --- a/tests/installer/test_plugin_selection.py +++ b/tests/installer/test_plugin_selection.py @@ -30,7 +30,7 @@ def test_plugins_flag_selects_plugins(two_plugin_bundle: Path, fake_home: Path, prefix = tmp_path / "root" assert main(["install", "--agent", "claude-code", "--prefix", str(prefix), "--yes", "--plugins", "rf-core,rf-web,rf-example"]) == 0 - assert _skills(prefix) == {"rf-language", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium", + assert _skills(prefix) == {"rf-language", "rf-setup", "rf-python-library", "rf-test-quality", "rf-browser", "rf-selenium", "rf-example"} @@ -42,7 +42,7 @@ def test_default_skips_the_bundle_and_installs_every_content_plugin( (path,) = list(tmp_path.rglob("installed.json")) files = [f for ins in json.loads(path.read_text())["installations"] for f in ins["files"]] assert {f["plugin"] for f in files} == {"rf-core", "rf-web", "rf-api", "rf-mobile", "rf-desktop"} - assert len(_skills(prefix)) == 9 + assert len(_skills(prefix)) == 10 def test_bundle_in_file_mode_installs_its_plugins(two_plugin_bundle: Path, fake_home: Path, tmp_path: Path) -> None: @@ -50,7 +50,7 @@ def test_bundle_in_file_mode_installs_its_plugins(two_plugin_bundle: Path, fake_ prefix = tmp_path / "root" assert main(["install", "--agent", "claude-code", "--prefix", str(prefix), "--yes", "--plugins", "rf-agentskills"]) == 0 - assert len(_skills(prefix)) == 9 + assert len(_skills(prefix)) == 10 assert (prefix / "rf-core-files" / "scripts").is_dir() assert not (prefix / "rf-agentskills-files").exists() diff --git a/tests/installer/test_reinstall_prune.py b/tests/installer/test_reinstall_prune.py index 07ac871..72f1c38 100644 --- a/tests/installer/test_reinstall_prune.py +++ b/tests/installer/test_reinstall_prune.py @@ -213,7 +213,7 @@ def test_upgrade_removes_managed_robotcode_overlap_skills_and_agent(fake_home: P assert not path.exists(), path for name in ("rf-libdoc", "rf-results", "rf-robotcode"): assert not (skills / name).exists(), name - assert len([d for d in skills.iterdir() if d.name.startswith("rf-")]) == 9 + assert len([d for d in skills.iterdir() if d.name.startswith("rf-")]) == 10 agents = sorted(p.stem for p in (fake_home / ".claude" / "agents").glob("rf-*.md")) assert agents == ["rf-debug-expert", "rf-migration-guide", "rf-test-architect"] assert "5 removed" in capsys.readouterr().out diff --git a/tests/installer/test_split_upgrade.py b/tests/installer/test_split_upgrade.py index 8a17f89..fd82af1 100644 --- a/tests/installer/test_split_upgrade.py +++ b/tests/installer/test_split_upgrade.py @@ -67,7 +67,7 @@ def test_file_mode_upgrade(agent: str, old_bundle: Path, fake_home: Path, tmp_pa project.mkdir() _old_install(project, agent, old_bundle, monkeypatch) skills_before = _skills(project) - assert len(skills_before) == 9 + assert len(skills_before) == 10 assert _install(project, agent) == 0 root = project / ROOTS[agent] @@ -104,7 +104,7 @@ def test_narrowing_is_explicit(old_bundle: Path, fake_home: Path, tmp_path: Path assert _install(project, "claude-code") == 0 capsys.readouterr() assert _install(project, "claude-code", "--plugins", "rf-core,rf-web") == 0 - assert _skills(project) == {"rf-language", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium"} + assert _skills(project) == {"rf-language", "rf-setup", "rf-python-library", "rf-test-quality", "rf-browser", "rf-selenium"} assert set(_record(project)) == {"rf-core", "rf-web"} out = " ".join(capsys.readouterr().out.split()) assert "skills rf-appium, rf-platynui, rf-requests, rf-restinstance" in out diff --git a/tests/test_skill_commands.py b/tests/test_skill_commands.py index cab77af..3b21eca 100644 --- a/tests/test_skill_commands.py +++ b/tests/test_skill_commands.py @@ -21,10 +21,11 @@ "root": standalone_skills(), "plugin": PLUGIN / "skills", } -SCRIPT_SKILLS = {"rf-language": "rf_conventions.py", "rf-python-library": "check_library.py"} +SCRIPT_SKILLS = {"rf-language": "rf_conventions.py", "rf-python-library": "check_library.py", + "rf-test-quality": "mutation_ledger.py"} #: Skills whose content is Robot Framework code, so `${var}` is expected; only #: agent-specific variables such as ${CLAUDE_SKILL_DIR} are forbidden there. -RF_CODE_SKILLS = {"rf-language", "rf-python-library"} +RF_CODE_SKILLS = {"rf-language", "rf-python-library", "rf-test-quality"} #: Any fenced block (so ```python / ```toml blocks pair correctly); only shell #: blocks (bash/sh/shell/console or no language) count as commands. diff --git a/tests/test_skill_descriptions.py b/tests/test_skill_descriptions.py index 28730df..3886a4f 100644 --- a/tests/test_skill_descriptions.py +++ b/tests/test_skill_descriptions.py @@ -192,6 +192,7 @@ "rf-setup": ("browser", "selenium", "appium", "requests", "restinstance", "platynui", "language"), "rf-language": ("browser", "requests", "setup", "python-library"), "rf-python-library": ("language", "setup"), + "rf-test-quality": ("language", "setup"), } THREE_COLUMN_HEADERS: dict[str, tuple[str, ...]] = {} From f7f6ec8c6a53ac663e80a633e9c600da82c39080 Mon Sep 17 00:00:00 2001 From: ai94iq <5349989+ai94iq@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:55:43 +0300 Subject: [PATCH 10/15] test(rf-test-quality): expect the project copy of mutants.resource --- plugins/rf-core/tests/test_test_quality_skill.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/rf-core/tests/test_test_quality_skill.py b/plugins/rf-core/tests/test_test_quality_skill.py index 810496c..6e049a8 100644 --- a/plugins/rf-core/tests/test_test_quality_skill.py +++ b/plugins/rf-core/tests/test_test_quality_skill.py @@ -58,8 +58,8 @@ def test_resource_classifies_with_is_kill(): assert re.search(r"Is Kill\s+\$\{\{\$status == 'PASS'\}\}\s+\$\{message\}", body) -def test_skill_example_uses_skill_dir_and_a_real_value(): +def test_skill_example_uses_a_project_copy_and_a_real_value(): text = (SKILL / "SKILL.md").read_text(encoding="utf-8") - assert "${CLAUDE_SKILL_DIR}/assets/mutants.resource" in text + assert "resources/mutants.resource" in text and "${CLAUDE_SKILL_DIR}" not in text.split("```bash")[0] assert not re.search(r"(?m)^python ", text) assert "200 500" not in text From 755a958cb499aa7d198f037ac70f5b7cba008a8b Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Mon, 16 Nov 2026 15:42:13 +0300 Subject: [PATCH 11/15] refactor(rf-run-digest): read results with robotcode results, drop bundled parser The skill now points to robotcode results summary/show/stats/log/diff instead of its own output.xml parser and listener. Removes the scripts, their tests and the shrink option; updates the bundle copy, the command test map, the trigger set and the eval task. --- .../skills/rf-run-digest/SKILL.md | 41 ++-- .../rf-run-digest/scripts/digest_listener.py | 75 ------ .../rf-run-digest/scripts/digest_selftest.py | 73 ------ .../rf-run-digest/scripts/run_digest.py | 214 ----------------- .../tasks/narrow/narrow-run-digest-01.yaml | 15 +- .../rf-core/evals/triggers/rf-run-digest.yaml | 2 +- plugins/rf-core/skills/rf-run-digest/SKILL.md | 43 ++-- .../rf-run-digest/scripts/digest_listener.py | 75 ------ .../rf-run-digest/scripts/digest_selftest.py | 73 ------ .../rf-run-digest/scripts/run_digest.py | 214 ----------------- .../rf-core/tests/test_run_digest_skill.py | 217 ------------------ tests/test_skill_commands.py | 3 +- 12 files changed, 42 insertions(+), 1003 deletions(-) delete mode 100644 bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py delete mode 100644 bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_selftest.py delete mode 100644 bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py delete mode 100644 plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py delete mode 100644 plugins/rf-core/skills/rf-run-digest/scripts/digest_selftest.py delete mode 100644 plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py delete mode 100644 plugins/rf-core/tests/test_run_digest_skill.py diff --git a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md index 7d9a2e1..931331d 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md @@ -1,50 +1,41 @@ --- name: rf-run-digest -description: "Use after a Robot Framework run, to read its result from a short digest (summary, grouped failures) instead of output.xml or log.html." +description: "Use after a Robot Framework run, to read its result with `robotcode results` (summary, failing tests, grouped stats, diff) instead of output.xml or log.html." license: Apache-2.0 -compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled run_digest script. Stdlib only otherwise." +compatibility: "Requires robotcode[runner]>=2.8 in the project environment. No bundled scripts." metadata: author: manykarim - version: "1.0.0" + version: "2.0.0" --- # Robot Framework Run Digest Skill ## When to use -Load this skill after a `robot` or `pabot` run, before you look at its result. output.xml and log.html are large; reading them wastes time and hides the few failure groups that matter. +Load this skill after a `robot` or `pabot` run, before you look at its result. output.xml and log.html are large; reading or parsing them wastes time and hides the few failures that matter. ## The rule -Read the digest first, not output.xml. Open output.xml only for one named test, with `trace`. +Read results with `robotcode results`, not output.xml. Never write your own output.xml parser. -## Write a digest on every run +## Read a run -Attach `scripts/digest_listener.py` as a listener. When the run ends, it writes `digest.txt` next to output.xml: +Each command reads the latest output.xml. Pass `-o ` after the subcommand to name another file; `--format json` goes before `results`. ```bash -robot --listener "${CLAUDE_SKILL_DIR}/scripts/digest_listener.py" -d results/run tests/ -cat results/run/digest.txt +uv run robotcode results summary # headline status and counts +uv run robotcode results summary --failed # plus each failing test with its message +uv run robotcode results show --failed # failing tests, one line each +uv run robotcode results stats --by suite --by tag # counts per suite and tag +uv run robotcode results log -bl "" --level FAIL # keyword tree of one test +uv run robotcode results diff before.xml after.xml --only new-failures ``` -The listener leaves output.xml as robot wrote it. To shrink it, pass the `shrink` -argument: `--listener "${CLAUDE_SKILL_DIR}/scripts/digest_listener.py:shrink"`. This rewrites output.xml in -place with `rebot --removekeywords PASSED`, so the keywords of passed tests are gone -from it and from any log built from it later. +Not using uv? Run the same commands with `.venv/bin/robotcode` or `poetry run robotcode`; see rf-setup. -## Read an existing output.xml +Start with `summary --failed`. Open `log` for one named test only. Quote the summary counts and the failure messages when you report a run. -```bash -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" summary results/output.xml # totals and per-suite counts -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" failures results/output.xml --max 3 # failures grouped by signature -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" diff before.xml after.xml # what changed between two runs -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" trace results/output.xml --test "Name" -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" selftest -``` - -Failures that differ only by ids, hosts or timestamps fall into one group, so a run with many failures reads as a few groups. Quote the summary line and the group messages when you report a run. - -It reads Browser, SeleniumLibrary, AppiumLibrary, MaestroLibrary and RequestsLibrary failures; the `screenshot:` line on a failed test comes from the last `` that test logged. +`--search TEXT` or `--search-regex PATTERN` narrows any command to tests whose names, messages, tags or log text match; `-i`/`-e` filter by tag and `-s`/`-t` by suite or test name. ## Companion Skills diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py deleted file mode 100644 index a80b741..0000000 --- a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_listener.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -"""Robot listener: write digest.txt, point the human at report.html. - -With the `shrink` argument (`--listener digest_listener.py:shrink`) it also rewrites -output.xml in place with the keywords of PASSED tests removed. Off by default. - -Attach it to any `robot` or `pabot` command; it fires when the run tears down. - - robot --listener scripts/digest_listener.py -d results/run \ - --output output.xml --log NONE --report NONE tests/ > /dev/null 2>&1 - cat results/run/digest.txt - -Always-on for a shell: - - # PowerShell - $env:ROBOT_OPTIONS = '--listener "C:\\path\\to\\tools\\digest_listener.py"' - # bash - export ROBOT_OPTIONS='--listener /abs/path/scripts/digest_listener.py' - -Read digest.txt first. A person opens robot's own report.html (the listener prints its path -when a run fails and never opens it). Neither one ever reads output.xml or log.html. - -Under pabot each process writes its own digest.txt; read the merged run's instead -(`uv run python scripts/run_digest.py failures `). - -Errors are swallowed: a reporting tool must never fail a run. -""" -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import run_digest # noqa: E402 (sibling file) - -ROBOT_LISTENER_API_VERSION = 3 - - - -def _max_per_group(): - try: - return max(1, int(os.environ.get("DIGEST_MAX_PER_GROUP", "3"))) - except ValueError: - return 3 - - -class digest_listener: # noqa: N801 (robot resolves the listener by file/class name) - - def __init__(self, *args): - self.output_xml = None - self.report = None - self.shrink = "shrink" in args - - def output_file(self, path): - self.output_xml = str(path) - - def report_file(self, path): - self.report = str(path) if path and str(path) != "NONE" else None - - def close(self): - try: - self._write_digest() - except Exception as e: # never raise into the run - print("[digest_listener] skipped: %s" % e) - - def _write_digest(self): - if not self.output_xml or not os.path.exists(self.output_xml): - return - if self.shrink: - run_digest.shrink(self.output_xml) # rewrites output.xml without PASSED keywords - text, failed = run_digest.digest(self.output_xml, _max_per_group()) - path = os.path.join(os.path.dirname(self.output_xml), "digest.txt") - with open(path, "w", encoding="utf-8") as f: - f.write(text) - print("\n[digest_listener] %d failed -> %s" % (failed, path)) - if failed and self.report and os.path.exists(self.report): - print("[digest_listener] report: %s" % os.path.abspath(self.report)) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_selftest.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_selftest.py deleted file mode 100644 index d3ba122..0000000 --- a/bundles/rf-agentskills/skills/rf-run-digest/scripts/digest_selftest.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Self-check for run_digest.py on a tiny generated output.xml.""" -import contextlib -import io -import os -import tempfile - -from run_digest import cmd_diff, cmd_summary, cmd_trace, digest, norm, parse - -SAMPLE = """ - - - - - POST Request : url=http://h/api/orders -path_url=/api/orders/{ID} - POST Response : url=http://h/api/orders -status=400, reason=Bad Request -body=b'{{"error":"Validation failed"}}' - - 400.0 != 200.0 - - - POST Request : url=http://h/api/orders -path_url=/api/orders/{ID2} - 400.0 != 200.0 - - - GET Request : url=http://h/api/orders -path_url=/api/orders - - - - - - - -""" - - -def selftest(): - d = tempfile.mkdtemp() - uid = "11111111-2222-3333-4444-555555555555" - uid2 = "99999999-2222-3333-4444-555555555555" - p = os.path.join(d, "out.xml") - open(p, "w", encoding="utf-8").write(SAMPLE.format(ID=uid, ID2=uid2)) - tests = parse(p) - assert len(tests) == 3, tests - assert [t["status"] for t in tests] == ["FAIL", "FAIL", "PASS"] - t = tests[0] - assert t["suite"] == "01 Orders" and t["req"] == "/api/orders/" + uid, t - assert t["res"] == "400", t - # two failures differing only by UUID must collapse into ONE group - sigs = set(norm(x["msg"]) for x in tests if x["status"] == "FAIL") - assert len(sigs) == 1, sigs - assert norm("Url: /api/x/%s Expected 504 != 200" % uid) == "Url: /api/x/{id} Expected 504 != 200" - assert norm("Url: https://h.dev/api/x/42?p=1 504") == "Url: /api/x/{id} 504" - q = os.path.join(d, "after.xml") - open(q, "w", encoding="utf-8").write( - SAMPLE.format(ID=uid, ID2=uid2).replace('status="FAIL">400.0 != 200.0', 'status="PASS">', 1)) - text, nfail = digest(p) - assert nfail == 2 and "2x 400.0 != 200.0" in text, text - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - cmd_summary(tests + [dict(tests[2], name="Skipped", status="SKIP")]) - assert buf.getvalue().startswith("4 tests, 1 passed, 2 failed, 1 skipped\n"), buf.getvalue() - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - cmd_summary(tests) - assert buf.getvalue().startswith("3 tests, 1 passed, 2 failed\n"), buf.getvalue() - cmd_diff(tests, parse(q)) - cmd_trace(tests, "Create Order Basic") - print(text) - print("selftest ok") diff --git a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py b/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py deleted file mode 100644 index 0b08829..0000000 --- a/bundles/rf-agentskills/skills/rf-run-digest/scripts/run_digest.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# requires-python = ">=3.10" -# dependencies = ["robotframework>=7"] -# /// -"""Read a robot run without reading output.xml. Stdlib only. - - run_digest.py summary OUT.xml # totals + per-suite - run_digest.py failures OUT.xml [--max 3] # grouped by signature - run_digest.py diff BEFORE.xml AFTER.xml # status vs message changes - run_digest.py trace OUT.xml --test NAME # keyword path to the failure - run_digest.py selftest - -Never `cat` output.xml, never open log.html. -""" -import argparse -import io -import os -import re -import subprocess -import sys -import xml.etree.ElementTree as ET - -UUID = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.I) -URL = re.compile(r"https?://\S+|/api/\S*") -TS = re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}\S*") -PATH_URL = re.compile(r"path_url=(\S+)") -STATUS = re.compile(r"\bstatus=(\d+)") -IMG = re.compile(r"]*?src=\"([^\"]+)\"", re.I) -FILE_PATH = re.compile(r"(?:\b[A-Za-z]:[\\/]|(? {id}.""" - u = re.sub(r"^https?://[^/\s]+", "", m.group(0)).split("?")[0] - u = UUID.sub("{id}", u) - return re.sub(r"/\d+(?=/|$)", "/{id}", u) or "/" - - -def norm(msg): - """Grouping signature: drop what varies per test, keep status numbers and path template.""" - s = (msg or "").split("\nCall log:")[0] - s = URL.sub(_path_template, s) - s = FILE_PATH.sub(r"\1", s) - s = UUID.sub("", s) - s = HEX_ID.sub("", s) - s = TS.sub("", s) - return " ".join(s.split())[:200] - - -def clip(s, n=160): - s = " ".join((s or "").split()) - return s[:n] + (" ..." if len(s) > n else "") - - -def parse(path): - """-> one dict per test, in run order, with the last request path and response status. - - "last request" is not always the failing one; `trace` gives the failing keyword path. - """ - tests, suites, cur = [], [], None - for event, el in ET.iterparse(path, events=("start", "end")): - tag = el.tag - if event == "start": - if tag == "suite": - suites.append(el.get("name")) - elif tag == "test": - cur = {"name": el.get("name"), "suite": suites[-1], - "status": "", "msg": "", "req": None, "res": None, - "shot": None, "kws": []} - continue - if cur is not None and tag == "msg": - text = el.text or "" - shot = IMG.search(text) - if shot: # UI libraries log screenshots as - cur["shot"] = shot.group(1) - m = PATH_URL.search(text) - if m: - cur["req"] = m.group(1) - elif STATUS.search(text) and "body=" in text: - cur["res"] = STATUS.search(text).group(1) - elif cur is not None and tag == "kw": - st = el.find("status") - cur["kws"].append((el.get("name") or "", st.get("status", "") if st is not None else "")) - elif tag == "test": - st = el.find("status") - cur["status"] = st.get("status", "") if st is not None else "" - cur["msg"] = (st.text or "").strip() if st is not None else "" - tests.append(cur) - cur = None - el.clear() - elif tag == "suite": - suites.pop() - el.clear() - return tests - - -def cmd_summary(tests): - fail = [t for t in tests if t["status"] == "FAIL"] - skip = sum(t["status"] == "SKIP" for t in tests) - # SKIP counts as neither passed nor failed. - line = "%d tests, %d passed, %d failed" % (len(tests), len(tests) - len(fail) - skip, len(fail)) - print(line + (", %d skipped" % skip if skip else "")) - per = {} - for t in tests: - p, f = per.get(t["suite"], (0, 0)) - per[t["suite"]] = (p + (t["status"] == "PASS"), f + (t["status"] == "FAIL")) - for suite, (p, f) in sorted(per.items(), key=lambda kv: -kv[1][1]): - if f: - print(" %3d failed / %-4d %s" % (f, p + f, suite)) - - -def cmd_failures(tests, max_per_group=3): - groups = {} - for t in tests: - if t["status"] == "FAIL": - groups.setdefault(norm(t["msg"]), []).append(t) - if not groups: - return print("no failures") - for sig, ts in sorted(groups.items(), key=lambda kv: -len(kv[1])): - print("\n %dx %s" % (len(ts), sig)) - for t in ts[:max_per_group]: - print(" %s :: %s" % (t["suite"], t["name"])) - if t["req"]: - print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) - if t["shot"]: - print(" screenshot: %s" % t["shot"]) - if len(ts) > max_per_group: - print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) - - -def cmd_diff(before, after): - b = dict(((t["suite"], t["name"]), t) for t in before) - a = dict(((t["suite"], t["name"]), t) for t in after) - status, message = [], 0 - for k in sorted(set(a) & set(b)): - if a[k]["status"] != b[k]["status"]: - status.append((k, b[k]["status"], a[k]["status"])) - elif norm(a[k]["msg"]) != norm(b[k]["msg"]): - message += 1 - print("%d status changes, %d message changes, +%d new, -%d gone" - % (len(status), message, len(set(a) - set(b)), len(set(b) - set(a)))) - for (suite, name), old, new in status[:40]: - print(" %s -> %s %s :: %s" % (old, new, suite, name)) - - -def cmd_trace(tests, name): - hits = [t for t in tests if t["name"] == name] - if not hits: - sys.exit("no test named %r" % name) - for t in hits: - print("%s :: %s [%s]" % (t["suite"], t["name"], t["status"])) - for kw, st in t["kws"]: - if st == "FAIL": - print(" FAIL %s" % kw) - print(" message: %s" % clip(t["msg"], 400)) - if t["req"]: - print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) - if t["shot"] and t["status"] == "FAIL": - print(" screenshot: %s" % t["shot"]) - - -def shrink(output_xml): - """Rewrite output.xml without PASSED keywords, via rebot in a child process. - - In process, rebot() would re-fire the calling listener's close(): robot's LOGGER is global. - """ - tmp = output_xml + ".small" - r = subprocess.run([sys.executable, "-m", "robot.rebot", "--removekeywords", "PASSED", - "--log", "NONE", "--report", "NONE", "--output", tmp, output_xml], - capture_output=True) - # rebot exits with the failed test count (0-249). - if r.returncode < 250 and os.path.exists(tmp): - os.replace(tmp, output_xml) - return True - return False - - -def digest(output_xml, max_per_group=3): - """summary + failures as one string, for digest_listener.py.""" - tests = parse(output_xml) - buf, real = io.StringIO(), sys.stdout - try: - sys.stdout = buf - cmd_summary(tests) - cmd_failures(tests, max_per_group) - finally: - sys.stdout = real - return buf.getvalue(), sum(1 for t in tests if t["status"] == "FAIL") - - -def main(argv): - ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("cmd", choices=["summary", "failures", "diff", "trace", "selftest"]) - ap.add_argument("xml", nargs="*") - ap.add_argument("--max", type=int, default=3) - ap.add_argument("--test") - a = ap.parse_args(argv) - if a.cmd == "selftest": - from digest_selftest import selftest - return selftest() - if a.cmd == "diff": - return cmd_diff(parse(a.xml[0]), parse(a.xml[1])) - tests = parse(a.xml[0]) - if a.cmd == "trace": - return cmd_trace(tests, a.test) - cmd_summary(tests) - if a.cmd == "failures": - cmd_failures(tests, a.max) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml b/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml index d4a3e60..5e7144c 100644 --- a/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml +++ b/plugins/rf-core/evals/tasks/narrow/narrow-run-digest-01.yaml @@ -2,11 +2,12 @@ id: narrow-run-digest-01 skill: rf-run-digest tier: narrow description: | - Run a suite with the digest listener and report from the digest. Gated by - a digest.txt written next to output.xml that holds the summary line. + Run a suite and report from `robotcode results summary --failed`. Gated by + a results.txt that holds the command's output. prompt: | - Run tests/example.robot into results/ so that a digest.txt is written next - to output.xml, then report the pass and fail counts from that digest. + Run tests/example.robot into results/, save the output of the robotcode + results summary --failed command to results/results.txt, then report the + pass and fail counts from it. model: claude-haiku-5-5 max_turns: 14 timeout_seconds: 300 @@ -20,8 +21,8 @@ allowed_tools: - Skill grader_checks: - type: file_contains - name: digest_written - path: results/digest.txt - regex: '\d+ tests, \d+ passed' + name: results_saved + path: results/results.txt + regex: '_Total:_ \d+' fixture: sut-minimal primary_metric: file_contains diff --git a/plugins/rf-core/evals/triggers/rf-run-digest.yaml b/plugins/rf-core/evals/triggers/rf-run-digest.yaml index ec79f3c..f3ae5d6 100644 --- a/plugins/rf-core/evals/triggers/rf-run-digest.yaml +++ b/plugins/rf-core/evals/triggers/rf-run-digest.yaml @@ -10,7 +10,7 @@ queries: - {id: rd-t01, split: train, should_trigger: true, query: "My robot run just finished with 80 failures, what went wrong?"} - {id: rd-t02, split: train, should_trigger: true, query: "Summarize results/output.xml from the last Robot Framework run"} - {id: rd-t03, split: train, should_trigger: true, note: "indirect", query: "log.html is huge, how do I see which robot tests failed and why?"} - - {id: rd-t04, split: train, should_trigger: true, note: "edit existing file", query: "Edit run_tests.sh so every robot call writes a digest.txt next to output.xml"} + - {id: rd-t04, split: train, should_trigger: true, note: "edit existing file", query: "Edit run_tests.sh so it prints the failed tests from output.xml after every robot call"} - {id: rd-t05, split: train, should_trigger: true, note: "indirect", query: "The nightly robot job went red again, I just need the short story of what broke"} # should trigger (validation) - {id: rd-v01, split: validation, should_trigger: true, query: "Group the failures in this Robot output.xml by error message"} diff --git a/plugins/rf-core/skills/rf-run-digest/SKILL.md b/plugins/rf-core/skills/rf-run-digest/SKILL.md index 6061ff1..931331d 100644 --- a/plugins/rf-core/skills/rf-run-digest/SKILL.md +++ b/plugins/rf-core/skills/rf-run-digest/SKILL.md @@ -1,52 +1,41 @@ --- name: rf-run-digest -description: "Use after a Robot Framework run, to read its result from a short digest (summary, grouped failures) instead of output.xml or log.html." +description: "Use after a Robot Framework run, to read its result with `robotcode results` (summary, failing tests, grouped stats, diff) instead of output.xml or log.html." license: Apache-2.0 -compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled run_digest script. Stdlib only otherwise." +compatibility: "Requires robotcode[runner]>=2.8 in the project environment. No bundled scripts." metadata: author: manykarim - version: "1.0.0" + version: "2.0.0" --- # Robot Framework Run Digest Skill ## When to use -Load this skill after a `robot` or `pabot` run, before you look at its result. output.xml and log.html are large; reading them wastes time and hides the few failure groups that matter. +Load this skill after a `robot` or `pabot` run, before you look at its result. output.xml and log.html are large; reading or parsing them wastes time and hides the few failures that matter. ## The rule -Read the digest first, not output.xml. Open output.xml only for one named test, with `trace`. +Read results with `robotcode results`, not output.xml. Never write your own output.xml parser. -## Write a digest on every run +## Read a run -Attach `scripts/digest_listener.py` as a listener; `` is the folder containing this SKILL.md. When the run ends, it writes `digest.txt` next to output.xml: +Each command reads the latest output.xml. Pass `-o ` after the subcommand to name another file; `--format json` goes before `results`. ```bash -robot --listener /scripts/digest_listener.py -d results/run tests/ -cat results/run/digest.txt +uv run robotcode results summary # headline status and counts +uv run robotcode results summary --failed # plus each failing test with its message +uv run robotcode results show --failed # failing tests, one line each +uv run robotcode results stats --by suite --by tag # counts per suite and tag +uv run robotcode results log -bl "" --level FAIL # keyword tree of one test +uv run robotcode results diff before.xml after.xml --only new-failures ``` -The listener leaves output.xml as robot wrote it. To shrink it, pass the `shrink` -argument: `--listener /scripts/digest_listener.py:shrink`. This rewrites output.xml in -place with `rebot --removekeywords PASSED`, so the keywords of passed tests are gone -from it and from any log built from it later. +Not using uv? Run the same commands with `.venv/bin/robotcode` or `poetry run robotcode`; see rf-setup. -## Read an existing output.xml +Start with `summary --failed`. Open `log` for one named test only. Quote the summary counts and the failure messages when you report a run. -```bash -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" summary results/output.xml # totals and per-suite counts -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" failures results/output.xml --max 3 # failures grouped by signature -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" diff before.xml after.xml # what changed between two runs -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" trace results/output.xml --test "Name" -uv run python "${CLAUDE_SKILL_DIR}/scripts/run_digest.py" selftest -``` - -The script path is relative to this skill's directory; output.xml paths are relative to the project root. Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. - -Failures that differ only by ids, hosts or timestamps fall into one group, so a run with many failures reads as a few groups. Quote the summary line and the group messages when you report a run. - -It reads Browser, SeleniumLibrary, AppiumLibrary, MaestroLibrary and RequestsLibrary failures; the `screenshot:` line on a failed test comes from the last `` that test logged. +`--search TEXT` or `--search-regex PATTERN` narrows any command to tests whose names, messages, tags or log text match; `-i`/`-e` filter by tag and `-s`/`-t` by suite or test name. ## Companion Skills diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py b/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py deleted file mode 100644 index a80b741..0000000 --- a/plugins/rf-core/skills/rf-run-digest/scripts/digest_listener.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -"""Robot listener: write digest.txt, point the human at report.html. - -With the `shrink` argument (`--listener digest_listener.py:shrink`) it also rewrites -output.xml in place with the keywords of PASSED tests removed. Off by default. - -Attach it to any `robot` or `pabot` command; it fires when the run tears down. - - robot --listener scripts/digest_listener.py -d results/run \ - --output output.xml --log NONE --report NONE tests/ > /dev/null 2>&1 - cat results/run/digest.txt - -Always-on for a shell: - - # PowerShell - $env:ROBOT_OPTIONS = '--listener "C:\\path\\to\\tools\\digest_listener.py"' - # bash - export ROBOT_OPTIONS='--listener /abs/path/scripts/digest_listener.py' - -Read digest.txt first. A person opens robot's own report.html (the listener prints its path -when a run fails and never opens it). Neither one ever reads output.xml or log.html. - -Under pabot each process writes its own digest.txt; read the merged run's instead -(`uv run python scripts/run_digest.py failures `). - -Errors are swallowed: a reporting tool must never fail a run. -""" -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import run_digest # noqa: E402 (sibling file) - -ROBOT_LISTENER_API_VERSION = 3 - - - -def _max_per_group(): - try: - return max(1, int(os.environ.get("DIGEST_MAX_PER_GROUP", "3"))) - except ValueError: - return 3 - - -class digest_listener: # noqa: N801 (robot resolves the listener by file/class name) - - def __init__(self, *args): - self.output_xml = None - self.report = None - self.shrink = "shrink" in args - - def output_file(self, path): - self.output_xml = str(path) - - def report_file(self, path): - self.report = str(path) if path and str(path) != "NONE" else None - - def close(self): - try: - self._write_digest() - except Exception as e: # never raise into the run - print("[digest_listener] skipped: %s" % e) - - def _write_digest(self): - if not self.output_xml or not os.path.exists(self.output_xml): - return - if self.shrink: - run_digest.shrink(self.output_xml) # rewrites output.xml without PASSED keywords - text, failed = run_digest.digest(self.output_xml, _max_per_group()) - path = os.path.join(os.path.dirname(self.output_xml), "digest.txt") - with open(path, "w", encoding="utf-8") as f: - f.write(text) - print("\n[digest_listener] %d failed -> %s" % (failed, path)) - if failed and self.report and os.path.exists(self.report): - print("[digest_listener] report: %s" % os.path.abspath(self.report)) diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/digest_selftest.py b/plugins/rf-core/skills/rf-run-digest/scripts/digest_selftest.py deleted file mode 100644 index d3ba122..0000000 --- a/plugins/rf-core/skills/rf-run-digest/scripts/digest_selftest.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Self-check for run_digest.py on a tiny generated output.xml.""" -import contextlib -import io -import os -import tempfile - -from run_digest import cmd_diff, cmd_summary, cmd_trace, digest, norm, parse - -SAMPLE = """ - - - - - POST Request : url=http://h/api/orders -path_url=/api/orders/{ID} - POST Response : url=http://h/api/orders -status=400, reason=Bad Request -body=b'{{"error":"Validation failed"}}' - - 400.0 != 200.0 - - - POST Request : url=http://h/api/orders -path_url=/api/orders/{ID2} - 400.0 != 200.0 - - - GET Request : url=http://h/api/orders -path_url=/api/orders - - - - - - - -""" - - -def selftest(): - d = tempfile.mkdtemp() - uid = "11111111-2222-3333-4444-555555555555" - uid2 = "99999999-2222-3333-4444-555555555555" - p = os.path.join(d, "out.xml") - open(p, "w", encoding="utf-8").write(SAMPLE.format(ID=uid, ID2=uid2)) - tests = parse(p) - assert len(tests) == 3, tests - assert [t["status"] for t in tests] == ["FAIL", "FAIL", "PASS"] - t = tests[0] - assert t["suite"] == "01 Orders" and t["req"] == "/api/orders/" + uid, t - assert t["res"] == "400", t - # two failures differing only by UUID must collapse into ONE group - sigs = set(norm(x["msg"]) for x in tests if x["status"] == "FAIL") - assert len(sigs) == 1, sigs - assert norm("Url: /api/x/%s Expected 504 != 200" % uid) == "Url: /api/x/{id} Expected 504 != 200" - assert norm("Url: https://h.dev/api/x/42?p=1 504") == "Url: /api/x/{id} 504" - q = os.path.join(d, "after.xml") - open(q, "w", encoding="utf-8").write( - SAMPLE.format(ID=uid, ID2=uid2).replace('status="FAIL">400.0 != 200.0', 'status="PASS">', 1)) - text, nfail = digest(p) - assert nfail == 2 and "2x 400.0 != 200.0" in text, text - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - cmd_summary(tests + [dict(tests[2], name="Skipped", status="SKIP")]) - assert buf.getvalue().startswith("4 tests, 1 passed, 2 failed, 1 skipped\n"), buf.getvalue() - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - cmd_summary(tests) - assert buf.getvalue().startswith("3 tests, 1 passed, 2 failed\n"), buf.getvalue() - cmd_diff(tests, parse(q)) - cmd_trace(tests, "Create Order Basic") - print(text) - print("selftest ok") diff --git a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py b/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py deleted file mode 100644 index 0b08829..0000000 --- a/plugins/rf-core/skills/rf-run-digest/scripts/run_digest.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# requires-python = ">=3.10" -# dependencies = ["robotframework>=7"] -# /// -"""Read a robot run without reading output.xml. Stdlib only. - - run_digest.py summary OUT.xml # totals + per-suite - run_digest.py failures OUT.xml [--max 3] # grouped by signature - run_digest.py diff BEFORE.xml AFTER.xml # status vs message changes - run_digest.py trace OUT.xml --test NAME # keyword path to the failure - run_digest.py selftest - -Never `cat` output.xml, never open log.html. -""" -import argparse -import io -import os -import re -import subprocess -import sys -import xml.etree.ElementTree as ET - -UUID = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.I) -URL = re.compile(r"https?://\S+|/api/\S*") -TS = re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}\S*") -PATH_URL = re.compile(r"path_url=(\S+)") -STATUS = re.compile(r"\bstatus=(\d+)") -IMG = re.compile(r"]*?src=\"([^\"]+)\"", re.I) -FILE_PATH = re.compile(r"(?:\b[A-Za-z]:[\\/]|(? {id}.""" - u = re.sub(r"^https?://[^/\s]+", "", m.group(0)).split("?")[0] - u = UUID.sub("{id}", u) - return re.sub(r"/\d+(?=/|$)", "/{id}", u) or "/" - - -def norm(msg): - """Grouping signature: drop what varies per test, keep status numbers and path template.""" - s = (msg or "").split("\nCall log:")[0] - s = URL.sub(_path_template, s) - s = FILE_PATH.sub(r"\1", s) - s = UUID.sub("", s) - s = HEX_ID.sub("", s) - s = TS.sub("", s) - return " ".join(s.split())[:200] - - -def clip(s, n=160): - s = " ".join((s or "").split()) - return s[:n] + (" ..." if len(s) > n else "") - - -def parse(path): - """-> one dict per test, in run order, with the last request path and response status. - - "last request" is not always the failing one; `trace` gives the failing keyword path. - """ - tests, suites, cur = [], [], None - for event, el in ET.iterparse(path, events=("start", "end")): - tag = el.tag - if event == "start": - if tag == "suite": - suites.append(el.get("name")) - elif tag == "test": - cur = {"name": el.get("name"), "suite": suites[-1], - "status": "", "msg": "", "req": None, "res": None, - "shot": None, "kws": []} - continue - if cur is not None and tag == "msg": - text = el.text or "" - shot = IMG.search(text) - if shot: # UI libraries log screenshots as - cur["shot"] = shot.group(1) - m = PATH_URL.search(text) - if m: - cur["req"] = m.group(1) - elif STATUS.search(text) and "body=" in text: - cur["res"] = STATUS.search(text).group(1) - elif cur is not None and tag == "kw": - st = el.find("status") - cur["kws"].append((el.get("name") or "", st.get("status", "") if st is not None else "")) - elif tag == "test": - st = el.find("status") - cur["status"] = st.get("status", "") if st is not None else "" - cur["msg"] = (st.text or "").strip() if st is not None else "" - tests.append(cur) - cur = None - el.clear() - elif tag == "suite": - suites.pop() - el.clear() - return tests - - -def cmd_summary(tests): - fail = [t for t in tests if t["status"] == "FAIL"] - skip = sum(t["status"] == "SKIP" for t in tests) - # SKIP counts as neither passed nor failed. - line = "%d tests, %d passed, %d failed" % (len(tests), len(tests) - len(fail) - skip, len(fail)) - print(line + (", %d skipped" % skip if skip else "")) - per = {} - for t in tests: - p, f = per.get(t["suite"], (0, 0)) - per[t["suite"]] = (p + (t["status"] == "PASS"), f + (t["status"] == "FAIL")) - for suite, (p, f) in sorted(per.items(), key=lambda kv: -kv[1][1]): - if f: - print(" %3d failed / %-4d %s" % (f, p + f, suite)) - - -def cmd_failures(tests, max_per_group=3): - groups = {} - for t in tests: - if t["status"] == "FAIL": - groups.setdefault(norm(t["msg"]), []).append(t) - if not groups: - return print("no failures") - for sig, ts in sorted(groups.items(), key=lambda kv: -len(kv[1])): - print("\n %dx %s" % (len(ts), sig)) - for t in ts[:max_per_group]: - print(" %s :: %s" % (t["suite"], t["name"])) - if t["req"]: - print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) - if t["shot"]: - print(" screenshot: %s" % t["shot"]) - if len(ts) > max_per_group: - print(" ... and %d more (not shown)" % (len(ts) - max_per_group)) - - -def cmd_diff(before, after): - b = dict(((t["suite"], t["name"]), t) for t in before) - a = dict(((t["suite"], t["name"]), t) for t in after) - status, message = [], 0 - for k in sorted(set(a) & set(b)): - if a[k]["status"] != b[k]["status"]: - status.append((k, b[k]["status"], a[k]["status"])) - elif norm(a[k]["msg"]) != norm(b[k]["msg"]): - message += 1 - print("%d status changes, %d message changes, +%d new, -%d gone" - % (len(status), message, len(set(a) - set(b)), len(set(b) - set(a)))) - for (suite, name), old, new in status[:40]: - print(" %s -> %s %s :: %s" % (old, new, suite, name)) - - -def cmd_trace(tests, name): - hits = [t for t in tests if t["name"] == name] - if not hits: - sys.exit("no test named %r" % name) - for t in hits: - print("%s :: %s [%s]" % (t["suite"], t["name"], t["status"])) - for kw, st in t["kws"]: - if st == "FAIL": - print(" FAIL %s" % kw) - print(" message: %s" % clip(t["msg"], 400)) - if t["req"]: - print(" last request: %s -> %s" % (t["req"], t["res"] or "?")) - if t["shot"] and t["status"] == "FAIL": - print(" screenshot: %s" % t["shot"]) - - -def shrink(output_xml): - """Rewrite output.xml without PASSED keywords, via rebot in a child process. - - In process, rebot() would re-fire the calling listener's close(): robot's LOGGER is global. - """ - tmp = output_xml + ".small" - r = subprocess.run([sys.executable, "-m", "robot.rebot", "--removekeywords", "PASSED", - "--log", "NONE", "--report", "NONE", "--output", tmp, output_xml], - capture_output=True) - # rebot exits with the failed test count (0-249). - if r.returncode < 250 and os.path.exists(tmp): - os.replace(tmp, output_xml) - return True - return False - - -def digest(output_xml, max_per_group=3): - """summary + failures as one string, for digest_listener.py.""" - tests = parse(output_xml) - buf, real = io.StringIO(), sys.stdout - try: - sys.stdout = buf - cmd_summary(tests) - cmd_failures(tests, max_per_group) - finally: - sys.stdout = real - return buf.getvalue(), sum(1 for t in tests if t["status"] == "FAIL") - - -def main(argv): - ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("cmd", choices=["summary", "failures", "diff", "trace", "selftest"]) - ap.add_argument("xml", nargs="*") - ap.add_argument("--max", type=int, default=3) - ap.add_argument("--test") - a = ap.parse_args(argv) - if a.cmd == "selftest": - from digest_selftest import selftest - return selftest() - if a.cmd == "diff": - return cmd_diff(parse(a.xml[0]), parse(a.xml[1])) - tests = parse(a.xml[0]) - if a.cmd == "trace": - return cmd_trace(tests, a.test) - cmd_summary(tests) - if a.cmd == "failures": - cmd_failures(tests, a.max) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/plugins/rf-core/tests/test_run_digest_skill.py b/plugins/rf-core/tests/test_run_digest_skill.py deleted file mode 100644 index d8226bb..0000000 --- a/plugins/rf-core/tests/test_run_digest_skill.py +++ /dev/null @@ -1,217 +0,0 @@ -"""rf-run-digest: the digest reports the right totals and groups failures by signature.""" -import subprocess -import sys -from pathlib import Path - -SCRIPT = Path(__file__).resolve().parents[1] / "skills" / "rf-run-digest" / "scripts" / "run_digest.py" - -OUTPUT_XML = """ - - - - Expected 404 but got 200 - Expected 404 but got 200 - - - -""" - - -def _run(*args): - return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True) - - -def test_selftest_passes(): - r = _run("selftest") - assert r.returncode == 0, r.stdout + r.stderr - assert "selftest ok" in r.stdout - - -def test_summary_and_failures_on_tiny_output(tmp_path): - xml = tmp_path / "output.xml" - xml.write_text(OUTPUT_XML, encoding="utf-8") - r = _run("failures", str(xml)) - assert r.returncode == 0, r.stderr - assert r.stdout.startswith("3 tests, 1 passed, 2 failed\n"), r.stdout - assert "2x Expected 404 but got 200" in r.stdout, r.stdout - - -LISTENER = SCRIPT.parent / "digest_listener.py" - - -def _load_listener(): - import importlib.util - spec = importlib.util.spec_from_file_location("digest_listener", LISTENER) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -def test_listener_leaves_output_xml_alone_by_default(tmp_path, monkeypatch): - mod = _load_listener() - calls = [] - monkeypatch.setattr(mod.run_digest, "shrink", lambda path: calls.append(path)) - xml = tmp_path / "output.xml" - xml.write_text(OUTPUT_XML, encoding="utf-8") - listener = mod.digest_listener() - listener.output_file(str(xml)) - listener.close() - assert calls == [] - assert xml.read_text(encoding="utf-8") == OUTPUT_XML - assert (tmp_path / "digest.txt").exists() - - -def test_listener_shrinks_only_when_asked(tmp_path, monkeypatch): - mod = _load_listener() - calls = [] - monkeypatch.setattr(mod.run_digest, "shrink", lambda path: calls.append(path)) - xml = tmp_path / "output.xml" - xml.write_text(OUTPUT_XML, encoding="utf-8") - listener = mod.digest_listener("shrink") - listener.output_file(str(xml)) - listener.close() - assert calls == [str(xml)] - - -SECRET_XML = """ - - - - POST Request : url=http://h/api/login -path_url=/api/login -body=b'{"password": "hunter2"}' - POST Response : url=http://h/api/login -status=500, reason=Server Error -body=b'{"token": "sekret-tok"}' - - 500 != 200 - - - - -""" - - -def test_failures_never_print_request_or_response_bodies(tmp_path): - xml = tmp_path / "output.xml" - xml.write_text(SECRET_XML, encoding="utf-8") - r = _run("failures", str(xml)) - assert r.returncode == 0, r.stderr - assert "hunter2" not in r.stdout, r.stdout - assert "sekret-tok" not in r.stdout, r.stdout - assert "last request: /api/login" in r.stdout, r.stdout - - -def _ui_xml(tests): - """tests: (name, status, screenshot_html_or_None, message). Message text is XML-escaped.""" - from xml.sax.saxutils import escape - rows = [] - for i, (name, status, shot, msg) in enumerate(tests, 1): - kw = '' % status - if shot: - kw = ('%s' - '' % (escape(shot), status)) - rows.append('%s%s' - % (i, name, kw, status, escape(msg))) - return ('\n%s' - '' % "".join(rows)) - - -def _failures(tmp_path, tests): - xml = tmp_path / "output.xml" - xml.write_text(_ui_xml(tests), encoding="utf-8") - r = _run("failures", str(xml)) - assert r.returncode == 0, r.stderr - return r.stdout - - -BROWSER_SHOT = '' -BROWSER_MSG = ("TimeoutError: locator.waitFor: Timeout 60000ms exceeded.\nCall log:\n" - " - waiting for locator('role=main') to be visible\n - %d x locator resolved to hidden") - - -def test_browser_failures_group_across_call_log_and_show_screenshot(tmp_path): - out = _failures(tmp_path, [ - ("Passes", "PASS", '', ""), - ("One", "FAIL", BROWSER_SHOT, BROWSER_MSG % 12), - ("Two", "FAIL", BROWSER_SHOT, BROWSER_MSG % 31), - ]) - assert "2x TimeoutError: locator.waitFor: Timeout 60000ms exceeded.\n" in out, out - assert "screenshot: Direct_Calls_With_IAM_Token_FAILURE_SCREENSHOT_1.png" in out, out - assert "pass-shot.png" not in out, out - - -def test_selenium_failure_shows_screenshot(tmp_path): - out = _failures(tmp_path, [("Login", "FAIL", '', - "Element 'id=login' not visible after 5 seconds.")]) - assert "1x Element 'id=login' not visible after 5 seconds." in out, out - assert "screenshot: selenium-screenshot-1.png" in out, out - - -def test_appium_failure_shows_screenshot(tmp_path): - # Built from AppiumLibrary's message format, not from a live run. - shot = '' - out = _failures(tmp_path, [("Tap", "FAIL", shot, - "Element locator 'accessibility_id=Login' did not match any elements.")]) - assert "1x Element locator 'accessibility_id=Login' did not match any elements." in out, out - assert "screenshot: appium-screenshot-1.png" in out, out - - -def test_maestro_failures_group_across_absolute_paths(tmp_path): - msg = ("Screenshot size mismatch: Assert screenshot matches %s (threshold: 100%%)" - " - expected 900x2000") - shot = '' - out = _failures(tmp_path, [ - ("A", "FAIL", shot, msg % "C:/Users/x/scratchpad/diff-out/probe-full.png"), - ("B", "FAIL", shot, msg % "/home/ci/run2/probe-full.png"), - ("C", "FAIL", shot, "Element not found: Text matching regex: Network.*"), - ]) - assert "2x Screenshot size mismatch: Assert screenshot matches probe-full.png" in out, out - assert "1x Element not found: Text matching regex: Network.*" in out, out - assert "screenshot: maestro-screenshot-1.jpg" in out, out - assert "C:/Users" not in out, out - - -def test_requests_failure_keeps_endpoint_and_no_screenshot(tmp_path): - xml = tmp_path / "output.xml" - xml.write_text(SECRET_XML, encoding="utf-8") - out = _run("failures", str(xml)).stdout - assert "last request: /api/login -> 500" in out, out - assert "screenshot:" not in out, out - - -def test_session_ids_collapse(tmp_path): - out = _failures(tmp_path, [ - ("A", "FAIL", None, "invalid session id: " + "a" * 32), - ("B", "FAIL", None, "invalid session id: " + "b1" * 20), - ]) - assert "2x invalid session id: " in out, out - - -def test_trace_shows_screenshot(tmp_path): - xml = tmp_path / "output.xml" - xml.write_text(_ui_xml([("One", "FAIL", BROWSER_SHOT, BROWSER_MSG % 3)]), encoding="utf-8") - out = _run("trace", str(xml), "--test", "One").stdout - assert "screenshot: Direct_Calls_With_IAM_Token_FAILURE_SCREENSHOT_1.png" in out, out - - -def test_listener_survives_bad_env_value(tmp_path, monkeypatch): - monkeypatch.setenv("DIGEST_MAX_PER_GROUP", "not-a-number") - mod = _load_listener() - xml = tmp_path / "output.xml" - xml.write_text(SECRET_XML, encoding="utf-8") - listener = mod.digest_listener() - listener.output_file(str(xml)) - listener.close() - assert (tmp_path / "digest.txt").exists() - - -def test_parse_keeps_no_request_or_response_bodies(tmp_path): - import importlib.util - spec = importlib.util.spec_from_file_location("run_digest", SCRIPT) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - xml = tmp_path / "output.xml" - xml.write_text(SECRET_XML, encoding="utf-8") - for test in mod.parse(str(xml)): - assert "sent" not in test and "body" not in test, test diff --git a/tests/test_skill_commands.py b/tests/test_skill_commands.py index 91e19f5..cab77af 100644 --- a/tests/test_skill_commands.py +++ b/tests/test_skill_commands.py @@ -21,8 +21,7 @@ "root": standalone_skills(), "plugin": PLUGIN / "skills", } -SCRIPT_SKILLS = {"rf-language": "rf_conventions.py", "rf-python-library": "check_library.py", - "rf-run-digest": "run_digest.py"} +SCRIPT_SKILLS = {"rf-language": "rf_conventions.py", "rf-python-library": "check_library.py"} #: Skills whose content is Robot Framework code, so `${var}` is expected; only #: agent-specific variables such as ${CLAUDE_SKILL_DIR} are forbidden there. RF_CODE_SKILLS = {"rf-language", "rf-python-library"} From 166e2a1e90a3931d9c79b7dfb333ff9bc945601f Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Mon, 16 Nov 2026 15:48:45 +0300 Subject: [PATCH 12/15] docs(rf-run-digest): match the robotcode results docs (result file lookup, JSON grouping) --- bundles/rf-agentskills/skills/rf-run-digest/SKILL.md | 6 ++++-- plugins/rf-core/skills/rf-run-digest/SKILL.md | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md index 931331d..3ccdb2b 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md @@ -20,7 +20,7 @@ Read results with `robotcode results`, not output.xml. Never write your own outp ## Read a run -Each command reads the latest output.xml. Pass `-o ` after the subcommand to name another file; `--format json` goes before `results`. +Without `-o`, each command finds the result file from the project's robot.toml profile (output.xml or output.json). Pass `-o ` after the subcommand to name another one. `--format json` goes before `results`. ```bash uv run robotcode results summary # headline status and counts @@ -35,7 +35,9 @@ Not using uv? Run the same commands with `.venv/bin/robotcode` or `poetry run ro Start with `summary --failed`. Open `log` for one named test only. Quote the summary counts and the failure messages when you report a run. -`--search TEXT` or `--search-regex PATTERN` narrows any command to tests whose names, messages, tags or log text match; `-i`/`-e` filter by tag and `-s`/`-t` by suite or test name. +`--search TEXT` or `--search-regex PATTERN` narrows any command to tests whose names, messages, tags or log text match; `-i`/`-e` filter by tag and `-s`/`-t` by suite or test name. Filters run before counting, so `stats --by tag --search Timeout` counts only the matching tests. + +`results` has no failure grouping. To group failures that differ only by ids or hosts, read `uv run robotcode --format json results summary --failed` and group the `failed[].message` values yourself. `results diff` always exits 0; read `newFailures` from its JSON to gate a build. ## Companion Skills diff --git a/plugins/rf-core/skills/rf-run-digest/SKILL.md b/plugins/rf-core/skills/rf-run-digest/SKILL.md index 931331d..3ccdb2b 100644 --- a/plugins/rf-core/skills/rf-run-digest/SKILL.md +++ b/plugins/rf-core/skills/rf-run-digest/SKILL.md @@ -20,7 +20,7 @@ Read results with `robotcode results`, not output.xml. Never write your own outp ## Read a run -Each command reads the latest output.xml. Pass `-o ` after the subcommand to name another file; `--format json` goes before `results`. +Without `-o`, each command finds the result file from the project's robot.toml profile (output.xml or output.json). Pass `-o ` after the subcommand to name another one. `--format json` goes before `results`. ```bash uv run robotcode results summary # headline status and counts @@ -35,7 +35,9 @@ Not using uv? Run the same commands with `.venv/bin/robotcode` or `poetry run ro Start with `summary --failed`. Open `log` for one named test only. Quote the summary counts and the failure messages when you report a run. -`--search TEXT` or `--search-regex PATTERN` narrows any command to tests whose names, messages, tags or log text match; `-i`/`-e` filter by tag and `-s`/`-t` by suite or test name. +`--search TEXT` or `--search-regex PATTERN` narrows any command to tests whose names, messages, tags or log text match; `-i`/`-e` filter by tag and `-s`/`-t` by suite or test name. Filters run before counting, so `stats --by tag --search Timeout` counts only the matching tests. + +`results` has no failure grouping. To group failures that differ only by ids or hosts, read `uv run robotcode --format json results summary --failed` and group the `failed[].message` values yourself. `results diff` always exits 0; read `newFailures` from its JSON to gate a build. ## Companion Skills From 2adc3d13b2170ae4708f3a07f33262bbb7733d4c Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Mon, 16 Nov 2026 16:11:04 +0300 Subject: [PATCH 13/15] docs(rf-run-digest): shorter description --- bundles/rf-agentskills/skills/rf-run-digest/SKILL.md | 2 +- plugins/rf-core/skills/rf-run-digest/SKILL.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md index 7bed8e7..226f315 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md @@ -1,6 +1,6 @@ --- name: rf-run-digest -description: "Use after a Robot Framework run, to read its result with `robotcode results` (summary, failing tests, grouped stats, diff) instead of output.xml or log.html." +description: "Use after a Robot Framework run: read its result with robotcode results, not output.xml or log.html." license: Apache-2.0 compatibility: "Requires robotcode[all]>=2.8 (or robotcode[runner]) in the project environment for the results commands. No bundled scripts." metadata: diff --git a/plugins/rf-core/skills/rf-run-digest/SKILL.md b/plugins/rf-core/skills/rf-run-digest/SKILL.md index 7bed8e7..226f315 100644 --- a/plugins/rf-core/skills/rf-run-digest/SKILL.md +++ b/plugins/rf-core/skills/rf-run-digest/SKILL.md @@ -1,6 +1,6 @@ --- name: rf-run-digest -description: "Use after a Robot Framework run, to read its result with `robotcode results` (summary, failing tests, grouped stats, diff) instead of output.xml or log.html." +description: "Use after a Robot Framework run: read its result with robotcode results, not output.xml or log.html." license: Apache-2.0 compatibility: "Requires robotcode[all]>=2.8 (or robotcode[runner]) in the project environment for the results commands. No bundled scripts." metadata: From f06155d837abd5e547b10f488cf34bd9ffbc46bd Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Mon, 16 Nov 2026 16:56:05 +0300 Subject: [PATCH 14/15] docs(rf-test-quality): shorter description --- bundles/rf-agentskills/skills/rf-test-quality/SKILL.md | 8 ++++---- plugins/rf-core/skills/rf-test-quality/SKILL.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md index 5fcc33a..e46d4c6 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md @@ -1,6 +1,6 @@ --- name: rf-test-quality -description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." +description: "Use after changing an RF test, to prove it fails when its code breaks (mutants)." license: Apache-2.0 compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled mutation_ledger script (stdlib only otherwise)." metadata: @@ -20,11 +20,11 @@ Every new or changed test is seen failing once. Break the thing it checks, run i ## Mutant checks in a suite -Import `assets/mutants.resource` and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: +Copy `assets/mutants.resource` from this skill into the project's `resources/` folder, import it, and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: ```robotframework *** Settings *** -Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource +Resource ../resources/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code @@ -47,7 +47,7 @@ uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate output/mutation-ledger.jsonl ``` -Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. +The script path is relative to this skill's directory. Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/plugins/rf-core/skills/rf-test-quality/SKILL.md b/plugins/rf-core/skills/rf-test-quality/SKILL.md index b0c66af..e46d4c6 100644 --- a/plugins/rf-core/skills/rf-test-quality/SKILL.md +++ b/plugins/rf-core/skills/rf-test-quality/SKILL.md @@ -1,6 +1,6 @@ --- name: rf-test-quality -description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." +description: "Use after changing an RF test, to prove it fails when its code breaks (mutants)." license: Apache-2.0 compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled mutation_ledger script (stdlib only otherwise)." metadata: From d36753cf222665f2f8ebc3d96f2dcd1a560391d4 Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Mon, 16 Nov 2026 16:56:29 +0300 Subject: [PATCH 15/15] docs(rf-run-digest): shorter description --- bundles/rf-agentskills/skills/rf-run-digest/SKILL.md | 2 +- plugins/rf-core/skills/rf-run-digest/SKILL.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md index 226f315..eb37b18 100644 --- a/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-run-digest/SKILL.md @@ -1,6 +1,6 @@ --- name: rf-run-digest -description: "Use after a Robot Framework run: read its result with robotcode results, not output.xml or log.html." +description: "Use after an RF run: read results with robotcode results, not output.xml." license: Apache-2.0 compatibility: "Requires robotcode[all]>=2.8 (or robotcode[runner]) in the project environment for the results commands. No bundled scripts." metadata: diff --git a/plugins/rf-core/skills/rf-run-digest/SKILL.md b/plugins/rf-core/skills/rf-run-digest/SKILL.md index 226f315..eb37b18 100644 --- a/plugins/rf-core/skills/rf-run-digest/SKILL.md +++ b/plugins/rf-core/skills/rf-run-digest/SKILL.md @@ -1,6 +1,6 @@ --- name: rf-run-digest -description: "Use after a Robot Framework run: read its result with robotcode results, not output.xml or log.html." +description: "Use after an RF run: read results with robotcode results, not output.xml." license: Apache-2.0 compatibility: "Requires robotcode[all]>=2.8 (or robotcode[runner]) in the project environment for the results commands. No bundled scripts." metadata: