From bba998d42dfd98404c6f907937aaeb893356ee8d Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Sun, 15 Nov 2026 09:50:52 +0300 Subject: [PATCH 1/5] feat: add rf-test-quality mutant checks skill Adds a skill, a Mutant Must Fail resource and a stdlib kill ledger so a new or changed test is shown to fail when its check is broken. An empty ledger reports rate 0.0, never a full kill. Verified: pytest plugins/rf-core/tests/test_test_quality_skill.py, 4 passed; with the ledger broken, 3 failed. build-catalog --check, governance, validate-skills, plugin references and docs voice pass. --- .../skills/rf-test-quality/SKILL.md | 44 +++++++++++++ .../rf-test-quality/assets/mutants.resource | 22 +++++++ .../scripts/mutation_ledger.py | 64 +++++++++++++++++++ .../narrow/narrow-test-quality-mutant-01.yaml | 33 ++++++++++ .../evals/triggers/rf-test-quality.yaml | 20 ++++++ .../rf-core/skills/rf-test-quality/SKILL.md | 44 +++++++++++++ .../rf-test-quality/assets/mutants.resource | 22 +++++++ .../scripts/mutation_ledger.py | 64 +++++++++++++++++++ .../rf-core/tests/test_test_quality_skill.py | 37 +++++++++++ 9 files changed, 350 insertions(+) create mode 100644 bundles/rf-agentskills/skills/rf-test-quality/SKILL.md create mode 100644 bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource create mode 100644 bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py create mode 100644 plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml create mode 100644 plugins/rf-core/evals/triggers/rf-test-quality.yaml create mode 100644 plugins/rf-core/skills/rf-test-quality/SKILL.md create mode 100644 plugins/rf-core/skills/rf-test-quality/assets/mutants.resource create mode 100644 plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py create mode 100644 plugins/rf-core/tests/test_test_quality_skill.py diff --git a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md new file mode 100644 index 0000000..578eb3a --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md @@ -0,0 +1,44 @@ +--- +name: rf-test-quality +description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." +license: Apache-2.0 +compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +metadata: + author: manykarim + version: "1.0.0" +--- + +# Robot Framework Test Quality Skill + +## When to use + +Load this skill after you add or change a test, before you call it done. A test that passes against broken code proves nothing. + +## The rule + +Every new or changed test is seen failing once. Break the thing it checks, run it, and confirm it goes red. Then restore the code and confirm it goes green. + +## Mutant checks in a suite + +Import `assets/mutants.resource` and wrap a check in `Mutant Must Fail` with a mutated input or expectation: + +```robotframework +*** Settings *** +Resource mutants.resource + +*** Test Cases *** +Status Check Catches A Wrong Code + ${killed}= Mutant Must Fail T1 flip-status Should Be Equal As Integers 200 500 + Should Be True ${killed} +``` + +Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. + +## Reading the ledger + +```bash +python scripts/mutation_ledger.py rate results/mutation-ledger.jsonl +python scripts/mutation_ledger.py selftest +``` + +An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource b/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource new file mode 100644 index 0000000..3ea4ec6 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource @@ -0,0 +1,22 @@ +*** Settings *** +Documentation Mutant checks: a new or changed test must fail when its code or data is broken. +Library ../scripts/mutation_ledger.py + +*** Variables *** +${MUTATION_LOG} ${OUTPUT_DIR}${/}mutation-ledger.jsonl + +*** Keywords *** +Mutant Must Fail + [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail. + ... Writes one ledger row and returns True when the mutant was killed. + ... A survivor is a finding about the check, never a pass. + [Arguments] ${test_id} ${operator} ${keyword} @{args} + ${passed}= Run Keyword And Return Status ${keyword} @{args} + ${killed}= Evaluate not $passed + Record Mutation ${MUTATION_LOG} ${test_id} ${operator} ${killed} + RETURN ${killed} + +Mutation Kill Rate + [Documentation] Returns ``{killed, total, rate, survivors}`` read from the ledger. + ${r}= Kill Rate ${MUTATION_LOG} + RETURN ${r} diff --git a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py new file mode 100644 index 0000000..75a40e4 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Kill ledger for hand-made mutants. Stdlib only. + +A check is worth keeping only when it fails once its code or data is broken. +Each mutant run appends one JSON line; the kill rate is read back from the file. + + python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} + python scripts/mutation_ledger.py selftest + +Also importable as a Robot Framework library (see assets/mutants.resource). +""" +import json +import os +import sys +import tempfile + + +def record_mutation(path, test_id, operator, killed): + """Append one row. ``killed`` is True when the mutated run failed.""" + row = {"test": str(test_id), "operator": str(operator), "killed": bool(killed)} + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(row) + "\n") + return row + + +def kill_rate(path): + """Summarise the ledger. A missing or empty ledger has rate 0.0, never 1.0.""" + rows = [] + if os.path.exists(path): + with open(path, encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] + killed = sum(1 for r in rows if r["killed"]) + survivors = [f'{r["test"]}:{r["operator"]}' for r in rows if not r["killed"]] + total = len(rows) + return {"killed": killed, "total": total, + "rate": round(killed / total, 3) if total else 0.0, + "survivors": survivors} + + +def _selftest(): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "kills.jsonl") + assert kill_rate(p)["rate"] == 0.0 + record_mutation(p, "T1", "flip-status", True) + record_mutation(p, "T1", "drop-field", False) + r = kill_rate(p) + assert r == {"killed": 1, "total": 2, "rate": 0.5, + "survivors": ["T1:drop-field"]}, r + print(json.dumps({"selftest": "ok"})) + + +def main(argv): + if argv[:1] == ["selftest"]: + _selftest() + return 0 + if len(argv) == 2 and argv[0] == "rate": + print(json.dumps(kill_rate(argv[1]))) + return 0 + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml b/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml new file mode 100644 index 0000000..dab0517 --- /dev/null +++ b/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml @@ -0,0 +1,33 @@ +id: narrow-test-quality-mutant-01 +skill: rf-test-quality +tier: narrow +description: | + Add a mutant check for an existing equality test and run it. Gated by a + Mutant Must Fail call in the suite and a mutation ledger row written by + the run. +prompt: | + tests/smoke.robot checks that a value equals 200. Prove the check fails + when the expected value is wrong: add a mutant check with a wrong value, + run the suite into results/, and report the kill rate. +model: claude-haiku-5-5 +max_turns: 14 +timeout_seconds: 300 +allowed_tools: + - Read + - Write + - Edit + - Bash + - Glob + - Grep + - Skill +grader_checks: + - type: file_contains + name: suite_uses_mutant_check + path: tests/smoke.robot + regex: 'Mutant Must Fail' + - type: file_contains + name: ledger_row_written + path: results/mutation-ledger.jsonl + regex: '"killed": true' +fixture: sut-minimal +primary_metric: file_contains diff --git a/plugins/rf-core/evals/triggers/rf-test-quality.yaml b/plugins/rf-core/evals/triggers/rf-test-quality.yaml new file mode 100644 index 0000000..5d113db --- /dev/null +++ b/plugins/rf-core/evals/triggers/rf-test-quality.yaml @@ -0,0 +1,20 @@ +# Trigger query set for rf-test-quality (format: eval/triggers/README.md). +# Tune descriptions on the train split only; never edit validation queries +# to make a description pass (rotate them only when the skill's intent changes). +skill: rf-test-quality +model: claude-haiku-5-5 +runs: 3 +threshold: 0.5 +queries: + # should trigger (train) + - {id: tq-t01, split: train, should_trigger: true, query: "I added a Robot Framework test, how do I prove it fails when the code is broken?"} + - {id: tq-t02, split: train, should_trigger: true, query: "Add mutant checks to my Robot suite and report the kill rate"} + - {id: tq-t03, split: train, should_trigger: true, note: "user phrasing", query: "my robot test always passes even when the endpoint is wrong, is the check too weak?"} + # should trigger (validation) + - {id: tq-v01, split: validation, should_trigger: true, query: "Show that the new status code check in tests/api.robot goes red on a wrong value"} + - {id: tq-v02, split: validation, should_trigger: true, query: "Which of my changed Robot tests survive a mutated expectation?"} + # should NOT trigger (train) + - {id: tq-n01, split: train, should_trigger: false, note: "sibling rf-setup", query: "Install robotframework-browser into our uv project"} + - {id: tq-n02, split: train, should_trigger: false, note: "sibling rf-language", query: "Add a suite setup in an __init__.robot file"} + # should NOT trigger (validation) + - {id: tq-nv01, split: validation, should_trigger: false, note: "non-RF look-alike", query: "Run mutmut on my Python package"} diff --git a/plugins/rf-core/skills/rf-test-quality/SKILL.md b/plugins/rf-core/skills/rf-test-quality/SKILL.md new file mode 100644 index 0000000..578eb3a --- /dev/null +++ b/plugins/rf-core/skills/rf-test-quality/SKILL.md @@ -0,0 +1,44 @@ +--- +name: rf-test-quality +description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." +license: Apache-2.0 +compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +metadata: + author: manykarim + version: "1.0.0" +--- + +# Robot Framework Test Quality Skill + +## When to use + +Load this skill after you add or change a test, before you call it done. A test that passes against broken code proves nothing. + +## The rule + +Every new or changed test is seen failing once. Break the thing it checks, run it, and confirm it goes red. Then restore the code and confirm it goes green. + +## Mutant checks in a suite + +Import `assets/mutants.resource` and wrap a check in `Mutant Must Fail` with a mutated input or expectation: + +```robotframework +*** Settings *** +Resource mutants.resource + +*** Test Cases *** +Status Check Catches A Wrong Code + ${killed}= Mutant Must Fail T1 flip-status Should Be Equal As Integers 200 500 + Should Be True ${killed} +``` + +Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. + +## Reading the ledger + +```bash +python scripts/mutation_ledger.py rate results/mutation-ledger.jsonl +python scripts/mutation_ledger.py selftest +``` + +An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource b/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource new file mode 100644 index 0000000..3ea4ec6 --- /dev/null +++ b/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource @@ -0,0 +1,22 @@ +*** Settings *** +Documentation Mutant checks: a new or changed test must fail when its code or data is broken. +Library ../scripts/mutation_ledger.py + +*** Variables *** +${MUTATION_LOG} ${OUTPUT_DIR}${/}mutation-ledger.jsonl + +*** Keywords *** +Mutant Must Fail + [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail. + ... Writes one ledger row and returns True when the mutant was killed. + ... A survivor is a finding about the check, never a pass. + [Arguments] ${test_id} ${operator} ${keyword} @{args} + ${passed}= Run Keyword And Return Status ${keyword} @{args} + ${killed}= Evaluate not $passed + Record Mutation ${MUTATION_LOG} ${test_id} ${operator} ${killed} + RETURN ${killed} + +Mutation Kill Rate + [Documentation] Returns ``{killed, total, rate, survivors}`` read from the ledger. + ${r}= Kill Rate ${MUTATION_LOG} + RETURN ${r} diff --git a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py new file mode 100644 index 0000000..75a40e4 --- /dev/null +++ b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Kill ledger for hand-made mutants. Stdlib only. + +A check is worth keeping only when it fails once its code or data is broken. +Each mutant run appends one JSON line; the kill rate is read back from the file. + + python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} + python scripts/mutation_ledger.py selftest + +Also importable as a Robot Framework library (see assets/mutants.resource). +""" +import json +import os +import sys +import tempfile + + +def record_mutation(path, test_id, operator, killed): + """Append one row. ``killed`` is True when the mutated run failed.""" + row = {"test": str(test_id), "operator": str(operator), "killed": bool(killed)} + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(row) + "\n") + return row + + +def kill_rate(path): + """Summarise the ledger. A missing or empty ledger has rate 0.0, never 1.0.""" + rows = [] + if os.path.exists(path): + with open(path, encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] + killed = sum(1 for r in rows if r["killed"]) + survivors = [f'{r["test"]}:{r["operator"]}' for r in rows if not r["killed"]] + total = len(rows) + return {"killed": killed, "total": total, + "rate": round(killed / total, 3) if total else 0.0, + "survivors": survivors} + + +def _selftest(): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "kills.jsonl") + assert kill_rate(p)["rate"] == 0.0 + record_mutation(p, "T1", "flip-status", True) + record_mutation(p, "T1", "drop-field", False) + r = kill_rate(p) + assert r == {"killed": 1, "total": 2, "rate": 0.5, + "survivors": ["T1:drop-field"]}, r + print(json.dumps({"selftest": "ok"})) + + +def main(argv): + if argv[:1] == ["selftest"]: + _selftest() + return 0 + if len(argv) == 2 and argv[0] == "rate": + print(json.dumps(kill_rate(argv[1]))) + return 0 + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/plugins/rf-core/tests/test_test_quality_skill.py b/plugins/rf-core/tests/test_test_quality_skill.py new file mode 100644 index 0000000..cd947a1 --- /dev/null +++ b/plugins/rf-core/tests/test_test_quality_skill.py @@ -0,0 +1,37 @@ +"""rf-test-quality: the kill ledger counts survivors and never reports an empty run as killed.""" +import importlib.util +import subprocess +import sys +from pathlib import Path + +SKILL = Path(__file__).resolve().parents[1] / "skills" / "rf-test-quality" +SCRIPT = SKILL / "scripts" / "mutation_ledger.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("mutation_ledger", SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def test_survivor_is_counted(tmp_path): + m = _load() + p = tmp_path / "k.jsonl" + m.record_mutation(str(p), "T1", "flip", True) + m.record_mutation(str(p), "T1", "drop", False) + assert m.kill_rate(str(p)) == {"killed": 1, "total": 2, "rate": 0.5, "survivors": ["T1:drop"]} + + +def test_empty_ledger_is_not_a_full_kill(tmp_path): + assert _load().kill_rate(str(tmp_path / "none.jsonl"))["rate"] == 0.0 + + +def test_selftest_passes(): + r = subprocess.run([sys.executable, str(SCRIPT), "selftest"], capture_output=True, text=True) + assert r.returncode == 0 and '"ok"' in r.stdout + + +def test_resource_defines_keywords(): + text = (SKILL / "assets" / "mutants.resource").read_text(encoding="utf-8") + assert "Mutant Must Fail" in text and "Mutation Kill Rate" in text From 73da3625ba45094d52ed1814afc4f568adbe4ea6 Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Sun, 15 Nov 2026 10:51:35 +0300 Subject: [PATCH 2/5] fix: score only real kills, latest row per mutant Mutant Must Fail now counts a kill only on an assertion failure; a missing keyword, missing variable or wrong argument count is a survivor. kill_rate keeps the latest ledger row per test and operator. SKILL.md mutates the value the test checks, imports the resource through ${CLAUDE_SKILL_DIR} and runs scripts with uv run. Trigger set grows to 8 positive and 8 negative queries with the sibling and look-alike quotas; the narrow task targets tests/example.robot and adds robot_pass. Tests use 3-of-4 data and assert the exact rate. Verified: uv run --frozen python -m pytest -q plugins/rf-core/tests/test_test_quality_skill.py tests/eval/test_trigger_sets.py: red 6 failed, 29 passed (../red.log), green 35 passed (../green.log). selftest ok. Plugin and bundle copies identical (cmp). --- .../skills/rf-test-quality/SKILL.md | 17 +++++--- .../rf-test-quality/assets/mutants.resource | 7 ++-- .../scripts/mutation_ledger.py | 27 +++++++++--- .../narrow/narrow-test-quality-mutant-01.yaml | 14 ++++--- .../evals/triggers/rf-test-quality.yaml | 10 ++++- .../rf-core/skills/rf-test-quality/SKILL.md | 17 +++++--- .../rf-test-quality/assets/mutants.resource | 7 ++-- .../scripts/mutation_ledger.py | 27 +++++++++--- .../rf-core/tests/test_test_quality_skill.py | 42 +++++++++++++++---- 9 files changed, 125 insertions(+), 43 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md index 578eb3a..8f4cb56 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md @@ -20,25 +20,30 @@ Every new or changed test is seen failing once. Break the thing it checks, run i ## Mutant checks in a suite -Import `assets/mutants.resource` and wrap a check in `Mutant Must Fail` with a mutated input or expectation: +Import `assets/mutants.resource` and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: ```robotframework *** Settings *** -Resource mutants.resource +Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code - ${killed}= Mutant Must Fail T1 flip-status Should Be Equal As Integers 200 500 + ${resp}= GET On Session api /health + Should Be Equal As Integers ${resp.status_code} 200 + ${killed}= Mutant Must Fail T1 flip-status + ... Should Be Equal As Integers ${resp.status_code} 500 Should Be True ${killed} ``` -Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. +Only an assertion failure counts as a kill. A mutant that fails because of a typo, a missing keyword, a missing variable or a wrong argument count is recorded as a survivor: fix the mutant call. + +Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. A rerun of the same test and operator replaces its earlier row, so the rate uses the latest result per mutant. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. ## Reading the ledger ```bash -python scripts/mutation_ledger.py rate results/mutation-ledger.jsonl -python scripts/mutation_ledger.py selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate results/mutation-ledger.jsonl +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest ``` An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource b/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource index 3ea4ec6..d025da0 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource +++ b/bundles/rf-agentskills/skills/rf-test-quality/assets/mutants.resource @@ -7,12 +7,13 @@ ${MUTATION_LOG} ${OUTPUT_DIR}${/}mutation-ledger.jsonl *** Keywords *** Mutant Must Fail - [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail. + [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail + ... on its assertion. A typo or a missing keyword is not a kill. ... Writes one ledger row and returns True when the mutant was killed. ... A survivor is a finding about the check, never a pass. [Arguments] ${test_id} ${operator} ${keyword} @{args} - ${passed}= Run Keyword And Return Status ${keyword} @{args} - ${killed}= Evaluate not $passed + ${status} ${message}= Run Keyword And Ignore Error ${keyword} @{args} + ${killed}= Is Kill ${{$status == 'PASS'}} ${message} Record Mutation ${MUTATION_LOG} ${test_id} ${operator} ${killed} RETURN ${killed} diff --git a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py index 75a40e4..c35eb5b 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py +++ b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py @@ -4,16 +4,28 @@ A check is worth keeping only when it fails once its code or data is broken. Each mutant run appends one JSON line; the kill rate is read back from the file. - python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} - python scripts/mutation_ledger.py selftest + uv run python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} + uv run python scripts/mutation_ledger.py selftest Also importable as a Robot Framework library (see assets/mutants.resource). """ import json import os +import re import sys import tempfile +# Failures that mean the mutant never ran its check (typo, bad call), not a kill. +_NOT_A_KILL = re.compile( + r"No keyword with name|Variable '.*' not found|expected \d+(?: to \d+)? arguments?, got" + r"|Importing .* failed|Resource file .* does not exist" +) + + +def is_kill(passed, message): + """True only when the mutated check failed on its assertion.""" + return not passed and not _NOT_A_KILL.search(str(message)) + def record_mutation(path, test_id, operator, killed): """Append one row. ``killed`` is True when the mutated run failed.""" @@ -29,6 +41,8 @@ def kill_rate(path): if os.path.exists(path): with open(path, encoding="utf-8") as fh: rows = [json.loads(line) for line in fh if line.strip()] + # The ledger is append-only: a rerun of a mutant replaces its earlier row. + rows = list({(r["test"], r["operator"]): r for r in rows}.values()) killed = sum(1 for r in rows if r["killed"]) survivors = [f'{r["test"]}:{r["operator"]}' for r in rows if not r["killed"]] total = len(rows) @@ -41,11 +55,12 @@ def _selftest(): with tempfile.TemporaryDirectory() as d: p = os.path.join(d, "kills.jsonl") assert kill_rate(p)["rate"] == 0.0 - record_mutation(p, "T1", "flip-status", True) - record_mutation(p, "T1", "drop-field", False) + for op, killed in (("a", False), ("a", True), ("b", True), ("c", True), ("d", False)): + record_mutation(p, "T1", op, killed) r = kill_rate(p) - assert r == {"killed": 1, "total": 2, "rate": 0.5, - "survivors": ["T1:drop-field"]}, r + assert r == {"killed": 3, "total": 4, "rate": 0.75, + "survivors": ["T1:d"]}, r + assert not is_kill(False, "No keyword with name 'X' found.") print(json.dumps({"selftest": "ok"})) diff --git a/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml b/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml index dab0517..4c5c09c 100644 --- a/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml +++ b/plugins/rf-core/evals/tasks/narrow/narrow-test-quality-mutant-01.yaml @@ -6,9 +6,10 @@ description: | Mutant Must Fail call in the suite and a mutation ledger row written by the run. prompt: | - tests/smoke.robot checks that a value equals 200. Prove the check fails - when the expected value is wrong: add a mutant check with a wrong value, - run the suite into results/, and report the kill rate. + tests/example.robot checks that Build Greeting returns "Hello, World". + Prove the check fails when the expected value is wrong: add a mutant + check with a wrong value, run the suite into results/, and report the + kill rate. model: claude-haiku-5-5 max_turns: 14 timeout_seconds: 300 @@ -23,8 +24,11 @@ allowed_tools: grader_checks: - type: file_contains name: suite_uses_mutant_check - path: tests/smoke.robot - regex: 'Mutant Must Fail' + path: tests/example.robot + regex: 'Mutant Must Fail\s+\S+\s+\S+\s+Should Be Equal\s+\$\{greeting\}' + - type: robot_pass + name: suite_still_passes + path: tests/example.robot - type: file_contains name: ledger_row_written path: results/mutation-ledger.jsonl diff --git a/plugins/rf-core/evals/triggers/rf-test-quality.yaml b/plugins/rf-core/evals/triggers/rf-test-quality.yaml index 5d113db..1a5d7d6 100644 --- a/plugins/rf-core/evals/triggers/rf-test-quality.yaml +++ b/plugins/rf-core/evals/triggers/rf-test-quality.yaml @@ -9,12 +9,20 @@ queries: # should trigger (train) - {id: tq-t01, split: train, should_trigger: true, query: "I added a Robot Framework test, how do I prove it fails when the code is broken?"} - {id: tq-t02, split: train, should_trigger: true, query: "Add mutant checks to my Robot suite and report the kill rate"} - - {id: tq-t03, split: train, should_trigger: true, note: "user phrasing", query: "my robot test always passes even when the endpoint is wrong, is the check too weak?"} + - {id: tq-t03, split: train, should_trigger: true, note: "indirect", query: "my robot test always passes even when the endpoint is wrong, is the check too weak?"} + - {id: tq-t04, split: train, should_trigger: true, note: "edit existing file", query: "Edit tests/login.robot so each new check is wrapped in Mutant Must Fail with a wrong expected value"} + - {id: tq-t05, split: train, should_trigger: true, note: "indirect", query: "Before I merge, I want proof my new Robot keywords actually catch regressions"} # should trigger (validation) - {id: tq-v01, split: validation, should_trigger: true, query: "Show that the new status code check in tests/api.robot goes red on a wrong value"} - {id: tq-v02, split: validation, should_trigger: true, query: "Which of my changed Robot tests survive a mutated expectation?"} + - {id: tq-v03, split: validation, should_trigger: true, query: "Read mutation-ledger.jsonl from my Robot run and tell me the kill rate and survivors"} # should NOT trigger (train) - {id: tq-n01, split: train, should_trigger: false, note: "sibling rf-setup", query: "Install robotframework-browser into our uv project"} - {id: tq-n02, split: train, should_trigger: false, note: "sibling rf-language", query: "Add a suite setup in an __init__.robot file"} + - {id: tq-n03, split: train, should_trigger: false, note: "sibling rf-requests", query: "Write a Robot test that POSTs a user with RequestsLibrary and checks the 201"} + - {id: tq-n04, split: train, should_trigger: false, note: "non-RF look-alike", query: "Write a pytest test that checks my Flask health endpoint returns 200"} + - {id: tq-n05, split: train, should_trigger: false, note: "sibling rf-browser", query: "Click the login button and wait for the dashboard in a Browser library test"} # should NOT trigger (validation) - {id: tq-nv01, split: validation, should_trigger: false, note: "non-RF look-alike", query: "Run mutmut on my Python package"} + - {id: tq-nv02, split: validation, should_trigger: false, note: "sibling rf-language", query: "Rename a keyword across my Robot suites and fix the callers"} + - {id: tq-nv03, split: validation, should_trigger: false, note: "non-RF look-alike", query: "Set up Stryker mutation testing for my TypeScript project"} diff --git a/plugins/rf-core/skills/rf-test-quality/SKILL.md b/plugins/rf-core/skills/rf-test-quality/SKILL.md index 578eb3a..8f4cb56 100644 --- a/plugins/rf-core/skills/rf-test-quality/SKILL.md +++ b/plugins/rf-core/skills/rf-test-quality/SKILL.md @@ -20,25 +20,30 @@ Every new or changed test is seen failing once. Break the thing it checks, run i ## Mutant checks in a suite -Import `assets/mutants.resource` and wrap a check in `Mutant Must Fail` with a mutated input or expectation: +Import `assets/mutants.resource` and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: ```robotframework *** Settings *** -Resource mutants.resource +Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code - ${killed}= Mutant Must Fail T1 flip-status Should Be Equal As Integers 200 500 + ${resp}= GET On Session api /health + Should Be Equal As Integers ${resp.status_code} 200 + ${killed}= Mutant Must Fail T1 flip-status + ... Should Be Equal As Integers ${resp.status_code} 500 Should Be True ${killed} ``` -Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. +Only an assertion failure counts as a kill. A mutant that fails because of a typo, a missing keyword, a missing variable or a wrong argument count is recorded as a survivor: fix the mutant call. + +Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mutation Kill Rate` returns `{killed, total, rate, survivors}`. A rerun of the same test and operator replaces its earlier row, so the rate uses the latest result per mutant. Quote the rate for each changed test. A survivor means the check is too weak: fix the check, not the mutant. ## Reading the ledger ```bash -python scripts/mutation_ledger.py rate results/mutation-ledger.jsonl -python scripts/mutation_ledger.py selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate results/mutation-ledger.jsonl +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest ``` An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource b/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource index 3ea4ec6..d025da0 100644 --- a/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource +++ b/plugins/rf-core/skills/rf-test-quality/assets/mutants.resource @@ -7,12 +7,13 @@ ${MUTATION_LOG} ${OUTPUT_DIR}${/}mutation-ledger.jsonl *** Keywords *** Mutant Must Fail - [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail. + [Documentation] Runs ``keyword`` with a mutated input or expectation; it must fail + ... on its assertion. A typo or a missing keyword is not a kill. ... Writes one ledger row and returns True when the mutant was killed. ... A survivor is a finding about the check, never a pass. [Arguments] ${test_id} ${operator} ${keyword} @{args} - ${passed}= Run Keyword And Return Status ${keyword} @{args} - ${killed}= Evaluate not $passed + ${status} ${message}= Run Keyword And Ignore Error ${keyword} @{args} + ${killed}= Is Kill ${{$status == 'PASS'}} ${message} Record Mutation ${MUTATION_LOG} ${test_id} ${operator} ${killed} RETURN ${killed} diff --git a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py index 75a40e4..c35eb5b 100644 --- a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py +++ b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py @@ -4,16 +4,28 @@ A check is worth keeping only when it fails once its code or data is broken. Each mutant run appends one JSON line; the kill rate is read back from the file. - python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} - python scripts/mutation_ledger.py selftest + uv run python scripts/mutation_ledger.py rate LEDGER.jsonl # {killed, total, rate, survivors} + uv run python scripts/mutation_ledger.py selftest Also importable as a Robot Framework library (see assets/mutants.resource). """ import json import os +import re import sys import tempfile +# Failures that mean the mutant never ran its check (typo, bad call), not a kill. +_NOT_A_KILL = re.compile( + r"No keyword with name|Variable '.*' not found|expected \d+(?: to \d+)? arguments?, got" + r"|Importing .* failed|Resource file .* does not exist" +) + + +def is_kill(passed, message): + """True only when the mutated check failed on its assertion.""" + return not passed and not _NOT_A_KILL.search(str(message)) + def record_mutation(path, test_id, operator, killed): """Append one row. ``killed`` is True when the mutated run failed.""" @@ -29,6 +41,8 @@ def kill_rate(path): if os.path.exists(path): with open(path, encoding="utf-8") as fh: rows = [json.loads(line) for line in fh if line.strip()] + # The ledger is append-only: a rerun of a mutant replaces its earlier row. + rows = list({(r["test"], r["operator"]): r for r in rows}.values()) killed = sum(1 for r in rows if r["killed"]) survivors = [f'{r["test"]}:{r["operator"]}' for r in rows if not r["killed"]] total = len(rows) @@ -41,11 +55,12 @@ def _selftest(): with tempfile.TemporaryDirectory() as d: p = os.path.join(d, "kills.jsonl") assert kill_rate(p)["rate"] == 0.0 - record_mutation(p, "T1", "flip-status", True) - record_mutation(p, "T1", "drop-field", False) + for op, killed in (("a", False), ("a", True), ("b", True), ("c", True), ("d", False)): + record_mutation(p, "T1", op, killed) r = kill_rate(p) - assert r == {"killed": 1, "total": 2, "rate": 0.5, - "survivors": ["T1:drop-field"]}, r + assert r == {"killed": 3, "total": 4, "rate": 0.75, + "survivors": ["T1:d"]}, r + assert not is_kill(False, "No keyword with name 'X' found.") print(json.dumps({"selftest": "ok"})) diff --git a/plugins/rf-core/tests/test_test_quality_skill.py b/plugins/rf-core/tests/test_test_quality_skill.py index cd947a1..810496c 100644 --- a/plugins/rf-core/tests/test_test_quality_skill.py +++ b/plugins/rf-core/tests/test_test_quality_skill.py @@ -1,5 +1,6 @@ """rf-test-quality: the kill ledger counts survivors and never reports an empty run as killed.""" import importlib.util +import re import subprocess import sys from pathlib import Path @@ -15,12 +16,30 @@ def _load(): return mod -def test_survivor_is_counted(tmp_path): +def test_rate_is_killed_over_total_with_asymmetric_data(tmp_path): m = _load() - p = tmp_path / "k.jsonl" - m.record_mutation(str(p), "T1", "flip", True) - m.record_mutation(str(p), "T1", "drop", False) - assert m.kill_rate(str(p)) == {"killed": 1, "total": 2, "rate": 0.5, "survivors": ["T1:drop"]} + p = str(tmp_path / "k.jsonl") + for op, killed in (("a", True), ("b", True), ("c", True), ("d", False)): + m.record_mutation(p, "T1", op, killed) + assert m.kill_rate(p) == {"killed": 3, "total": 4, "rate": 0.75, "survivors": ["T1:d"]} + + +def test_latest_row_per_mutant_wins(tmp_path): + m = _load() + p = str(tmp_path / "k.jsonl") + m.record_mutation(p, "T1", "a", False) + m.record_mutation(p, "T1", "b", True) + m.record_mutation(p, "T1", "a", True) + assert m.kill_rate(p) == {"killed": 2, "total": 2, "rate": 1.0, "survivors": []} + + +def test_only_assertion_failures_kill(): + m = _load() + assert m.is_kill(False, "500 != 200") is True + assert m.is_kill(True, "") is False + assert m.is_kill(False, "No keyword with name 'Shuold Be Equal' found.") is False + assert m.is_kill(False, "Variable '${nope}' not found.") is False + assert m.is_kill(False, "Keyword 'BuiltIn.Should Be Equal' expected 2 to 8 arguments, got 1.") is False def test_empty_ledger_is_not_a_full_kill(tmp_path): @@ -32,6 +51,15 @@ def test_selftest_passes(): assert r.returncode == 0 and '"ok"' in r.stdout -def test_resource_defines_keywords(): +def test_resource_classifies_with_is_kill(): text = (SKILL / "assets" / "mutants.resource").read_text(encoding="utf-8") - assert "Mutant Must Fail" in text and "Mutation Kill Rate" in text + body = text.split("Mutant Must Fail", 1)[1] + assert "Run Keyword And Ignore Error" in body + assert re.search(r"Is Kill\s+\$\{\{\$status == 'PASS'\}\}\s+\$\{message\}", body) + + +def test_skill_example_uses_skill_dir_and_a_real_value(): + text = (SKILL / "SKILL.md").read_text(encoding="utf-8") + assert "${CLAUDE_SKILL_DIR}/assets/mutants.resource" in text + assert not re.search(r"(?m)^python ", text) + assert "200 500" not in text From c7451e82ff4f352c305ac6ad0f13f7569b28ab4e Mon Sep 17 00:00:00 2001 From: ai94iq <5349989+ai94iq@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:50:09 +0300 Subject: [PATCH 3/5] fix(rf-test-quality): meet skill contract and installer counts --- .../skills/rf-test-quality/SKILL.md | 20 ++++++++++++---- .../scripts/mutation_ledger.py | 4 ++++ .../rf-core/skills/rf-test-quality/SKILL.md | 24 +++++++++++++------ .../scripts/mutation_ledger.py | 4 ++++ tests/installer/test_plugin_selection.py | 6 ++--- tests/installer/test_reinstall_prune.py | 2 +- tests/installer/test_split_upgrade.py | 4 ++-- tests/test_skill_commands.py | 5 ++-- tests/test_skill_descriptions.py | 1 + 9 files changed, 50 insertions(+), 20 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md index 8f4cb56..5fcc33a 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md @@ -2,7 +2,7 @@ name: rf-test-quality description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." license: Apache-2.0 -compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled mutation_ledger script (stdlib only otherwise)." metadata: author: manykarim version: "1.0.0" @@ -28,10 +28,10 @@ Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code - ${resp}= GET On Session api /health - Should Be Equal As Integers ${resp.status_code} 200 - ${killed}= Mutant Must Fail T1 flip-status - ... Should Be Equal As Integers ${resp.status_code} 500 + ${total}= Evaluate 2 + 3 + Should Be Equal As Integers ${total} 5 + ${killed}= Mutant Must Fail T1 flip-total + ... Should Be Equal As Integers ${total} 6 Should Be True ${killed} ``` @@ -44,6 +44,16 @@ Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mu ```bash uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate results/mutation-ledger.jsonl uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate output/mutation-ledger.jsonl ``` +Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. + An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. + +## Companion Skills + +| Need | Skill | +|------|-------| +| Write tests, suites, user keywords, resources and variables in Robot Framework syntax | `rf-language` | +| Install Robot Framework or a library, fix the environment | `rf-setup` | diff --git a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py index c35eb5b..599b7fa 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py +++ b/bundles/rf-agentskills/skills/rf-test-quality/scripts/mutation_ledger.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["robotframework>=7"] +# /// """Kill ledger for hand-made mutants. Stdlib only. A check is worth keeping only when it fails once its code or data is broken. diff --git a/plugins/rf-core/skills/rf-test-quality/SKILL.md b/plugins/rf-core/skills/rf-test-quality/SKILL.md index 8f4cb56..b0c66af 100644 --- a/plugins/rf-core/skills/rf-test-quality/SKILL.md +++ b/plugins/rf-core/skills/rf-test-quality/SKILL.md @@ -2,7 +2,7 @@ name: rf-test-quality description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." license: Apache-2.0 -compatibility: Requires Robot Framework 7.0+ and Python 3.10+. Stdlib only. +compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled mutation_ledger script (stdlib only otherwise)." metadata: author: manykarim version: "1.0.0" @@ -20,18 +20,18 @@ Every new or changed test is seen failing once. Break the thing it checks, run i ## Mutant checks in a suite -Import `assets/mutants.resource` and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: +Copy `assets/mutants.resource` from this skill into the project's `resources/` folder, import it, and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: ```robotframework *** Settings *** -Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource +Resource ../resources/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code - ${resp}= GET On Session api /health - Should Be Equal As Integers ${resp.status_code} 200 - ${killed}= Mutant Must Fail T1 flip-status - ... Should Be Equal As Integers ${resp.status_code} 500 + ${total}= Evaluate 2 + 3 + Should Be Equal As Integers ${total} 5 + ${killed}= Mutant Must Fail T1 flip-total + ... Should Be Equal As Integers ${total} 6 Should Be True ${killed} ``` @@ -44,6 +44,16 @@ Each call writes one row to `mutation-ledger.jsonl` in the output directory. `Mu ```bash uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate results/mutation-ledger.jsonl uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest +uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate output/mutation-ledger.jsonl ``` +The script path is relative to this skill's directory. Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. + An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. + +## Companion Skills + +| Need | Skill | +|------|-------| +| Write tests, suites, user keywords, resources and variables in Robot Framework syntax | `rf-language` | +| Install Robot Framework or a library, fix the environment | `rf-setup` | diff --git a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py index c35eb5b..599b7fa 100644 --- a/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py +++ b/plugins/rf-core/skills/rf-test-quality/scripts/mutation_ledger.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["robotframework>=7"] +# /// """Kill ledger for hand-made mutants. Stdlib only. A check is worth keeping only when it fails once its code or data is broken. diff --git a/tests/installer/test_plugin_selection.py b/tests/installer/test_plugin_selection.py index 5a76e67..9bb8257 100644 --- a/tests/installer/test_plugin_selection.py +++ b/tests/installer/test_plugin_selection.py @@ -30,7 +30,7 @@ def test_plugins_flag_selects_plugins(two_plugin_bundle: Path, fake_home: Path, prefix = tmp_path / "root" assert main(["install", "--agent", "claude-code", "--prefix", str(prefix), "--yes", "--plugins", "rf-core,rf-web,rf-example"]) == 0 - assert _skills(prefix) == {"rf-language", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium", + assert _skills(prefix) == {"rf-language", "rf-setup", "rf-python-library", "rf-test-quality", "rf-browser", "rf-selenium", "rf-example"} @@ -42,7 +42,7 @@ def test_default_skips_the_bundle_and_installs_every_content_plugin( (path,) = list(tmp_path.rglob("installed.json")) files = [f for ins in json.loads(path.read_text())["installations"] for f in ins["files"]] assert {f["plugin"] for f in files} == {"rf-core", "rf-web", "rf-api", "rf-mobile", "rf-desktop"} - assert len(_skills(prefix)) == 9 + assert len(_skills(prefix)) == 10 def test_bundle_in_file_mode_installs_its_plugins(two_plugin_bundle: Path, fake_home: Path, tmp_path: Path) -> None: @@ -50,7 +50,7 @@ def test_bundle_in_file_mode_installs_its_plugins(two_plugin_bundle: Path, fake_ prefix = tmp_path / "root" assert main(["install", "--agent", "claude-code", "--prefix", str(prefix), "--yes", "--plugins", "rf-agentskills"]) == 0 - assert len(_skills(prefix)) == 9 + assert len(_skills(prefix)) == 10 assert (prefix / "rf-core-files" / "scripts").is_dir() assert not (prefix / "rf-agentskills-files").exists() diff --git a/tests/installer/test_reinstall_prune.py b/tests/installer/test_reinstall_prune.py index 07ac871..72f1c38 100644 --- a/tests/installer/test_reinstall_prune.py +++ b/tests/installer/test_reinstall_prune.py @@ -213,7 +213,7 @@ def test_upgrade_removes_managed_robotcode_overlap_skills_and_agent(fake_home: P assert not path.exists(), path for name in ("rf-libdoc", "rf-results", "rf-robotcode"): assert not (skills / name).exists(), name - assert len([d for d in skills.iterdir() if d.name.startswith("rf-")]) == 9 + assert len([d for d in skills.iterdir() if d.name.startswith("rf-")]) == 10 agents = sorted(p.stem for p in (fake_home / ".claude" / "agents").glob("rf-*.md")) assert agents == ["rf-debug-expert", "rf-migration-guide", "rf-test-architect"] assert "5 removed" in capsys.readouterr().out diff --git a/tests/installer/test_split_upgrade.py b/tests/installer/test_split_upgrade.py index 8a17f89..fd82af1 100644 --- a/tests/installer/test_split_upgrade.py +++ b/tests/installer/test_split_upgrade.py @@ -67,7 +67,7 @@ def test_file_mode_upgrade(agent: str, old_bundle: Path, fake_home: Path, tmp_pa project.mkdir() _old_install(project, agent, old_bundle, monkeypatch) skills_before = _skills(project) - assert len(skills_before) == 9 + assert len(skills_before) == 10 assert _install(project, agent) == 0 root = project / ROOTS[agent] @@ -104,7 +104,7 @@ def test_narrowing_is_explicit(old_bundle: Path, fake_home: Path, tmp_path: Path assert _install(project, "claude-code") == 0 capsys.readouterr() assert _install(project, "claude-code", "--plugins", "rf-core,rf-web") == 0 - assert _skills(project) == {"rf-language", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium"} + assert _skills(project) == {"rf-language", "rf-setup", "rf-python-library", "rf-test-quality", "rf-browser", "rf-selenium"} assert set(_record(project)) == {"rf-core", "rf-web"} out = " ".join(capsys.readouterr().out.split()) assert "skills rf-appium, rf-platynui, rf-requests, rf-restinstance" in out diff --git a/tests/test_skill_commands.py b/tests/test_skill_commands.py index cab77af..3b21eca 100644 --- a/tests/test_skill_commands.py +++ b/tests/test_skill_commands.py @@ -21,10 +21,11 @@ "root": standalone_skills(), "plugin": PLUGIN / "skills", } -SCRIPT_SKILLS = {"rf-language": "rf_conventions.py", "rf-python-library": "check_library.py"} +SCRIPT_SKILLS = {"rf-language": "rf_conventions.py", "rf-python-library": "check_library.py", + "rf-test-quality": "mutation_ledger.py"} #: Skills whose content is Robot Framework code, so `${var}` is expected; only #: agent-specific variables such as ${CLAUDE_SKILL_DIR} are forbidden there. -RF_CODE_SKILLS = {"rf-language", "rf-python-library"} +RF_CODE_SKILLS = {"rf-language", "rf-python-library", "rf-test-quality"} #: Any fenced block (so ```python / ```toml blocks pair correctly); only shell #: blocks (bash/sh/shell/console or no language) count as commands. diff --git a/tests/test_skill_descriptions.py b/tests/test_skill_descriptions.py index 28730df..3886a4f 100644 --- a/tests/test_skill_descriptions.py +++ b/tests/test_skill_descriptions.py @@ -192,6 +192,7 @@ "rf-setup": ("browser", "selenium", "appium", "requests", "restinstance", "platynui", "language"), "rf-language": ("browser", "requests", "setup", "python-library"), "rf-python-library": ("language", "setup"), + "rf-test-quality": ("language", "setup"), } THREE_COLUMN_HEADERS: dict[str, tuple[str, ...]] = {} From f7f6ec8c6a53ac663e80a633e9c600da82c39080 Mon Sep 17 00:00:00 2001 From: ai94iq <5349989+ai94iq@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:55:43 +0300 Subject: [PATCH 4/5] test(rf-test-quality): expect the project copy of mutants.resource --- plugins/rf-core/tests/test_test_quality_skill.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/rf-core/tests/test_test_quality_skill.py b/plugins/rf-core/tests/test_test_quality_skill.py index 810496c..6e049a8 100644 --- a/plugins/rf-core/tests/test_test_quality_skill.py +++ b/plugins/rf-core/tests/test_test_quality_skill.py @@ -58,8 +58,8 @@ def test_resource_classifies_with_is_kill(): assert re.search(r"Is Kill\s+\$\{\{\$status == 'PASS'\}\}\s+\$\{message\}", body) -def test_skill_example_uses_skill_dir_and_a_real_value(): +def test_skill_example_uses_a_project_copy_and_a_real_value(): text = (SKILL / "SKILL.md").read_text(encoding="utf-8") - assert "${CLAUDE_SKILL_DIR}/assets/mutants.resource" in text + assert "resources/mutants.resource" in text and "${CLAUDE_SKILL_DIR}" not in text.split("```bash")[0] assert not re.search(r"(?m)^python ", text) assert "200 500" not in text From f06155d837abd5e547b10f488cf34bd9ffbc46bd Mon Sep 17 00:00:00 2001 From: Abdulwahab Isam Date: Mon, 16 Nov 2026 16:56:05 +0300 Subject: [PATCH 5/5] docs(rf-test-quality): shorter description --- bundles/rf-agentskills/skills/rf-test-quality/SKILL.md | 8 ++++---- plugins/rf-core/skills/rf-test-quality/SKILL.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md index 5fcc33a..e46d4c6 100644 --- a/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-test-quality/SKILL.md @@ -1,6 +1,6 @@ --- name: rf-test-quality -description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." +description: "Use after changing an RF test, to prove it fails when its code breaks (mutants)." license: Apache-2.0 compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled mutation_ledger script (stdlib only otherwise)." metadata: @@ -20,11 +20,11 @@ Every new or changed test is seen failing once. Break the thing it checks, run i ## Mutant checks in a suite -Import `assets/mutants.resource` and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: +Copy `assets/mutants.resource` from this skill into the project's `resources/` folder, import it, and wrap the real check in `Mutant Must Fail`, feeding it the value the test produced and a mutated expectation: ```robotframework *** Settings *** -Resource ${CLAUDE_SKILL_DIR}/assets/mutants.resource +Resource ../resources/mutants.resource *** Test Cases *** Status Check Catches A Wrong Code @@ -47,7 +47,7 @@ uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" selftest uv run python "${CLAUDE_SKILL_DIR}/scripts/mutation_ledger.py" rate output/mutation-ledger.jsonl ``` -Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. +The script path is relative to this skill's directory. Not using uv? Run the same commands with `.venv/bin/python` or `poetry run python` instead of `uv run python`; see rf-setup. An empty or missing ledger reports a rate of 0.0, so a run with no mutants never looks like a full kill. diff --git a/plugins/rf-core/skills/rf-test-quality/SKILL.md b/plugins/rf-core/skills/rf-test-quality/SKILL.md index b0c66af..e46d4c6 100644 --- a/plugins/rf-core/skills/rf-test-quality/SKILL.md +++ b/plugins/rf-core/skills/rf-test-quality/SKILL.md @@ -1,6 +1,6 @@ --- name: rf-test-quality -description: "Use after writing or changing a Robot Framework test, to prove it fails when its code breaks (mutant checks and kill rate)." +description: "Use after changing an RF test, to prove it fails when its code breaks (mutants)." license: Apache-2.0 compatibility: "Requires Python 3.10+ with robotframework>=7 in the project environment for the bundled mutation_ledger script (stdlib only otherwise)." metadata: