From 413b165127985ba388d42cf5f0b5f280c3613014 Mon Sep 17 00:00:00 2001 From: stacknil Date: Tue, 25 Aug 2026 11:17:06 +0800 Subject: [PATCH 1/3] test(research): harden candidate oracle validation --- tests/test_episode_candidate_validation.py | 116 +++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 tests/test_episode_candidate_validation.py diff --git a/tests/test_episode_candidate_validation.py b/tests/test_episode_candidate_validation.py new file mode 100644 index 0000000..4f7fda6 --- /dev/null +++ b/tests/test_episode_candidate_validation.py @@ -0,0 +1,116 @@ +import copy +import json +import sys +import tempfile +import unittest +from contextlib import redirect_stderr +from io import StringIO +from pathlib import Path + +try: + from jsonschema import Draft202012Validator, FormatChecker +except ModuleNotFoundError: # The evaluator itself remains standard-library only. + Draft202012Validator = None + FormatChecker = None + + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) + +from scripts.evaluate_episode_candidate import ( # noqa: E402 + evaluate_fixture, + main, + validate_oracle, +) + + +FIXTURE_ROOT = ( + REPO_ROOT + / "tests" + / "fixtures" + / "episode_semantics_v0.7" + / "continuous_background_two_peaks" +) + + +class CandidateOracleValidationTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = json.loads( + (FIXTURE_ROOT / "fixture.json").read_text(encoding="utf-8") + ) + self.baseline = json.loads( + (FIXTURE_ROOT / "baseline.expected.json").read_text(encoding="utf-8") + ) + + @unittest.skipIf( + Draft202012Validator is None, + "install requirements-test.txt to validate the JSON Schema", + ) + def test_committed_oracle_conforms_to_draft_2020_12_schema(self) -> None: + schema = json.loads( + (FIXTURE_ROOT / "candidate-oracle.schema.json").read_text(encoding="utf-8") + ) + oracle = json.loads( + (FIXTURE_ROOT / "candidate.window-separated-v1.expected.json").read_text( + encoding="utf-8" + ) + ) + + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema, format_checker=FormatChecker()) + self.assertEqual(list(validator.iter_errors(oracle)), []) + + def test_validator_rejects_episode_evidence_drift(self) -> None: + oracle = evaluate_fixture(self.fixture, self.baseline) + oracle["segments"][0]["selected_episodes"][0]["event_ids"].pop() + + with self.assertRaisesRegex(ValueError, "episode evidence"): + validate_oracle(self.fixture, oracle) + + def test_validator_rejects_selected_candidate_without_episode(self) -> None: + oracle = evaluate_fixture(self.fixture, self.baseline) + oracle["segments"][0]["selected_episodes"].pop() + + with self.assertRaisesRegex(ValueError, "selected candidates"): + validate_oracle(self.fixture, oracle) + + def test_validator_rejects_event_candidate_reference_drift(self) -> None: + oracle = evaluate_fixture(self.fixture, self.baseline) + oracle["segments"][0]["event_decisions"][0]["candidate_ids"] = [] + + with self.assertRaisesRegex(ValueError, "candidate_ids"): + validate_oracle(self.fixture, oracle) + + def test_input_pair_fails_closed_on_boundary_or_baseline_drift(self) -> None: + exclusive = copy.deepcopy(self.fixture) + exclusive["rule"]["window_boundary"] = "exclusive" + with self.assertRaisesRegex(ValueError, "only inclusive"): + evaluate_fixture(exclusive, self.baseline) + + stale_baseline = copy.deepcopy(self.baseline) + stale_baseline["fixture_id"] = "another-fixture" + with self.assertRaisesRegex(ValueError, "fixture_id"): + evaluate_fixture(self.fixture, stale_baseline) + + def test_cli_returns_two_without_disclosing_invalid_input_path(self) -> None: + with tempfile.TemporaryDirectory() as directory: + malformed = Path(directory) / "fixture.json" + malformed.write_text("{", encoding="utf-8") + stderr = StringIO() + with redirect_stderr(stderr): + status = main( + [ + "--fixture", + str(malformed), + "--baseline", + str(FIXTURE_ROOT / "baseline.expected.json"), + ] + ) + + self.assertEqual(status, 2) + self.assertIn("invalid JSON", stderr.getvalue()) + self.assertNotIn(directory, stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main() From a7dec98249553c7053eaa09df646b68cb4fab882 Mon Sep 17 00:00:00 2001 From: stacknil Date: Tue, 25 Aug 2026 11:17:07 +0800 Subject: [PATCH 2/3] ci(research): require candidate schema validation --- .github/workflows/ci.yml | 3 +++ requirements-test.txt | 1 + 2 files changed, 4 insertions(+) create mode 100644 requirements-test.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beab655..4c73ab3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,9 @@ jobs: with: python-version: "3.14" + - name: Install Python test dependencies + run: python -m pip install --disable-pip-version-check -r requirements-test.txt + - name: Configure run: >- cmake -S . -B build diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..3ee1ea1 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1 @@ +jsonschema==4.26.0 From 056d65fc2c70a631613fa81c4daa1af81d0687d1 Mon Sep 17 00:00:00 2001 From: stacknil Date: Wed, 26 Aug 2026 10:14:14 +0800 Subject: [PATCH 3/3] test(research): adapt validation to baseline contract --- tests/test_episode_candidate_validation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_episode_candidate_validation.py b/tests/test_episode_candidate_validation.py index 4f7fda6..f78ba5a 100644 --- a/tests/test_episode_candidate_validation.py +++ b/tests/test_episode_candidate_validation.py @@ -65,26 +65,26 @@ def test_validator_rejects_episode_evidence_drift(self) -> None: oracle["segments"][0]["selected_episodes"][0]["event_ids"].pop() with self.assertRaisesRegex(ValueError, "episode evidence"): - validate_oracle(self.fixture, oracle) + validate_oracle(self.fixture, self.baseline, oracle) def test_validator_rejects_selected_candidate_without_episode(self) -> None: oracle = evaluate_fixture(self.fixture, self.baseline) oracle["segments"][0]["selected_episodes"].pop() with self.assertRaisesRegex(ValueError, "selected candidates"): - validate_oracle(self.fixture, oracle) + validate_oracle(self.fixture, self.baseline, oracle) def test_validator_rejects_event_candidate_reference_drift(self) -> None: oracle = evaluate_fixture(self.fixture, self.baseline) oracle["segments"][0]["event_decisions"][0]["candidate_ids"] = [] with self.assertRaisesRegex(ValueError, "candidate_ids"): - validate_oracle(self.fixture, oracle) + validate_oracle(self.fixture, self.baseline, oracle) def test_input_pair_fails_closed_on_boundary_or_baseline_drift(self) -> None: exclusive = copy.deepcopy(self.fixture) exclusive["rule"]["window_boundary"] = "exclusive" - with self.assertRaisesRegex(ValueError, "only inclusive"): + with self.assertRaisesRegex(ValueError, "candidate v1 supports only"): evaluate_fixture(exclusive, self.baseline) stale_baseline = copy.deepcopy(self.baseline)