diff --git a/docs/adr/0001-episode-semantics-boundaries.md b/docs/adr/0001-episode-semantics-boundaries.md index 0e610c7..44074e1 100644 --- a/docs/adr/0001-episode-semantics-boundaries.md +++ b/docs/adr/0001-episode-semantics-boundaries.md @@ -136,10 +136,25 @@ Reversing the fixture input produces the same ordered oracle. Boundary tests also retain one episode at an exact 600-second cooldown gap and permit two at 601 seconds. -This accepts the hypothesis for the bounded fixture only. Isolated bursts, -maximal-window ties, shared evidence, background calibration, and a production -complexity design still require independent evidence. `Detector::analyze()` and -`loglens.report.v3` remain unchanged. +The `isolated_dense_bursts` null control uses the same threshold of five and +inclusive 600-second rule window. Its two five-event bursts have a 1,080-second +inter-segment gap. Both the replayed v0.6 baseline and the candidate emit two +episodes, preserve all ten events exactly once, report no exclusions, and keep +`continuous_segment_split` false. The candidate preserves these deterministic +finding IDs: + +- `finding:brute_force:883357c5e7697574` +- `finding:brute_force:ba242483f59f6f4e` + +Reversing the input or replacing timestamps with equivalent timezone offsets +does not change the ordered oracle. This accepts the candidate as compatible +with already-correct isolated segmentation for this bounded null control; it +does not expand the candidate's production scope. + +Together, the two fixtures accept the recovery and null-control hypotheses for +their bounded cases only. Maximal-window ties, shared evidence, background +calibration, and a production complexity design still require independent +evidence. `Detector::analyze()` and `loglens.report.v3` remain unchanged. ## Alternatives considered @@ -178,4 +193,5 @@ complexity design still require independent evidence. `Detector::analyze()` and - [`Baseline contract tests`](../../tests/test_episode_candidate_baseline_contract.py) - [`Candidate evaluator`](../../scripts/evaluate_episode_candidate.py) - [`Candidate regression tests`](../../tests/test_episode_candidate.py) -- [`Candidate oracle`](../../tests/fixtures/episode_semantics_v0.7/continuous_background_two_peaks/candidate.window-separated-v1.expected.json) +- [`Continuous-background candidate oracle`](../../tests/fixtures/episode_semantics_v0.7/continuous_background_two_peaks/candidate.window-separated-v1.expected.json) +- [`Isolated-burst candidate oracle`](../../tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/candidate.window-separated-v1.expected.json) diff --git a/tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/baseline.expected.json b/tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/baseline.expected.json new file mode 100644 index 0000000..b19da6d --- /dev/null +++ b/tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/baseline.expected.json @@ -0,0 +1,142 @@ +{ + "format": "loglens.episode_baseline_expected.v1", + "fixture_id": "episode_semantics_v0.7.isolated_dense_bursts", + "algorithm": { + "id": "v0.6.activity_segments_best_count_window", + "implementation": "src/detector.cpp", + "segment_boundary": "split when an adjacent event gap is greater than the rule window", + "selection_tie_break": "keep the first maximum encountered in chronological scan order" + }, + "rule": { + "rule_id": "brute_force", + "grouping_key": "source_ip", + "subject": "203.0.113.78", + "threshold": 5, + "window_seconds": 600, + "window_boundary": "inclusive" + }, + "derived_input": { + "event_count": 10, + "dense_peak_count": 2, + "activity_segment_count": 2, + "activity_segments": [ + { + "segment_id": "segment:1", + "event_ids": [ + "line:1", + "line:2", + "line:3", + "line:4", + "line:5" + ], + "first_seen": "2026-03-11T09:00:00Z", + "last_seen": "2026-03-11T09:02:00Z", + "max_adjacent_gap_seconds": 30, + "boundary_observation": "Five dense events remain inside one inclusive 600-second activity segment." + }, + { + "segment_id": "segment:2", + "event_ids": [ + "line:6", + "line:7", + "line:8", + "line:9", + "line:10" + ], + "first_seen": "2026-03-11T09:20:00Z", + "last_seen": "2026-03-11T09:22:00Z", + "max_adjacent_gap_seconds": 30, + "boundary_observation": "The 1,080-second gap after segment 1 exceeds the rule window, so v0.6 starts a second activity segment." + } + ] + }, + "candidate_windows": [ + { + "candidate_id": "baseline:burst-a", + "threshold_crossing_event_id": "line:5", + "event_ids": [ + "line:1", + "line:2", + "line:3", + "line:4", + "line:5" + ], + "first_seen": "2026-03-11T09:00:00Z", + "last_seen": "2026-03-11T09:02:00Z", + "event_count": 5, + "threshold_met": true, + "decision": "selected", + "decision_reason": "Only threshold-eligible window in the first activity segment." + }, + { + "candidate_id": "baseline:burst-b", + "threshold_crossing_event_id": "line:10", + "event_ids": [ + "line:6", + "line:7", + "line:8", + "line:9", + "line:10" + ], + "first_seen": "2026-03-11T09:20:00Z", + "last_seen": "2026-03-11T09:22:00Z", + "event_count": 5, + "threshold_met": true, + "decision": "selected", + "decision_reason": "Only threshold-eligible window in the second activity segment." + } + ], + "expected_output": { + "episode_count": 2, + "findings": [ + { + "finding_id": "finding:brute_force:883357c5e7697574", + "episode_index": 1, + "rule_id": "brute_force", + "subject_kind": "source_ip", + "subject": "203.0.113.78", + "grouping_key": "source_ip", + "threshold": 5, + "observed_count": 5, + "event_count": 5, + "window_start": "2026-03-11 09:00:00", + "window_end": "2026-03-11 09:02:00", + "evidence_event_ids": [ + "line:1", + "line:2", + "line:3", + "line:4", + "line:5" + ], + "verdict_boundary": "triage_signal_not_compromise_or_attribution" + }, + { + "finding_id": "finding:brute_force:ba242483f59f6f4e", + "episode_index": 2, + "rule_id": "brute_force", + "subject_kind": "source_ip", + "subject": "203.0.113.78", + "grouping_key": "source_ip", + "threshold": 5, + "observed_count": 5, + "event_count": 5, + "window_start": "2026-03-11 09:20:00", + "window_end": "2026-03-11 09:22:00", + "evidence_event_ids": [ + "line:6", + "line:7", + "line:8", + "line:9", + "line:10" + ], + "verdict_boundary": "triage_signal_not_compromise_or_attribution" + } + ], + "excluded_event_decisions": [] + }, + "research_observation": { + "baseline_already_separates_dense_bursts": true, + "candidate_should_preserve_episode_count_and_identity": true, + "not_a_v0.6_regression": true + } +} diff --git a/tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/candidate.window-separated-v1.expected.json b/tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/candidate.window-separated-v1.expected.json new file mode 100644 index 0000000..0b50b31 --- /dev/null +++ b/tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/candidate.window-separated-v1.expected.json @@ -0,0 +1,245 @@ +{ + "format": "loglens.episode_candidate_oracle.v1", + "fixture_id": "episode_semantics_v0.7.isolated_dense_bursts", + "algorithm": { + "id": "research.window_separated_weighted_intervals", + "version": "1", + "status": "candidate" + }, + "rule": { + "rule_id": "brute_force", + "grouping_key": "source_ip", + "subject": "203.0.113.78", + "threshold": 5, + "window_seconds": 600, + "window_boundary": "inclusive" + }, + "segments": [ + { + "segment_id": "segment:1", + "event_ids": [ + "line:1", + "line:2", + "line:3", + "line:4", + "line:5" + ], + "first_seen": "2026-03-11T09:00:00Z", + "last_seen": "2026-03-11T09:02:00Z", + "candidate_windows": [ + { + "candidate_id": "candidate:line:1..line:5", + "event_ids": [ + "line:1", + "line:2", + "line:3", + "line:4", + "line:5" + ], + "first_seen": "2026-03-11T09:00:00Z", + "last_seen": "2026-03-11T09:02:00Z", + "threshold_crossing_event_ids": [ + "line:5" + ], + "overlap_event_ids": [], + "score": { + "metric": "event_count_then_compactness", + "value": 5, + "details": { + "event_count": 5, + "span_seconds": 120 + } + }, + "decision": { + "selected": true, + "reason_code": "selected", + "reason": "Selected by maximum covered events, then minimum total span and episode count." + } + } + ], + "selected_episodes": [ + { + "episode_index": 1, + "finding_id": "finding:brute_force:883357c5e7697574", + "candidate_id": "candidate:line:1..line:5", + "event_ids": [ + "line:1", + "line:2", + "line:3", + "line:4", + "line:5" + ], + "first_seen": "2026-03-11T09:00:00Z", + "last_seen": "2026-03-11T09:02:00Z", + "inclusion_reason": "Window is part of the optimal window-separated evidence set." + } + ], + "event_decisions": [ + { + "event_id": "line:1", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:1..line:5" + ], + "reason": "Included exactly once by a selected candidate window." + }, + { + "event_id": "line:2", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:1..line:5" + ], + "reason": "Included exactly once by a selected candidate window." + }, + { + "event_id": "line:3", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:1..line:5" + ], + "reason": "Included exactly once by a selected candidate window." + }, + { + "event_id": "line:4", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:1..line:5" + ], + "reason": "Included exactly once by a selected candidate window." + }, + { + "event_id": "line:5", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:1..line:5" + ], + "reason": "Included exactly once by a selected candidate window." + } + ] + }, + { + "segment_id": "segment:2", + "event_ids": [ + "line:6", + "line:7", + "line:8", + "line:9", + "line:10" + ], + "first_seen": "2026-03-11T09:20:00Z", + "last_seen": "2026-03-11T09:22:00Z", + "candidate_windows": [ + { + "candidate_id": "candidate:line:6..line:10", + "event_ids": [ + "line:6", + "line:7", + "line:8", + "line:9", + "line:10" + ], + "first_seen": "2026-03-11T09:20:00Z", + "last_seen": "2026-03-11T09:22:00Z", + "threshold_crossing_event_ids": [ + "line:10" + ], + "overlap_event_ids": [], + "score": { + "metric": "event_count_then_compactness", + "value": 5, + "details": { + "event_count": 5, + "span_seconds": 120 + } + }, + "decision": { + "selected": true, + "reason_code": "selected", + "reason": "Selected by maximum covered events, then minimum total span and episode count." + } + } + ], + "selected_episodes": [ + { + "episode_index": 2, + "finding_id": "finding:brute_force:ba242483f59f6f4e", + "candidate_id": "candidate:line:6..line:10", + "event_ids": [ + "line:6", + "line:7", + "line:8", + "line:9", + "line:10" + ], + "first_seen": "2026-03-11T09:20:00Z", + "last_seen": "2026-03-11T09:22:00Z", + "inclusion_reason": "Window is part of the optimal window-separated evidence set." + } + ], + "event_decisions": [ + { + "event_id": "line:6", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:6..line:10" + ], + "reason": "Included exactly once by a selected candidate window." + }, + { + "event_id": "line:7", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:6..line:10" + ], + "reason": "Included exactly once by a selected candidate window." + }, + { + "event_id": "line:8", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:6..line:10" + ], + "reason": "Included exactly once by a selected candidate window." + }, + { + "event_id": "line:9", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:6..line:10" + ], + "reason": "Included exactly once by a selected candidate window." + }, + { + "event_id": "line:10", + "decision": "included", + "reason_code": "selected_window", + "candidate_ids": [ + "candidate:line:6..line:10" + ], + "reason": "Included exactly once by a selected candidate window." + } + ] + } + ], + "comparison": { + "baseline_reference": "baseline.expected.json", + "baseline_episode_count": 2, + "candidate_episode_count": 2, + "continuous_segment_split": false, + "notes": [ + "Research-only candidate; Detector::analyze() and loglens.report.v3 are unchanged.", + "Selected windows require a gap strictly greater than one rule window; exact-boundary gaps remain one cooldown episode.", + "Selection maximizes covered evidence count, then minimizes total span and episode count, with a chronological final tie-break.", + "Exhaustive candidate materialization and overlap reporting have super-quadratic worst-case cost; hard limits keep this fixture tool bounded, not production-ready." + ] + } +} diff --git a/tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/fixture.json b/tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/fixture.json new file mode 100644 index 0000000..9508512 --- /dev/null +++ b/tests/fixtures/episode_semantics_v0.7/isolated_dense_bursts/fixture.json @@ -0,0 +1,100 @@ +{ + "format": "loglens.episode_research_fixture.v1", + "fixture_id": "episode_semantics_v0.7.isolated_dense_bursts", + "fixture_kind": "normalized_detector_signals", + "status": "research_only", + "purpose": "Verify that candidate v1 preserves v0.6 episodes when two dense bursts are already separated by more than one rule window.", + "safety_boundary": "Synthetic triage signals do not assert compromise, intent, attribution, authorization, or an incident boundary.", + "rule": { + "rule_id": "brute_force", + "grouping_key": "source_ip", + "subject": "203.0.113.78", + "threshold": 5, + "window_seconds": 600, + "window_boundary": "inclusive", + "signal_kind": "ssh_failed_password", + "counts_as_terminal_auth_failure": true + }, + "events": [ + { + "event_id": "line:1", + "line_number": 1, + "timestamp": "2026-03-11T09:00:00Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_a" + }, + { + "event_id": "line:2", + "line_number": 2, + "timestamp": "2026-03-11T09:00:30Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_a" + }, + { + "event_id": "line:3", + "line_number": 3, + "timestamp": "2026-03-11T09:01:00Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_a" + }, + { + "event_id": "line:4", + "line_number": 4, + "timestamp": "2026-03-11T09:01:30Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_a" + }, + { + "event_id": "line:5", + "line_number": 5, + "timestamp": "2026-03-11T09:02:00Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_a" + }, + { + "event_id": "line:6", + "line_number": 6, + "timestamp": "2026-03-11T09:20:00Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_b" + }, + { + "event_id": "line:7", + "line_number": 7, + "timestamp": "2026-03-11T09:20:30Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_b" + }, + { + "event_id": "line:8", + "line_number": 8, + "timestamp": "2026-03-11T09:21:00Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_b" + }, + { + "event_id": "line:9", + "line_number": 9, + "timestamp": "2026-03-11T09:21:30Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_b" + }, + { + "event_id": "line:10", + "line_number": 10, + "timestamp": "2026-03-11T09:22:00Z", + "event_type": "ssh_failed_password", + "source_ip": "203.0.113.78", + "role": "isolated_burst_b" + } + ] +} diff --git a/tests/test_episode_candidate.py b/tests/test_episode_candidate.py index 43197cb..24e1f22 100644 --- a/tests/test_episode_candidate.py +++ b/tests/test_episode_candidate.py @@ -23,6 +23,13 @@ / "episode_semantics_v0.7" / "continuous_background_two_peaks" ) +ISOLATED_FIXTURE_ROOT = ( + REPO_ROOT + / "tests" + / "fixtures" + / "episode_semantics_v0.7" + / "isolated_dense_bursts" +) class ContinuousBackgroundFixtureTests(unittest.TestCase): @@ -122,5 +129,73 @@ def test_equivalent_timezone_offsets_produce_the_same_oracle(self) -> None: evaluate_fixture(self.fixture, self.baseline), ) + +class IsolatedDenseBurstsFixtureTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = json.loads( + (ISOLATED_FIXTURE_ROOT / "fixture.json").read_text(encoding="utf-8") + ) + self.baseline = json.loads( + (ISOLATED_FIXTURE_ROOT / "baseline.expected.json").read_text( + encoding="utf-8" + ) + ) + + def test_candidate_preserves_already_separated_baseline(self) -> None: + oracle = evaluate_fixture(self.fixture, self.baseline) + baseline_finding_ids = [ + finding["finding_id"] + for finding in self.baseline["expected_output"]["findings"] + ] + candidate_finding_ids = [ + episode["finding_id"] + for segment in oracle["segments"] + for episode in segment["selected_episodes"] + ] + included_event_ids = [ + decision["event_id"] + for segment in oracle["segments"] + for decision in segment["event_decisions"] + if decision["decision"] == "included" + ] + + self.assertEqual(oracle["comparison"]["baseline_episode_count"], 2) + self.assertEqual(oracle["comparison"]["candidate_episode_count"], 2) + self.assertFalse(oracle["comparison"]["continuous_segment_split"]) + self.assertEqual(candidate_finding_ids, baseline_finding_ids) + self.assertEqual(included_event_ids, [f"line:{i}" for i in range(1, 11)]) + self.assertEqual(len(included_event_ids), len(set(included_event_ids))) + + def test_input_representation_does_not_change_control_oracle(self) -> None: + expected = evaluate_fixture(self.fixture, self.baseline) + variants = { + "reversed": copy.deepcopy(self.fixture), + "equivalent timezone": copy.deepcopy(self.fixture), + } + variants["reversed"]["events"].reverse() + offset = timezone(timedelta(hours=-5)) + for event in variants["equivalent timezone"]["events"]: + event["timestamp"] = parse_timestamp(event["timestamp"]).astimezone( + offset + ).isoformat() + + for label, fixture in variants.items(): + with self.subTest(label=label): + self.assertEqual(evaluate_fixture(fixture, self.baseline), expected) + + def test_generated_oracle_matches_committed_control_and_cross_references( + self, + ) -> None: + oracle = evaluate_fixture(self.fixture, self.baseline) + expected = json.loads( + ( + ISOLATED_FIXTURE_ROOT + / "candidate.window-separated-v1.expected.json" + ).read_text(encoding="utf-8") + ) + + validate_oracle(self.fixture, self.baseline, oracle) + self.assertEqual(oracle, expected) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_episode_candidate_validation.py b/tests/test_episode_candidate_validation.py index f78ba5a..6a8ed0d 100644 --- a/tests/test_episode_candidate_validation.py +++ b/tests/test_episode_candidate_validation.py @@ -31,6 +31,14 @@ / "episode_semantics_v0.7" / "continuous_background_two_peaks" ) +CANDIDATE_ORACLE_ROOTS = ( + FIXTURE_ROOT, + REPO_ROOT + / "tests" + / "fixtures" + / "episode_semantics_v0.7" + / "isolated_dense_bursts", +) class CandidateOracleValidationTests(unittest.TestCase): @@ -46,19 +54,20 @@ def setUp(self) -> None: Draft202012Validator is None, "install requirements-test.txt to validate the JSON Schema", ) - def test_committed_oracle_conforms_to_draft_2020_12_schema(self) -> None: + def test_committed_oracles_conform_to_draft_2020_12_schema(self) -> None: schema = json.loads( (FIXTURE_ROOT / "candidate-oracle.schema.json").read_text(encoding="utf-8") ) - oracle = json.loads( - (FIXTURE_ROOT / "candidate.window-separated-v1.expected.json").read_text( - encoding="utf-8" - ) - ) - Draft202012Validator.check_schema(schema) validator = Draft202012Validator(schema, format_checker=FormatChecker()) - self.assertEqual(list(validator.iter_errors(oracle)), []) + for root in CANDIDATE_ORACLE_ROOTS: + with self.subTest(fixture=root.name): + oracle = json.loads( + (root / "candidate.window-separated-v1.expected.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(list(validator.iter_errors(oracle)), []) def test_validator_rejects_episode_evidence_drift(self) -> None: oracle = evaluate_fixture(self.fixture, self.baseline)