From 8e4807edb8906b07509503df338d7ea5bb18b069 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 06:58:14 +0000 Subject: [PATCH 01/15] test(recovery): add offline restore timing evidence and regression coverage --- scripts/measure_restore_drill.py | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 scripts/measure_restore_drill.py diff --git a/scripts/measure_restore_drill.py b/scripts/measure_restore_drill.py new file mode 100644 index 00000000..3edfb59d --- /dev/null +++ b/scripts/measure_restore_drill.py @@ -0,0 +1,58 @@ +"""Offline timing evidence for an operator-controlled restore drill. + +This reports observed intervals only; it never performs a restore or network request. +""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone + + +def parse_timestamp(value: str) -> datetime: + """Require an explicitly timezone-aware ISO-8601 timestamp.""" + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("timestamp must be ISO-8601 with timezone") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError("timestamp must include a timezone") + return parsed.astimezone(timezone.utc) + + +def measure(last_durable: str, failure: str, ready: str) -> dict[str, object]: + """Calculate observed RPO/RTO proxies from operator-supplied UTC instants.""" + durable_at = parse_timestamp(last_durable) + failure_at = parse_timestamp(failure) + ready_at = parse_timestamp(ready) + if durable_at > failure_at: + raise ValueError("last durable evidence cannot follow failure") + if ready_at < failure_at: + raise ValueError("readiness cannot precede failure") + return { + "last_durable_at": durable_at.isoformat(), + "failure_at": failure_at.isoformat(), + "ready_at": ready_at.isoformat(), + "observed_rpo_seconds": (failure_at - durable_at).total_seconds(), + "observed_rto_seconds": (ready_at - failure_at).total_seconds(), + "scope": "operator-supplied isolated drill timestamps; not production SLA", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--last-durable", required=True) + parser.add_argument("--failure", required=True) + parser.add_argument("--ready", required=True) + args = parser.parse_args() + try: + report = measure(args.last_durable, args.failure, args.ready) + except ValueError as exc: + parser.error(str(exc)) + print(json.dumps(report, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From bf2e3393abcaba878f539f5e902474e72b0854cc Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 06:58:16 +0000 Subject: [PATCH 02/15] test(recovery): add offline restore timing evidence and regression coverage --- tests/test_restore_drill_measurement.py | 41 +++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/test_restore_drill_measurement.py diff --git a/tests/test_restore_drill_measurement.py b/tests/test_restore_drill_measurement.py new file mode 100644 index 00000000..a7098ed8 --- /dev/null +++ b/tests/test_restore_drill_measurement.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import pytest + +from scripts.measure_restore_drill import measure, parse_timestamp + + +def test_measurement_uses_actual_elapsed_seconds_and_normalizes_offsets(): + result = measure( + "2026-11-15T10:00:00+01:00", + "2026-11-15T09:03:00Z", + "2026-11-15T09:05:30+00:00", + ) + assert result["observed_rpo_seconds"] == 180.0 + assert result["observed_rto_seconds"] == 150.0 + assert result["last_durable_at"] == "2026-11-15T09:00:00+00:00" + assert "not production SLA" in str(result["scope"]) + + +@pytest.mark.parametrize( + "value", + ["2026-11-15T09:00:00", "not-a-time", "2026-13-15T09:00:00Z"], +) +def test_rejects_naive_or_invalid_timestamps(value): + with pytest.raises(ValueError, match="timestamp"): + parse_timestamp(value) + + +def test_rejects_inconsistent_evidence_order(): + with pytest.raises(ValueError, match="durable"): + measure( + "2026-11-15T09:04:00Z", + "2026-11-15T09:03:00Z", + "2026-11-15T09:05:00Z", + ) + with pytest.raises(ValueError, match="readiness"): + measure( + "2026-11-15T09:00:00Z", + "2026-11-15T09:03:00Z", + "2026-11-15T09:02:00Z", + ) From 306651467add44cd8e5e355a723accaf1b4985a4 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 06:58:18 +0000 Subject: [PATCH 03/15] test(recovery): add offline restore timing evidence and regression coverage --- docs/restore-drill-timing-evidence.md | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docs/restore-drill-timing-evidence.md diff --git a/docs/restore-drill-timing-evidence.md b/docs/restore-drill-timing-evidence.md new file mode 100644 index 00000000..0d3b1355 --- /dev/null +++ b/docs/restore-drill-timing-evidence.md @@ -0,0 +1,34 @@ +# Isolated restore drill: timing evidence + +This **offline, read-only** helper calculates two observed intervals from +operator-recorded timestamps. It does not create, verify, or restore backups, +contact an API, or assert that an application is healthy. + +Run only after an authorized isolated staging restore with synthetic data: + +```bash +python scripts/measure_restore_drill.py \ + --last-durable 2026-11-15T09:00:00Z \ + --failure 2026-11-15T09:03:00Z \ + --ready 2026-11-15T09:05:30Z +``` + +Record the source of each timestamp alongside the JSON output: + +- **last-durable:** newest independently verified, restorable evidence + recovered in staging (not merely the most recent backup attempt). +- **failure:** simulated failure start, from the drill operator's clock. +- **ready:** time authenticated readiness, inventory access, and audit + integrity were **actually confirmed** after the isolated restore. + +The helper rejects missing timezone offsets and impossible event ordering. +Its observed RPO proxy is failure minus last durable evidence; observed RTO +proxy is ready minus failure. These are *drill observations*, not guaranteed +production RPO/RTO, a measured data-loss count, or proof of recovery. + +For acceptance, separately retain backup hash/verification results, schema +version, authenticated API checks, audit-chain evidence, operator sign-off, +and clock synchronization details. Do not include credentials, personal +inventory, production databases, or secrets in the evidence bundle. +See [Track A restore drill](track-a-restore-drill.md) for a historical local +drill; its schema version and timing are not a current production baseline. From b59dc0c7697ee9970884a7bbefcd4e59b9d2ae53 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 09:56:54 +0000 Subject: [PATCH 04/15] test(recovery): cover offline drill CLI JSON and fail-closed errors --- tests/test_restore_drill_cli_contract.py | 65 ++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 tests/test_restore_drill_cli_contract.py diff --git a/tests/test_restore_drill_cli_contract.py b/tests/test_restore_drill_cli_contract.py new file mode 100644 index 00000000..e10b1299 --- /dev/null +++ b/tests/test_restore_drill_cli_contract.py @@ -0,0 +1,65 @@ +"""CLI contract checks for the offline restore timing evidence helper.""" + +from __future__ import annotations + +import json +import sys + +import pytest + +from scripts.measure_restore_drill import main + + +def test_cli_emits_machine_readable_observations(monkeypatch, capsys): + monkeypatch.setattr( + sys, + "argv", + [ + "measure_restore_drill.py", + "--last-durable", + "2026-11-15T09:00:00Z", + "--failure", + "2026-11-15T09:03:00Z", + "--ready", + "2026-11-15T09:05:30Z", + ], + ) + assert main() == 0 + captured = capsys.readouterr() + assert captured.err == "" + report = json.loads(captured.out) + assert report["observed_rpo_seconds"] == 180.0 + assert report["observed_rto_seconds"] == 150.0 + assert "not production SLA" in report["scope"] + + +@pytest.mark.parametrize( + "last_durable,failure,ready", + [ + ("2026-11-15T09:04:00Z", "2026-11-15T09:03:00Z", "2026-11-15T09:05:00Z"), + ("2026-11-15T09:00:00Z", "2026-11-15T09:03:00Z", "2026-11-15T09:02:00Z"), + ("2026-11-15T09:00:00", "2026-11-15T09:03:00Z", "2026-11-15T09:05:00Z"), + ], +) +def test_cli_rejects_invalid_evidence_without_json( + monkeypatch, capsys, last_durable, failure, ready +): + monkeypatch.setattr( + sys, + "argv", + [ + "measure_restore_drill.py", + "--last-durable", + last_durable, + "--failure", + failure, + "--ready", + ready, + ], + ) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "error:" in captured.err From c4c01be89d02147ca64e6d115187b75b890b6e37 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 13:01:32 +0000 Subject: [PATCH 05/15] fix(recovery): reject timezone normalization overflow as invalid evidence --- scripts/measure_restore_drill.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/measure_restore_drill.py b/scripts/measure_restore_drill.py index 3edfb59d..94b8c1dd 100644 --- a/scripts/measure_restore_drill.py +++ b/scripts/measure_restore_drill.py @@ -18,7 +18,10 @@ def parse_timestamp(value: str) -> datetime: raise ValueError("timestamp must be ISO-8601 with timezone") from exc if parsed.tzinfo is None or parsed.utcoffset() is None: raise ValueError("timestamp must include a timezone") - return parsed.astimezone(timezone.utc) + try: + return parsed.astimezone(timezone.utc) + except (OverflowError, ValueError) as exc: + raise ValueError("timestamp outside supported UTC range") from exc def measure(last_durable: str, failure: str, ready: str) -> dict[str, object]: From 5d4c4caae3d58aab3e04e760441b36856db6db89 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 13:01:38 +0000 Subject: [PATCH 06/15] test(recovery): cover minimum and maximum UTC overflow boundaries --- tests/test_restore_drill_measurement.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_restore_drill_measurement.py b/tests/test_restore_drill_measurement.py index a7098ed8..4564697b 100644 --- a/tests/test_restore_drill_measurement.py +++ b/tests/test_restore_drill_measurement.py @@ -39,3 +39,15 @@ def test_rejects_inconsistent_evidence_order(): "2026-11-15T09:03:00Z", "2026-11-15T09:02:00Z", ) + + +@pytest.mark.parametrize( + "value", + [ + "0001-01-01T00:00:00+01:00", + "9999-12-31T23:59:59-01:00", + ], +) +def test_rejects_utc_normalization_overflow(value): + with pytest.raises(ValueError, match="supported UTC range"): + parse_timestamp(value) From f07d53256192e2dbd6e7e89d6436849dc7318579 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 13:01:43 +0000 Subject: [PATCH 07/15] test(recovery): require clean CLI error for out-of-range UTC instants --- tests/test_restore_drill_cli_contract.py | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_restore_drill_cli_contract.py b/tests/test_restore_drill_cli_contract.py index e10b1299..9cc31cc2 100644 --- a/tests/test_restore_drill_cli_contract.py +++ b/tests/test_restore_drill_cli_contract.py @@ -63,3 +63,30 @@ def test_cli_rejects_invalid_evidence_without_json( captured = capsys.readouterr() assert captured.out == "" assert "error:" in captured.err + + +@pytest.mark.parametrize( + "last_durable", + ["0001-01-01T00:00:00+01:00", "9999-12-31T23:59:59-01:00"], +) +def test_cli_reports_utc_overflow_without_traceback(monkeypatch, capsys, last_durable): + monkeypatch.setattr( + sys, + "argv", + [ + "measure_restore_drill.py", + "--last-durable", + last_durable, + "--failure", + "2026-11-15T09:03:00Z", + "--ready", + "2026-11-15T09:05:00Z", + ], + ) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "supported UTC range" in captured.err + assert "Traceback" not in captured.err From 732a94b7a650e857485e46f09303d00f8c64101a Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 13:01:50 +0000 Subject: [PATCH 08/15] docs(recovery): document fail-closed UTC boundary handling --- docs/restore-drill-timing-evidence.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/restore-drill-timing-evidence.md b/docs/restore-drill-timing-evidence.md index 0d3b1355..7cfc19cd 100644 --- a/docs/restore-drill-timing-evidence.md +++ b/docs/restore-drill-timing-evidence.md @@ -21,7 +21,9 @@ Record the source of each timestamp alongside the JSON output: - **ready:** time authenticated readiness, inventory access, and audit integrity were **actually confirmed** after the isolated restore. -The helper rejects missing timezone offsets and impossible event ordering. +The helper rejects missing timezone offsets, UTC normalization overflow at the +minimum/maximum supported calendar dates, and impossible event ordering. +Invalid inputs return CLI exit code 2 with no JSON evidence output. Its observed RPO proxy is failure minus last durable evidence; observed RTO proxy is ready minus failure. These are *drill observations*, not guaranteed production RPO/RTO, a measured data-loss count, or proof of recovery. From 0ca53cb2307cdef35ae9c884f3f6d0664dbcbbf8 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 14:58:08 +0000 Subject: [PATCH 09/15] fix(recovery): reject ambiguous restore evidence timestamp formats --- scripts/measure_restore_drill.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/measure_restore_drill.py b/scripts/measure_restore_drill.py index 94b8c1dd..61753682 100644 --- a/scripts/measure_restore_drill.py +++ b/scripts/measure_restore_drill.py @@ -7,11 +7,21 @@ import argparse import json +import re from datetime import datetime, timezone +_TIMESTAMP_PATTERN = re.compile( + r"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,6})?(?:Z|[+-]\\d{2}:\\d{2})\\Z" +) + + def parse_timestamp(value: str) -> datetime: """Require an explicitly timezone-aware ISO-8601 timestamp.""" + # Restrict operator evidence to a single unambiguous RFC 3339-like form. + # datetime.fromisoformat alone also accepts week dates and arbitrary separators. + if not _TIMESTAMP_PATTERN.fullmatch(value): + raise ValueError("timestamp must be YYYY-MM-DDTHH:MM:SS[.ffffff] with timezone") try: parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError as exc: From c112c43671e164ca1b62bf29ee617b1dcc0f8416 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 14:58:15 +0000 Subject: [PATCH 10/15] test(recovery): cover ambiguous timestamps and fractional durations --- tests/test_restore_drill_measurement.py | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_restore_drill_measurement.py b/tests/test_restore_drill_measurement.py index 4564697b..ae988462 100644 --- a/tests/test_restore_drill_measurement.py +++ b/tests/test_restore_drill_measurement.py @@ -51,3 +51,30 @@ def test_rejects_inconsistent_evidence_order(): def test_rejects_utc_normalization_overflow(value): with pytest.raises(ValueError, match="supported UTC range"): parse_timestamp(value) + + +@pytest.mark.parametrize( + "value", + [ + "2026-11-15X09:00:00Z", + "2026-W46-7T09:00:00Z", + "2026-11-15 09:00:00+00:00", + "2026-11-15T09:00Z", + "2026-11-15T09:00:00+00:00:30", + "2026-11-15T09:00:00.1234567Z", + "2026-11-15T09:00:00Z" + " " * 10000, + ], +) +def test_rejects_ambiguous_or_unbounded_timestamp_formats(value): + with pytest.raises(ValueError, match="timestamp"): + parse_timestamp(value) + + +def test_accepts_fractional_seconds_and_explicit_offset(): + result = measure( + "2026-11-15T10:00:00.125+01:00", + "2026-11-15T09:00:01.125Z", + "2026-11-15T09:00:03.625+00:00", + ) + assert result["observed_rpo_seconds"] == 1.0 + assert result["observed_rto_seconds"] == 2.5 From 2e42a3c724839f567fd68f91bc7acc8967c7fa10 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 14:58:21 +0000 Subject: [PATCH 11/15] docs(recovery): specify strict operator timestamp evidence format --- docs/restore-drill-timing-evidence.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/restore-drill-timing-evidence.md b/docs/restore-drill-timing-evidence.md index 7cfc19cd..4ae43d64 100644 --- a/docs/restore-drill-timing-evidence.md +++ b/docs/restore-drill-timing-evidence.md @@ -21,7 +21,7 @@ Record the source of each timestamp alongside the JSON output: - **ready:** time authenticated readiness, inventory access, and audit integrity were **actually confirmed** after the isolated restore. -The helper rejects missing timezone offsets, UTC normalization overflow at the +Use full `YYYY-MM-DDTHH:MM:SS` timestamps with `Z` or `+HH:MM`/`-HH:MM` offsets; optional fractional seconds are limited to six digits. Week dates, spaces or arbitrary date/time separators, offset seconds, and extra trailing data are rejected rather than silently normalized.\n\nThe helper rejects missing timezone offsets, UTC normalization overflow at the minimum/maximum supported calendar dates, and impossible event ordering. Invalid inputs return CLI exit code 2 with no JSON evidence output. Its observed RPO proxy is failure minus last durable evidence; observed RTO From ce200a4e565763224bb04bf37da76ac4b3674cf4 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 14:58:29 +0000 Subject: [PATCH 12/15] docs(recovery): wrap strict timestamp format guidance --- docs/restore-drill-timing-evidence.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/restore-drill-timing-evidence.md b/docs/restore-drill-timing-evidence.md index 4ae43d64..c6b86814 100644 --- a/docs/restore-drill-timing-evidence.md +++ b/docs/restore-drill-timing-evidence.md @@ -21,7 +21,7 @@ Record the source of each timestamp alongside the JSON output: - **ready:** time authenticated readiness, inventory access, and audit integrity were **actually confirmed** after the isolated restore. -Use full `YYYY-MM-DDTHH:MM:SS` timestamps with `Z` or `+HH:MM`/`-HH:MM` offsets; optional fractional seconds are limited to six digits. Week dates, spaces or arbitrary date/time separators, offset seconds, and extra trailing data are rejected rather than silently normalized.\n\nThe helper rejects missing timezone offsets, UTC normalization overflow at the +Use full `YYYY-MM-DDTHH:MM:SS` timestamps with `Z` or `+HH:MM`/`-HH:MM`\noffsets; optional fractional seconds are limited to six digits. Week dates,\nspaces or arbitrary date/time separators, offset seconds, and extra trailing\ndata are rejected rather than silently normalized.\n\nThe helper rejects missing timezone offsets, UTC normalization overflow at the minimum/maximum supported calendar dates, and impossible event ordering. Invalid inputs return CLI exit code 2 with no JSON evidence output. Its observed RPO proxy is failure minus last durable evidence; observed RTO From 6bcfddd586d3c189c51c33e219f245678136587e Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 14:58:37 +0000 Subject: [PATCH 13/15] fix(recovery): correct raw regex escapes for strict timestamp parsing --- scripts/measure_restore_drill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/measure_restore_drill.py b/scripts/measure_restore_drill.py index 61753682..cee6956f 100644 --- a/scripts/measure_restore_drill.py +++ b/scripts/measure_restore_drill.py @@ -12,7 +12,7 @@ _TIMESTAMP_PATTERN = re.compile( - r"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,6})?(?:Z|[+-]\\d{2}:\\d{2})\\Z" + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})\Z" ) From f4fcb7cb7b7e39dfb36537e907a360df695a8ce0 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 15:55:35 +0000 Subject: [PATCH 14/15] style(tests): normalize UTC overflow parametrization for Black --- tests/test_restore_drill_measurement.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_restore_drill_measurement.py b/tests/test_restore_drill_measurement.py index ae988462..f5370cc9 100644 --- a/tests/test_restore_drill_measurement.py +++ b/tests/test_restore_drill_measurement.py @@ -43,10 +43,7 @@ def test_rejects_inconsistent_evidence_order(): @pytest.mark.parametrize( "value", - [ - "0001-01-01T00:00:00+01:00", - "9999-12-31T23:59:59-01:00", - ], + ["0001-01-01T00:00:00+01:00", "9999-12-31T23:59:59-01:00"], ) def test_rejects_utc_normalization_overflow(value): with pytest.raises(ValueError, match="supported UTC range"): From bbe2611638ba5d974133355981e825b85d6084f5 Mon Sep 17 00:00:00 2001 From: Adam Ghanem Date: Sun, 15 Nov 2026 17:03:02 +0000 Subject: [PATCH 15/15] style(recovery): normalize Black spacing before timestamp pattern --- scripts/measure_restore_drill.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/measure_restore_drill.py b/scripts/measure_restore_drill.py index cee6956f..460deeea 100644 --- a/scripts/measure_restore_drill.py +++ b/scripts/measure_restore_drill.py @@ -10,7 +10,6 @@ import re from datetime import datetime, timezone - _TIMESTAMP_PATTERN = re.compile( r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})\Z" )