Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
8e4807e
test(recovery): add offline restore timing evidence and regression co…
Adam-Ghanem Nov 15, 2026
bf2e339
test(recovery): add offline restore timing evidence and regression co…
Adam-Ghanem Nov 15, 2026
3066514
test(recovery): add offline restore timing evidence and regression co…
Adam-Ghanem Nov 15, 2026
b59dc0c
test(recovery): cover offline drill CLI JSON and fail-closed errors
Adam-Ghanem Nov 15, 2026
c4c01be
fix(recovery): reject timezone normalization overflow as invalid evid…
Adam-Ghanem Nov 15, 2026
5d4c4ca
test(recovery): cover minimum and maximum UTC overflow boundaries
Adam-Ghanem Nov 15, 2026
f07d532
test(recovery): require clean CLI error for out-of-range UTC instants
Adam-Ghanem Nov 15, 2026
732a94b
docs(recovery): document fail-closed UTC boundary handling
Adam-Ghanem Nov 15, 2026
0ca53cb
fix(recovery): reject ambiguous restore evidence timestamp formats
Adam-Ghanem Nov 15, 2026
c112c43
test(recovery): cover ambiguous timestamps and fractional durations
Adam-Ghanem Nov 15, 2026
2e42a3c
docs(recovery): specify strict operator timestamp evidence format
Adam-Ghanem Nov 15, 2026
ce200a4
docs(recovery): wrap strict timestamp format guidance
Adam-Ghanem Nov 15, 2026
6bcfddd
fix(recovery): correct raw regex escapes for strict timestamp parsing
Adam-Ghanem Nov 15, 2026
f4fcb7c
style(tests): normalize UTC overflow parametrization for Black
Adam-Ghanem Nov 15, 2026
bbe2611
style(recovery): normalize Black spacing before timestamp pattern
Adam-Ghanem Nov 15, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions docs/restore-drill-timing-evidence.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Isolated restore drill: timing evidence

This **offline, read-only** helper calculates two observed intervals from
operator-recorded timestamps. It does not create, verify, or restore backups,
contact an API, or assert that an application is healthy.

Run only after an authorized isolated staging restore with synthetic data:

```bash
python scripts/measure_restore_drill.py \
--last-durable 2026-11-15T09:00:00Z \
--failure 2026-11-15T09:03:00Z \
--ready 2026-11-15T09:05:30Z
```

Record the source of each timestamp alongside the JSON output:

- **last-durable:** newest independently verified, restorable evidence
recovered in staging (not merely the most recent backup attempt).
- **failure:** simulated failure start, from the drill operator's clock.
- **ready:** time authenticated readiness, inventory access, and audit
integrity were **actually confirmed** after the isolated restore.

Use full `YYYY-MM-DDTHH:MM:SS` timestamps with `Z` or `+HH:MM`/`-HH:MM`\noffsets; optional fractional seconds are limited to six digits. Week dates,\nspaces or arbitrary date/time separators, offset seconds, and extra trailing\ndata are rejected rather than silently normalized.\n\nThe helper rejects missing timezone offsets, UTC normalization overflow at the
minimum/maximum supported calendar dates, and impossible event ordering.
Invalid inputs return CLI exit code 2 with no JSON evidence output.
Its observed RPO proxy is failure minus last durable evidence; observed RTO
proxy is ready minus failure. These are *drill observations*, not guaranteed
production RPO/RTO, a measured data-loss count, or proof of recovery.

For acceptance, separately retain backup hash/verification results, schema
version, authenticated API checks, audit-chain evidence, operator sign-off,
and clock synchronization details. Do not include credentials, personal
inventory, production databases, or secrets in the evidence bundle.
See [Track A restore drill](track-a-restore-drill.md) for a historical local
drill; its schema version and timing are not a current production baseline.
70 changes: 70 additions & 0 deletions scripts/measure_restore_drill.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
"""Offline timing evidence for an operator-controlled restore drill.

This reports observed intervals only; it never performs a restore or network request.
"""

from __future__ import annotations

import argparse
import json
import re
from datetime import datetime, timezone

_TIMESTAMP_PATTERN = re.compile(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})\Z"
)


def parse_timestamp(value: str) -> datetime:
"""Require an explicitly timezone-aware ISO-8601 timestamp."""
# Restrict operator evidence to a single unambiguous RFC 3339-like form.
# datetime.fromisoformat alone also accepts week dates and arbitrary separators.
if not _TIMESTAMP_PATTERN.fullmatch(value):
raise ValueError("timestamp must be YYYY-MM-DDTHH:MM:SS[.ffffff] with timezone")
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise ValueError("timestamp must be ISO-8601 with timezone") from exc
if parsed.tzinfo is None or parsed.utcoffset() is None:
raise ValueError("timestamp must include a timezone")
try:
return parsed.astimezone(timezone.utc)
except (OverflowError, ValueError) as exc:
raise ValueError("timestamp outside supported UTC range") from exc


def measure(last_durable: str, failure: str, ready: str) -> dict[str, object]:
"""Calculate observed RPO/RTO proxies from operator-supplied UTC instants."""
durable_at = parse_timestamp(last_durable)
failure_at = parse_timestamp(failure)
ready_at = parse_timestamp(ready)
if durable_at > failure_at:
raise ValueError("last durable evidence cannot follow failure")
if ready_at < failure_at:
raise ValueError("readiness cannot precede failure")
return {
"last_durable_at": durable_at.isoformat(),
"failure_at": failure_at.isoformat(),
"ready_at": ready_at.isoformat(),
"observed_rpo_seconds": (failure_at - durable_at).total_seconds(),
"observed_rto_seconds": (ready_at - failure_at).total_seconds(),
"scope": "operator-supplied isolated drill timestamps; not production SLA",
}


def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--last-durable", required=True)
parser.add_argument("--failure", required=True)
parser.add_argument("--ready", required=True)
args = parser.parse_args()
try:
report = measure(args.last_durable, args.failure, args.ready)
except ValueError as exc:
parser.error(str(exc))
print(json.dumps(report, sort_keys=True))
return 0


if __name__ == "__main__":
raise SystemExit(main())
92 changes: 92 additions & 0 deletions tests/test_restore_drill_cli_contract.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
"""CLI contract checks for the offline restore timing evidence helper."""

from __future__ import annotations

import json
import sys

import pytest

from scripts.measure_restore_drill import main


def test_cli_emits_machine_readable_observations(monkeypatch, capsys):
monkeypatch.setattr(
sys,
"argv",
[
"measure_restore_drill.py",
"--last-durable",
"2026-11-15T09:00:00Z",
"--failure",
"2026-11-15T09:03:00Z",
"--ready",
"2026-11-15T09:05:30Z",
],
)
assert main() == 0
captured = capsys.readouterr()
assert captured.err == ""
report = json.loads(captured.out)
assert report["observed_rpo_seconds"] == 180.0
assert report["observed_rto_seconds"] == 150.0
assert "not production SLA" in report["scope"]


@pytest.mark.parametrize(
"last_durable,failure,ready",
[
("2026-11-15T09:04:00Z", "2026-11-15T09:03:00Z", "2026-11-15T09:05:00Z"),
("2026-11-15T09:00:00Z", "2026-11-15T09:03:00Z", "2026-11-15T09:02:00Z"),
("2026-11-15T09:00:00", "2026-11-15T09:03:00Z", "2026-11-15T09:05:00Z"),
],
)
def test_cli_rejects_invalid_evidence_without_json(
monkeypatch, capsys, last_durable, failure, ready
):
monkeypatch.setattr(
sys,
"argv",
[
"measure_restore_drill.py",
"--last-durable",
last_durable,
"--failure",
failure,
"--ready",
ready,
],
)
with pytest.raises(SystemExit) as exc:
main()
assert exc.value.code == 2
captured = capsys.readouterr()
assert captured.out == ""
assert "error:" in captured.err


@pytest.mark.parametrize(
"last_durable",
["0001-01-01T00:00:00+01:00", "9999-12-31T23:59:59-01:00"],
)
def test_cli_reports_utc_overflow_without_traceback(monkeypatch, capsys, last_durable):
monkeypatch.setattr(
sys,
"argv",
[
"measure_restore_drill.py",
"--last-durable",
last_durable,
"--failure",
"2026-11-15T09:03:00Z",
"--ready",
"2026-11-15T09:05:00Z",
],
)
with pytest.raises(SystemExit) as exc:
main()
assert exc.value.code == 2
captured = capsys.readouterr()
assert captured.out == ""
assert "supported UTC range" in captured.err
assert "Traceback" not in captured.err
77 changes: 77 additions & 0 deletions tests/test_restore_drill_measurement.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
from __future__ import annotations

import pytest

from scripts.measure_restore_drill import measure, parse_timestamp


def test_measurement_uses_actual_elapsed_seconds_and_normalizes_offsets():
result = measure(
"2026-11-15T10:00:00+01:00",
"2026-11-15T09:03:00Z",
"2026-11-15T09:05:30+00:00",
)
assert result["observed_rpo_seconds"] == 180.0
assert result["observed_rto_seconds"] == 150.0
assert result["last_durable_at"] == "2026-11-15T09:00:00+00:00"
assert "not production SLA" in str(result["scope"])


@pytest.mark.parametrize(
"value",
["2026-11-15T09:00:00", "not-a-time", "2026-13-15T09:00:00Z"],
)
def test_rejects_naive_or_invalid_timestamps(value):
with pytest.raises(ValueError, match="timestamp"):
parse_timestamp(value)


def test_rejects_inconsistent_evidence_order():
with pytest.raises(ValueError, match="durable"):
measure(
"2026-11-15T09:04:00Z",
"2026-11-15T09:03:00Z",
"2026-11-15T09:05:00Z",
)
with pytest.raises(ValueError, match="readiness"):
measure(
"2026-11-15T09:00:00Z",
"2026-11-15T09:03:00Z",
"2026-11-15T09:02:00Z",
)


@pytest.mark.parametrize(
"value",
["0001-01-01T00:00:00+01:00", "9999-12-31T23:59:59-01:00"],
)
def test_rejects_utc_normalization_overflow(value):
with pytest.raises(ValueError, match="supported UTC range"):
parse_timestamp(value)


@pytest.mark.parametrize(
"value",
[
"2026-11-15X09:00:00Z",
"2026-W46-7T09:00:00Z",
"2026-11-15 09:00:00+00:00",
"2026-11-15T09:00Z",
"2026-11-15T09:00:00+00:00:30",
"2026-11-15T09:00:00.1234567Z",
"2026-11-15T09:00:00Z" + " " * 10000,
],
)
def test_rejects_ambiguous_or_unbounded_timestamp_formats(value):
with pytest.raises(ValueError, match="timestamp"):
parse_timestamp(value)


def test_accepts_fractional_seconds_and_explicit_offset():
result = measure(
"2026-11-15T10:00:00.125+01:00",
"2026-11-15T09:00:01.125Z",
"2026-11-15T09:00:03.625+00:00",
)
assert result["observed_rpo_seconds"] == 1.0
assert result["observed_rto_seconds"] == 2.5
Loading