diff --git a/changelog.d/3737.security.md b/changelog.d/3737.security.md index 6af3c3ad0c..28135946c6 100644 --- a/changelog.d/3737.security.md +++ b/changelog.d/3737.security.md @@ -1,11 +1,5 @@ -- **`rotate-key` keeps a rotation record, so a `serve` start no longer stamps a key once a run on - it has opened that record and not finished.** The command opens the record in - `secret_rotation_meta` before either data pass, after its store open, and closes it only when a - run completes. When a start finds that the key in use differs from its stamp, or has none, and the - record names that key or cannot be read, `serve` writes no stamp, does not reset the key's age, - and logs a WARNING. A run killed inside the store open, before the record opens, leaves no - record. On a store with a keyed audit chain, a start that then finds the key changed opens one - itself, as does a start before the first run; the `BACKLOG #4942` entry says when. **Once a run +- **`rotate-key` keeps a rotation record in `secret_rotation_meta`, and a `serve` start does not + stamp a changed store key while that record names it or cannot be read.** **Once a run has rolled a keyed audit chain to a new key there is no way back to the prior key**, and a run that skipped an uploaded file rolls like any other. **The upload pass opens only a regular file, and reads whole only one of a size an upload can be.** A FIFO no longer blocks it, a symlink is a skip and is not followed, and a file too large for an upload is a diff --git a/changelog.d/4936.security.md b/changelog.d/4936.security.md new file mode 100644 index 0000000000..39daa1cba2 --- /dev/null +++ b/changelog.d/4936.security.md @@ -0,0 +1,4 @@ +- **A `rotate-key` run killed inside its store open, after the open's new `before_first_write` + hook has opened the rotation record, now leaves that record open, so a later `serve` start does + not stamp the new key.** A writable open runs the hook before its first write under the store + key; a read-only open does not run it. (`BACKLOG #4936`) diff --git a/docs/PHI.md b/docs/PHI.md index 31c2c510d1..2ec896375c 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -751,8 +751,10 @@ for defense-in-depth without swapping the `aiosqlite` connector. uses, where the run prints a note naming the path it read. Before retiring the prior key, confirm the volume is mounted and the path is the one `serve` uses. **A `serve` start does not stamp a changed key while the rotation record names it or cannot be read (BACKLOG - #3737).** `rotate-key` opens a rotation record in `secret_rotation_meta` before either data pass - (the store open can already have written under the key) and closes it only when a run completes, + #3737).** `rotate-key` opens a rotation record in `secret_rotation_meta` inside the store open, + before the open's first write under the key (BACKLOG #4936; steps that write nothing under the + key can run before it, at least the schema step and, on SQL Server, the database option switch), + and closes it only when a run completes, so a run killed or failed after that point leaves it open. **A sealed record under a key the run was not given stops it (vault BACKLOG #5635).** When a user, role, open approval or directory-map seal names a key the run does not hold, a pre-flight read finds it before any re-seal or cell @@ -783,10 +785,10 @@ for defense-in-depth without swapping the `aiosqlite` connector. store with no keyed chain to ask, such as one whose audit rows open with no genesis row naming its key, so there a stamp proves only a changed key. On such a store, do not start `serve` between setting a new key and running `rotate-key`: that stamp stands however the run ends. On a - store with a keyed chain, do not start `serve` on the new key alone before a run's roll succeeds: - the chain's current range is then under a key it does not hold, so it keys new audit rows under - the new key, those rows never verify inside that range, and no later `rotate-key` can roll the - chain. **A roll that refuses for good holds the key for good (BACKLOG #4934, #4935).** The roll + store with a keyed chain, do not start `serve`, or run any other command that writes an audit + row, on the new key alone before a run's roll succeeds: the chain's current range is then under + a key it does not hold, so it keys new audit rows under the new key, those rows never verify + inside that range, and no later `rotate-key` can roll the chain. **A roll that refuses for good holds the key for good (BACKLOG #4934, #4935).** The roll verifies the whole chain first, so a chain whose rows do not verify, or whose rows open with no genesis row, refuses under every key, a new one included, and the record stays open. A newest range under a key that is not configured is not one of these, though `audit-verify` reports it diff --git a/messagefoundry/__main__.py b/messagefoundry/__main__.py index 9eb6e7e6e1..abdf9798d0 100644 --- a/messagefoundry/__main__.py +++ b/messagefoundry/__main__.py @@ -11695,8 +11695,9 @@ def _rotate_key(args: argparse.Namespace) -> int: own fingerprint (``active_key_id``, stored scheme-tagged, BACKLOG #3421), never the sub-key (ADR 0196 AC-7). - **What the order of the steps guarantees (BACKLOG #3737).** The rotation record opens before - either data pass, the audit range rolls after both, and the stamp and the record's close come + **What the order of the steps guarantees (BACKLOG #3737).** The rotation record opens inside + the store open, before its first write under the key, on every store (BACKLOG #4936), so before + either data pass too; the audit range rolls after both, and the stamp and the record's close come last, on a run with no skip, an uploads directory both its pre-check and its uploads pass could list (BACKLOG #4940; a listable directory that is not the one holding the uploads passes both, which docs/PHI.md states) and a good roll. One it cannot list at the start refuses before @@ -11854,16 +11855,26 @@ async def run() -> tuple[int, ResealResult, tuple[bool, str], bool, ResealOutcom # A key the open cannot resolve, or a store hop the posture refuses, exits 2 as one at # resolve_active_key above does. vault_transit mode never gets here (BACKLOG #2109). _warn_on_an_unrevoked_store_hop(settings) # serve's WARNING at warn (#2532) - store = await _open_store_or_refuse(open_store(settings.store, posture=posture)) + today = datetime.datetime.now(tz=datetime.UTC).date().isoformat() + # Whether the record names this key: a run with the key already in use opens none. + record_named = False + + async def open_record(opening: object) -> None: + # BACKLOG #3737, #4936: inside the store open, before its first write under the key, + # so a kill after it, the open's own writes and either data pass + # included, leaves any record it opened open until a run closes it. + nonlocal record_named + # Every backend's store has cipher_info(); the hook is typed for any of them. + info = getattr(opening, "cipher_info", None) + key = info().active_key_id if callable(info) else None + if isinstance(opening, SecretRotationMetaStore) and isinstance(key, str) and key: + record_named = await open_rotation_record(opening, key, today) + + store = await _open_store_or_refuse( + open_store(settings.store, posture=posture, before_first_write=open_record) + ) try: - today = datetime.datetime.now(tz=datetime.UTC).date().isoformat() key_id = store.cipher_info().active_key_id - # BACKLOG #3737: before either data pass, so a kill or an abort in one leaves it open too. - # The store open above can already have written under this key. - # Whether it names this key: a run with the key already in use opens none. - record_named = False - if isinstance(store, SecretRotationMetaStore) and key_id: - record_named = await open_rotation_record(store, key_id, today) count = await store.reencrypt_to_active() # Vault BACKLOG #5635: what the store pass left under a key it holds and could not # re-seal, for the note below. A record under a key it does not hold aborted the pass. diff --git a/messagefoundry/pipeline/secret_rotation.py b/messagefoundry/pipeline/secret_rotation.py index 2b85357269..af7fb6161b 100644 --- a/messagefoundry/pipeline/secret_rotation.py +++ b/messagefoundry/pipeline/secret_rotation.py @@ -277,9 +277,15 @@ async def open_rotation_record( meta_store: SecretRotationMetaStore, key_id: str, day: str, *, keep_another: bool = False ) -> bool: """Record that ``rotate-key`` is moving this store to the DEK ``key_id``, before either data - pass, so a crash or a failed step in one leaves the record standing (BACKLOG #3737). The store - open has run by then and can already have written under the key. A ``serve`` start calls it too, - for a changed key the keyed audit chain has not rolled to (BACKLOG #4942), with ``keep_another``: + pass, so a crash or a failed step in one leaves the record standing (BACKLOG #3737). + ``rotate-key`` calls it from the store open's ``before_first_write`` hook (BACKLOG #4936), so + before the open's first write under the key. Steps that write nothing under the key can run + before it, at least the schema step, the store id pin check and, on SQL Server, the database + option switch. A ``serve`` start calls it too, after its own open has written under the key, + through :func:`reconcile_rotation_meta`, which says when: at least for a changed key the keyed + audit chain has not rolled to, unless the record already names that key or cannot be read, + and for one it has rolled to while a record stands open for another key (BACKLOG #4942). It + passes ``keep_another``: it then writes nothing over a record open for another key. That record holds its own key, which a run may have rolled the chain to with an upload still under the prior key, so replacing it would let a later start on that key stamp it. ``rotate-key`` replaces it, keeping its dates. A start on @@ -338,23 +344,21 @@ async def close_rotation_record(meta_store: SecretRotationMetaStore, day: str) - #: The WARNING a start logs when it opened the rotation record itself (BACKLOG #4942). _OPENED_AT_START = ( - "secret rotation: the store key in use differs from its stamp, or has none, and the audit " - "chain has not been rolled to it, so no `messagefoundry rotate-key` run has completed on it. A " + "secret rotation: the store key in use is not confirmed by its stamp, and the open did not " + "find the audit chain rolled to it. A " "rotation record was opened for it, its rotation date was not stamped and its age was not " "reset. Stop the engine and run `messagefoundry rotate-key`, with the prior key in " - "MEFOR_STORE_ENCRYPTION_KEYS_RETIRED, until it exits 0. If it refuses at its audit chain step, " - "its error says whether a re-run can clear that, and the way out where none can." + "MEFOR_STORE_ENCRYPTION_KEYS_RETIRED." ) #: The WARNING a start logs when it held a key and kept a record open for another key (BACKLOG #: #4942): see :func:`open_rotation_record`'s ``keep_another``. _HELD_FOR_ANOTHER_RECORD = ( - "secret rotation: the store key in use differs from its stamp, or has none, while the " + "secret rotation: the store key in use is not confirmed by its stamp, while the " "rotation record stands open for another store key that no `messagefoundry rotate-key` run has " "completed on. That record was kept, this key's " "rotation date was not stamped and its age was not reset. Stop the engine and run " - "`messagefoundry rotate-key`, with every prior key in MEFOR_STORE_ENCRYPTION_KEYS_RETIRED, " - "until it exits 0." + "`messagefoundry rotate-key`, with every prior key in MEFOR_STORE_ENCRYPTION_KEYS_RETIRED." ) @@ -365,9 +369,10 @@ def _rotation_hold(record: SecretRotationMetaRow | None, fingerprint: str) -> st It holds for a record naming this key. It also holds for a record that cannot be read, because this function fails toward NOT stamping: an unstamped key over-reports its age, and a false stamp is what would lift an overdue refusal. A record naming another key does not hold here: the key - in use was changed by hand. Where the keyed audit chain has not rolled to that key, - :func:`reconcile_rotation_meta` holds it anyway and keeps the record (BACKLOG #4942); elsewhere - it stamps as any such change does.""" + in use was changed by hand. On a store with a keyed audit chain, + :func:`reconcile_rotation_meta` holds it anyway and keeps the record (BACKLOG #4942), whether or + not the chain has rolled to the key in use; on a store with none it stamps as any such change + does.""" if record is None or record.fingerprint == _ROTATION_RECORD_CLOSED: return None # A key id has one length, so one of any other length is a damaged record, not another key. diff --git a/messagefoundry/store/base.py b/messagefoundry/store/base.py index 355a82168d..61d601bf93 100644 --- a/messagefoundry/store/base.py +++ b/messagefoundry/store/base.py @@ -28,7 +28,15 @@ import sys import threading import weakref -from collections.abc import AsyncIterator, Callable, Collection, Iterable, Mapping, Sequence +from collections.abc import ( + AsyncIterator, + Awaitable, + Callable, + Collection, + Iterable, + Mapping, + Sequence, +) from dataclasses import dataclass from functools import cache, partial from pathlib import Path @@ -80,6 +88,7 @@ from messagefoundry.store.row_seal import AnyRowSealer, ResealOutcome from messagefoundry.store.store import ( ANY_CLAIM_OWNER, + OPEN_STORE_HOOK, OPEN_STORE_PIN, OPEN_STORE_PIN_CHECK, UPLOAD_RESERVATION_STALE_AFTER, @@ -4796,6 +4805,7 @@ async def open_store( refusal_hook: UnmarkedRefusalHook | None = None, custody: HostBlob | None = None, recovery_wrap: bytes = b"", + before_first_write: Callable[[Any], Awaitable[None]] | None = None, ) -> Store: """Open the store for the configured backend — the single backend-selection seam. @@ -4879,6 +4889,12 @@ async def open_store( Every custody refusal is a :class:`~messagefoundry.store.keyprovider.KeyProviderError`, with a fixed text and no cause or context; :func:`_host_custody_for_open` says which errors on the custody path are replaced (BACKLOG #4882). + + ``before_first_write`` (BACKLOG #4936) is awaited with the half-opened backend store on every + writable open, right after the pin check: the schema is in place, and nothing has yet bound a + salt, reserved a cipher block, sealed a row or written an audit row. ``rotate-key`` opens its + rotation record there. The schema step before it can already have run its DDL or a migration. + A read-only open does not run it. What it raises fails the open, with the handle closed. """ if create and read_only: raise ValueError("open_store: create and read_only exclude each other") @@ -4931,6 +4947,7 @@ async def open_store( # The batch failures a backend typed itself, and the pin refusals, are re-raised as they are. check = PinCheck() if minting_open() and settings.backend is not StoreBackend.SQLITE else None checking = OPEN_STORE_PIN_CHECK.set(check) + hooked = OPEN_STORE_HOOK.set(before_first_write) try: store = await _open_backend( settings, @@ -4947,6 +4964,7 @@ async def open_store( raise raise StoreCreateBatchFailedBeforeKeyUse(exc) from exc finally: + OPEN_STORE_HOOK.reset(hooked) OPEN_STORE_PIN_CHECK.reset(checking) OPEN_STORE_PIN.reset(pinned) if keyless_chain_refusal is not None and audit_mac_key is None and audit_mac_fn is None: diff --git a/messagefoundry/store/postgres.py b/messagefoundry/store/postgres.py index 981efdc746..1cfbddb8b3 100644 --- a/messagefoundry/store/postgres.py +++ b/messagefoundry/store/postgres.py @@ -395,6 +395,7 @@ roles_if_held, roll_audit_key_range, rotation_reseal, + run_open_store_hook, scope_if_source, scope_withdrawal, seal_approval_async, @@ -2221,6 +2222,7 @@ async def open( await store._ensure_schema(create=create, read_only=read_only) if not read_only: await check_open_store_pin(store) # vault BACKLOG #3411: before any write + await run_open_store_hook(store) # BACKLOG #4936: after the pin, before any write # ADR 0196: bind this store's salt BEFORE the first checkpoint, so the block # reserved next and every value sealed after it land under this store's own data # sub-key. A read-only handle seals nothing; every stored value names its own salt. diff --git a/messagefoundry/store/sqlserver.py b/messagefoundry/store/sqlserver.py index 289a1d4303..4a68c1909f 100644 --- a/messagefoundry/store/sqlserver.py +++ b/messagefoundry/store/sqlserver.py @@ -403,6 +403,7 @@ roles_if_held, roll_audit_key_range, rotation_reseal, + run_open_store_hook, scope_if_source, scope_withdrawal, seal_approval_async, @@ -4301,6 +4302,7 @@ async def open( await store._gate_claim_prepared() if not read_only: await check_open_store_pin(store) # vault BACKLOG #3411: before any write + await run_open_store_hook(store) # BACKLOG #4936: after the pin, before any write # ADR 0196: bind this store's salt BEFORE the first checkpoint, so the block # reserved next and every value sealed after it land under this store's own data # sub-key. A read-only handle seals nothing; every stored value names its own salt. diff --git a/messagefoundry/store/store.py b/messagefoundry/store/store.py index ed50395bcd..78cdf50ac9 100644 --- a/messagefoundry/store/store.py +++ b/messagefoundry/store/store.py @@ -6905,14 +6905,33 @@ def __init__(self, failure: Exception) -> None: #: construction; ``None`` for every open that is not under the ``host`` key provider. OPEN_STORE_PIN: ContextVar[StorePin | None] = ContextVar("mefor_open_store_pin", default=None) +#: The ``before_first_write`` hook of the store open in progress, set by ``store.base.open_store`` +#: around the backend open and run by :func:`run_open_store_hook` (BACKLOG #4936). A context +#: variable for :data:`OPEN_STORE_PIN`'s reason; ``None`` for an open that passes no hook. +OPEN_STORE_HOOK: ContextVar[Callable[[Any], Awaitable[None]] | None] = ContextVar( + "mefor_open_store_hook", default=None +) + + +async def run_open_store_hook(host: object) -> None: + """Run the open's ``before_first_write`` hook on the half-opened ``host`` (BACKLOG #4936). Each + backend's writable open calls this right after :func:`check_open_store_pin`, so once the schema + is in place and before it binds a salt, reserves a cipher block, seals a row or writes an audit + row. A read-only open never calls it. A no-op when no hook is set. What the hook raises fails + the open, which closes the handle.""" + hook = OPEN_STORE_HOOK.get() + if hook is not None: + await hook(host) + class PinCheck: - """Whether the open in progress has passed its store id pin check (:func:`check_open_store_pin`), - the last step before its first write under the store key. Under ``auto`` schema management the - open may already have committed its schema, and on SQL Server altered the database's options, - before it; neither is under the key. A mutable cell rather than a flag in a context variable, - so a pass recorded under a copied context still reaches the open that set it (vault BACKLOG - #8119 item 2).""" + """Whether the open in progress has passed its store id pin check (:func:`check_open_store_pin`). + The open's ``before_first_write`` hook (:func:`run_open_store_hook`, BACKLOG #4936) runs after + the pin check, and the open's first write under the store key after both. Under ``auto`` + schema management the open may already have committed its schema, and on SQL Server altered + the database's options, before the pin check; neither is under the key. A mutable cell rather + than a flag in a context variable, so a pass recorded under a copied context still reaches the + open that set it (vault BACKLOG #8119 item 2).""" __slots__ = ("passed",) @@ -11583,6 +11602,7 @@ async def open( # has not bound, a failure before the bind included (BACKLOG #5720). store._cipher_foreign = True await check_open_store_pin(store) # vault BACKLOG #3411: before any write + await run_open_store_hook(store) # BACKLOG #4936: after the pin, before any write # ADR 0196: bind this store's salt BEFORE the first checkpoint, so the block # reserved next and every value sealed after it land under this store's own data # sub-key. A read-only handle seals nothing; every stored value names its own salt. diff --git a/tests/test_rotate_key_open_hook_and_lost_range_key.py b/tests/test_rotate_key_open_hook_and_lost_range_key.py new file mode 100644 index 0000000000..c29fd54e74 --- /dev/null +++ b/tests/test_rotate_key_open_hook_and_lost_range_key.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""BACKLOG #4936, the owner ruling of 2026-11-04. + +``rotate-key`` opens its rotation record through ``open_store``'s ``before_first_write`` hook, so a +run killed inside the store open after the hook leaves any record it opened.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any + +import pytest + +from messagefoundry.__main__ import main +from messagefoundry.config.ai_policy import SecurityEnforcement +from messagefoundry.config.settings import SecretRotationSettings, StoreSettings +from messagefoundry.pipeline.secret_rotation import ( + ROTATION_RECORD_ID, + dek_fingerprint, + reconcile_rotation_meta, +) +from messagefoundry.store.base import open_store +from messagefoundry.store.crypto import generate_key +from messagefoundry.store.store import ( + MessageStore, + audit_chain_rolled, +) +from tests.test_rotate_key_rotation_record import ( + _DEK, + _OLD_DATE, + _env, + _key_id, + _open, + _seed, + _seed_keyless_chain, +) + + +class _Killed(Exception): + """Stands in for a kill inside the store open, after the hook ran.""" + + +def _kill_after_the_hook(monkeypatch: pytest.MonkeyPatch) -> None: + async def killed(*_args: object, **_kwargs: object) -> None: + raise _Killed("killed inside the store open") + + # The salt bind is the open's first write under the key, right after the hook. + monkeypatch.setattr("messagefoundry.store.store.bind_store_salt", killed) + + +async def _reconcile(db: Path, key: str, *retired: str) -> dict[str, Any]: + store = await _open(db, key, *retired) + try: + await reconcile_rotation_meta( + store, + SecretRotationSettings(), + dek_key_id=_key_id(key), + enforcement=SecurityEnforcement.WARN, + env_values={}, + audit_rolled=audit_chain_rolled(store), + ) + return dict(await store.get_secret_rotation_meta()) + finally: + await store.close() + + +def test_a_run_killed_inside_the_open_on_a_store_with_no_keyed_chain_leaves_no_stamp( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """#4936 step 3. On the code before the hook no record existed at the kill, and this start + stamped the new key, because a store with no keyed chain has nothing else to hold it.""" + monkeypatch.chdir(tmp_path) + db, uploads, key_a, key_b = ( + tmp_path / "k.db", + tmp_path / "uploads", + generate_key(), + generate_key(), + ) + asyncio.run(_seed_keyless_chain(db, uploads, key_a)) + _env(monkeypatch, key_b, key_a, uploads) + with monkeypatch.context() as patch: + _kill_after_the_hook(patch) + # run_guarded turns the uncaught kill into a failing exit. + assert main(["rotate-key", "--db", str(db)]) != 0 + capsys.readouterr() + rows = asyncio.run(_reconcile(db, key_b, key_a)) + assert rows[ROTATION_RECORD_ID].fingerprint == dek_fingerprint(_key_id(key_b)) + assert rows[_DEK].fingerprint == dek_fingerprint(_key_id(key_a)) + assert rows[_DEK].last_rotated == _OLD_DATE + + +def test_the_hook_runs_before_the_first_write_and_not_on_a_read_only_open(tmp_path: Path) -> None: + """The hook sees the store before the salt bind and the at-rest sweep, and a read-only open + never runs it.""" + db, key = tmp_path / "h.db", generate_key() + asyncio.run(_seed(db, tmp_path / "uploads", key)) + seen: list[str] = [] + + async def hook(store: object) -> None: + assert isinstance(store, MessageStore) + assert store._cipher_foreign # the salt is not bound yet + seen.append("hook") + + async def opened(read_only: bool) -> None: + settings = StoreSettings(path=str(db), encryption_key=key) + store = await open_store( + settings, read_only=read_only, keyless_chain_refusal=None, before_first_write=hook + ) + await store.close() + + asyncio.run(opened(False)) + assert seen == ["hook"] + asyncio.run(opened(True)) + assert seen == ["hook"] diff --git a/tests/test_rotate_key_rotation_record.py b/tests/test_rotate_key_rotation_record.py index dc38074555..0eae5cc024 100644 --- a/tests/test_rotate_key_rotation_record.py +++ b/tests/test_rotate_key_rotation_record.py @@ -322,7 +322,7 @@ def test_a_serve_start_before_the_first_run_opens_the_record_and_stamps_nothing( assert rows[_DEK].last_rotated == _OLD_DATE record = rows[ROTATION_RECORD_ID] assert record.fingerprint == dek_fingerprint(_key_id(key_b)) - assert "the audit chain has not been rolled to it" in caplog.text + assert "did not find the audit chain rolled to it" in caplog.text caplog.clear() with caplog.at_level(logging.WARNING, logger="messagefoundry.pipeline.secret_rotation"): @@ -345,8 +345,9 @@ def test_a_run_killed_before_it_opened_the_record_leaves_no_stamp( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - """BACKLOG #4936. The run stops after its store open and before the record opens, the window a - kill inside the open stands in. No record exists, and the next start must still not stamp.""" + """BACKLOG #4936. The record open itself fails, inside the store open's hook and before any + write under the key. No record exists, and the next start on a keyed chain must still not + stamp.""" db, uploads, key_a, key_b = rig _env(monkeypatch, key_b, key_a, uploads)