From 818ed0af89d488317878fe593bd6856f39364361 Mon Sep 17 00:00:00 2001 From: Jerry the Martian <38183696+jerryfane@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:25:11 +0200 Subject: [PATCH 1/7] Gitmoot merge adhoc-751e1dbc Merged by Gitmoot after policy gate passed. --- README.md | 4 + docs/answer_decision.md | 118 ++++ scripts/sqlite_sidecar_race_benchmark.py | 9 +- src/tendwire/backends/herdr_decision.py | 141 ++++ src/tendwire/backends/herdr_turns.py | 114 ++- src/tendwire/cli.py | 10 +- src/tendwire/command_submission.py | 417 ++++++++++- src/tendwire/core/commands.py | 170 ++++- src/tendwire/core/models.py | 1 + src/tendwire/core/turns.py | 39 ++ src/tendwire/store/sqlite.py | 615 +++++++++++++++- tests/test_answer_decision.py | 856 +++++++++++++++++++++++ tests/test_backend_pending.py | 14 +- tests/test_command_submission.py | 16 +- tests/test_commands.py | 74 +- tests/test_turns.py | 1 + 16 files changed, 2564 insertions(+), 35 deletions(-) create mode 100644 docs/answer_decision.md create mode 100644 src/tendwire/backends/herdr_decision.py create mode 100644 tests/test_answer_decision.py diff --git a/README.md b/README.md index bb20364..5581acd 100644 --- a/README.md +++ b/README.md @@ -1307,6 +1307,10 @@ Tendwire now exposes a minimal, safety-first command interface: echo '{"schema_version": 1, "action": "noop"}' | tendwire command --json ``` +The connector-facing structured Claude decision payload, semantic +`answer_decision` action, validation, and retry behavior are documented in +[docs/answer_decision.md](docs/answer_decision.md). + The `command --json` subcommand reads exactly one schema-v1 JSON request from stdin. A proven result is exactly one schema-v2 command envelope on JSON-only stdout with exit `0`/`1`; unresolved process ambiguity is no stdout envelope diff --git a/docs/answer_decision.md b/docs/answer_decision.md new file mode 100644 index 0000000..77f9c0d --- /dev/null +++ b/docs/answer_decision.md @@ -0,0 +1,118 @@ +# Remote decision contract + +`answer_decision` is Tendwire's semantic connector contract for answering the +current structured Claude decision on one worker. Connectors select public +option references or provide permitted write-in text. They never send pane IDs, +terminal IDs, raw keys, cursor movements, or calibration steps. Tendwire +validates the current prompt and owns the private Herdr calibration. + +## Pending payload + +A structured decision appears on a `pending_interactions` item as +`meta.decision`: + +```json +{ + "decision_ref": "decision-", + "kind": "single", + "prompt": "Choose a database", + "options": [ + {"ref": "1", "label": "Postgres"}, + {"ref": "2", "label": "SQLite"} + ], + "multi_select": false, + "question_count": 1 +} +``` + +`decision_ref` identifies one exact prompt instance and changes whenever the +prompt revision or its private source binding changes. Option `ref` values are +stable 1-based ordinals for that instance. Connectors must treat the whole +object as current-state data and fail closed when `question_count` is greater +than 1. + +Supported shapes are: + +- `single`: exactly one option, or nonempty write-in text; +- `plan`: exactly one option; +- `multi`: one or more options from a single question. + +Tendwire supports at most 11 source option rows. A source decision with more +than 11 rows, including a custom/write-in row beyond that boundary, is not +truncated. Unknown decision kinds, over-bound decisions, and multi-question +decisions do not produce `meta.decision` and cannot be sent. + +## Command request + +```json +{ + "schema_version": 1, + "action": "answer_decision", + "request_id": "connector-request-123", + "dry_run": false, + "target": {"worker_id": "worker-123"}, + "params": { + "decision_ref": "decision-", + "selection": {"option_refs": ["2"]} + } +} +``` + +`selection` contains exactly one of these forms: + +- `{"option_refs":[...]}`: every reference must exist in the current stored + options. `single` and `plan` require exactly one unique reference; `multi` + requires one or more unique references. +- `{"text":"..."}`: a nonempty write-in accepted only for `single`. + +Omitting `dry_run` means `dry_run: true`. A live answer therefore requires the +caller to set `dry_run: false` explicitly. Dry runs perform no pane operation +and write no mutation receipt. + +Before any pane operation, Tendwire freshly proves that the worker exists and +is open, the supplied reference is its current pending decision, the decision +shape is supported, and the selection is valid. A proven live success returns +`ok: true`, `status: "accepted"`, and a `terminal_accepted` disposition. + +The four typed validation failures are terminal for that attempted decision: + +- `unknown_worker`; +- `decision_not_pending`; +- `invalid_selection`; +- `unsupported_decision`. + +All fail before pane input. `unsupported_decision` covers multiple question +groups, more than 11 source rows, and unknown kinds. + +## Concurrency and retries + +Only one request may claim a decision at a time. A competing request receives +`ok: false`, `status: "answer_in_progress"`, with either `no_receipt` (before it +reserved a request) or `in_progress` (when its unsent reservation remains +recoverable). This status is retryable. It never creates a terminal receipt and +never causes a second pane mutation. + +If the winning request fails safely before sending, Tendwire releases its +decision claim so a loser may retry. An abandoned pre-send reservation and +claim can be taken over after their leases expire. Once sending may have +started, Tendwire fails closed with `request_state_uncertain` and does not +automatically resend. + +`request_id` uses the normal command deduplication contract. Repeating the same +canonical request ID replays its authoritative result without sending again; +reusing it for a different canonical mutation is rejected as +`duplicate_request`. + +## Paired adapter requirement + +The Tendwire and Herdres versions must be deployed as a matching pair that both +implement this contract. In particular, the paired Herdres adapter must emit a +single-question `AskUserQuestion` with `multiSelect: true` as a structured +`pending_decision` with `mode: "multi"`, `multi_select: true`, 1-based option +IDs, and no custom row. Older adapters that leave that prompt as an unstructured +`pending_interaction` cannot use semantic multi-select answering. Multi-question +prompts remain unstructured and unsupported. + +The Claude Code multi-select key behavior is a private backend assumption, not +part of this public contract. Its provisional calibration is isolated in +`src/tendwire/backends/herdr_decision.py` for live verification and retuning. diff --git a/scripts/sqlite_sidecar_race_benchmark.py b/scripts/sqlite_sidecar_race_benchmark.py index 226fb89..5d527a6 100755 --- a/scripts/sqlite_sidecar_race_benchmark.py +++ b/scripts/sqlite_sidecar_race_benchmark.py @@ -993,6 +993,13 @@ def _run_herdres_phase( if herdres_root.resolve() not in module_path.parents: raise RuntimeError("herdres_origin_failed") origin_ok = True + # The paired Herdres owns its turn page size (it changed 50 -> 100 in + # luminexord/herdres 31c3152); derive the expected command sequence from + # the paired checkout instead of hardcoding a value that breaks the + # pairing every time Herdres retunes it. + turn_page_limit = getattr(tendwire_client, "TURN_LIST_PAGE_LIMIT", 50) + if type(turn_page_limit) is not int or turn_page_limit < 1: + raise RuntimeError("herdres_turn_page_limit_invalid") runtime = source_sync.SyncRuntime( tendwire=tendwire_client.TendwireClient(timeout=10.0), telegram=telegram_delivery.TelegramClient(token="", dry_run=True), @@ -1043,7 +1050,7 @@ def _run_herdres_phase( "--schema-version", "2", "--limit", - "50", + str(turn_page_limit), "--json", ], ["--socket-path", str(socket_path), "pending", "--json"]) diff --git a/src/tendwire/backends/herdr_decision.py b/src/tendwire/backends/herdr_decision.py new file mode 100644 index 0000000..b9d06f0 --- /dev/null +++ b/src/tendwire/backends/herdr_decision.py @@ -0,0 +1,141 @@ +"""Translate semantic Claude decisions into private Herdr pane input. + +Calibration assumptions are deliberately confined to this backend module: + +* Claude Code displays single-choice and plan rows with 1-based decimal + shortcuts; typing the ordinal and then Enter chooses that row. +* A single-choice write-in row immediately follows the advertised options. + Its ordinal opens/focuses the text field without an intermediate Enter, so + Tendwire types ``N + 1`` and then submits the write-in text with Enter. +* Claude Code multi-select digit toggles are not treated as a supported + contract. Tendwire therefore assumes the cursor starts on row 1, Down/Up move + exactly one option row, Space toggles the current option without moving the + cursor, the Submit row immediately follows the final option, and Enter on + Submit submits. This provisional Claude Code behavior is isolated in + ``MULTI_SELECT_CALIBRATION`` below so live verification can retune it in one + place. +* Herdr's private ``pane.send_keys`` accepts decimal character keys plus + ``Down``, ``Up``, and ``Enter``. A literal multi-select Space is sent through + private ``pane.send_text``; write-in prose uses ``pane.send_input`` so Herdr + owns terminal text encoding and appends the final Enter atomically. + +These steps are internal calibration data. They are never accepted from a +connector and there is intentionally no public raw-key command action. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + + +MULTI_SELECT_CALIBRATION = { + "down": "Down", + "up": "Up", + "toggle_text": " ", + "submit": "Enter", +} + + +@dataclass(frozen=True) +class HerdrDecisionStep: + """One private, already-calibrated Herdr pane operation.""" + + operation: Literal["keys", "text", "input"] + keys: tuple[str, ...] = () + text: str | None = None + + def __post_init__(self) -> None: + if self.operation == "keys": + if not self.keys or self.text is not None: + raise ValueError("key calibration step requires only keys") + elif self.operation == "text": + if self.text != " " or self.keys: + raise ValueError("text calibration step requires one literal space") + elif self.operation == "input": + if not isinstance(self.text, str) or not self.text or self.keys != ("Enter",): + raise ValueError("input calibration step requires text plus Enter") + else: + raise ValueError("unsupported decision calibration operation") + + +def _digit_keys(value: int | str) -> tuple[str, ...]: + text = str(value) + if not text.isdigit() or int(text) < 1: + raise ValueError("decision ordinal must be a positive decimal") + return tuple(text) + + +def calibrate_decision_steps( + *, + kind: Literal["single", "multi", "plan"], + option_count: int, + option_refs: tuple[str, ...] = (), + text: str | None = None, +) -> tuple[HerdrDecisionStep, ...]: + """Return private pane operations for one validated semantic selection.""" + if ( + kind not in {"single", "multi", "plan"} + or not isinstance(option_count, int) + or isinstance(option_count, bool) + or option_count < 1 + ): + raise ValueError("invalid decision calibration context") + if text is not None: + if kind != "single" or option_refs or not isinstance(text, str) or not text: + raise ValueError("invalid decision write-in calibration") + return ( + HerdrDecisionStep("keys", keys=_digit_keys(option_count + 1)), + HerdrDecisionStep("input", keys=("Enter",), text=text), + ) + if not option_refs or len(option_refs) != len(set(option_refs)): + raise ValueError("decision option refs must be nonempty and unique") + ordinals: list[int] = [] + for ref in option_refs: + if not isinstance(ref, str) or not ref.isdigit(): + raise ValueError("decision option ref must be a decimal ordinal") + ordinal = int(ref) + if not 1 <= ordinal <= option_count: + raise ValueError("decision option ref is out of range") + ordinals.append(ordinal) + if kind in {"single", "plan"}: + if len(ordinals) != 1: + raise ValueError("single and plan decisions require one option") + return ( + HerdrDecisionStep( + "keys", + keys=(*_digit_keys(ordinals[0]), "Enter"), + ), + ) + + steps: list[HerdrDecisionStep] = [] + current_row = 1 + for ordinal in ordinals: + delta = ordinal - current_row + if delta: + direction = ( + MULTI_SELECT_CALIBRATION["down"] + if delta > 0 + else MULTI_SELECT_CALIBRATION["up"] + ) + steps.append(HerdrDecisionStep("keys", keys=(direction,) * abs(delta))) + steps.append( + HerdrDecisionStep( + "text", + text=MULTI_SELECT_CALIBRATION["toggle_text"], + ) + ) + current_row = ordinal + submit_navigation = (MULTI_SELECT_CALIBRATION["down"],) * ( + option_count + 1 - current_row + ) + steps.append( + HerdrDecisionStep( + "keys", + keys=( + *submit_navigation, + MULTI_SELECT_CALIBRATION["submit"], + ), + ) + ) + return tuple(steps) diff --git a/src/tendwire/backends/herdr_turns.py b/src/tendwire/backends/herdr_turns.py index 53ea526..b22cc49 100644 --- a/src/tendwire/backends/herdr_turns.py +++ b/src/tendwire/backends/herdr_turns.py @@ -386,8 +386,11 @@ def _extract_turn_payload(value: Any) -> Mapping[str, Any] | None: return value -_PENDING_MAX_CHOICES = 12 +PENDING_DECISION_MAX_OPTIONS = 11 _PENDING_TEXT_MAX = 2000 +_SINGLE_WRITE_IN_OPTION_IDS = frozenset( + {"custom", "other", "writein", "write_in", "write-in"} +) def _private_pending_revision(value: Mapping[str, Any]) -> str: @@ -409,7 +412,7 @@ def _pending_observation_from_turn(turn: Mapping[str, Any]) -> PendingObservatio return PendingObservation("read_succeeded_invalid_prompt") revision = _private_pending_revision(decision) question = redact_private_prompt_text( - decision.get("prompt"), + decision.get("prompt") or decision.get("question"), max_chars=_PENDING_TEXT_MAX, ) options = decision.get("options") @@ -417,8 +420,10 @@ def _pending_observation_from_turn(turn: Mapping[str, Any]) -> PendingObservatio options = [] if not isinstance(options, list): return PendingObservation("read_succeeded_invalid_prompt") + if len(options) > PENDING_DECISION_MAX_OPTIONS: + return PendingObservation("read_succeeded_unsupported_decision") choices: list[PendingObservedChoice] = [] - for ordinal, option in enumerate(options[:_PENDING_MAX_CHOICES], 1): + for ordinal, option in enumerate(options, 1): if not isinstance(option, Mapping): return PendingObservation("read_succeeded_invalid_prompt") label = redact_private_prompt_text( @@ -442,12 +447,95 @@ def _pending_observation_from_turn(turn: Mapping[str, Any]) -> PendingObservatio for option in options if isinstance(option, Mapping) } + raw_kind = str( + decision.get("kind") + or decision.get("tool_name") + or decision.get("name") + or "" + ).strip().lower().replace("-", "_") + compact_kind = raw_kind.replace("_", "") + raw_mode = ( + str(decision.get("mode") or "") + .strip() + .lower() + .replace("-", "_") + ) + compact_mode = raw_mode.replace("_", "") + if compact_kind not in { + "", + "askuserquestion", + "single", + "multi", + "multiselect", + "exitplanmode", + "plan", + } or compact_mode not in { + "", + "buttons", + "single", + "multi", + "multiselect", + "plan", + }: + return PendingObservation("read_succeeded_unsupported_decision") + raw_multi_select = decision.get( + "multi_select", + decision.get("multiSelect", False), + ) + if not isinstance(raw_multi_select, bool): + return PendingObservation("read_succeeded_invalid_prompt") + if ( + compact_mode == "plan" + or compact_kind in {"exitplanmode", "plan"} + or (not compact_kind and "approve" in option_ids) + ): + decision_kind: Literal["single", "multi", "plan"] = "plan" + elif ( + compact_mode in {"multi", "multiselect"} + or raw_multi_select + or compact_kind in {"multi", "multiselect"} + ): + decision_kind = "multi" + else: + decision_kind = "single" + if raw_multi_select is not (decision_kind == "multi"): + return PendingObservation("read_succeeded_unsupported_decision") + raw_question_count = decision.get("question_count") + if raw_question_count is None: + raw_questions = decision.get("questions") + raw_question_count = ( + len(raw_questions) if isinstance(raw_questions, list) else 1 + ) + if ( + not isinstance(raw_question_count, int) + or isinstance(raw_question_count, bool) + or raw_question_count < 1 + ): + return PendingObservation("read_succeeded_invalid_prompt") + decision_option_labels = [choice.label for choice in choices] + if ( + decision_kind == "single" + and options + and isinstance(options[len(decision_option_labels) - 1], Mapping) + and str( + options[len(decision_option_labels) - 1].get("id") or "" + ).strip().lower() + in _SINGLE_WRITE_IN_OPTION_IDS + ): + decision_option_labels.pop() + decision_options = tuple(decision_option_labels) + if not decision_options: + return PendingObservation("read_succeeded_invalid_prompt") return PendingObservation( "open_prompt", question=question, pending_kind="approval" if "approve" in option_ids else "question", choices=tuple(choices), revision_digest=revision, + decision_kind=decision_kind, + decision_options=decision_options, + decision_multi_select=decision_kind == "multi", + decision_question_count=raw_question_count, ) interaction = turn.get("pending_interaction") if interaction is not None: @@ -481,6 +569,24 @@ def _backend_pending_from_turn(turn: Mapping[str, Any]) -> dict[str, Any] | None observation = _pending_observation_from_turn(turn) if observation.kind != "open_prompt": return None + meta: dict[str, Any] = {"source": "backend"} + if observation.decision_kind is not None: + meta["decision"] = { + "decision_ref": ( + "decision-" + + stable_fingerprint( + {"decision_revision": observation.revision_digest} + ) + ), + "kind": observation.decision_kind, + "prompt": observation.question, + "options": [ + {"ref": str(ordinal), "label": label} + for ordinal, label in enumerate(observation.decision_options, 1) + ], + "multi_select": observation.decision_multi_select, + "question_count": observation.decision_question_count, + } return { "question": observation.question, "kind": observation.pending_kind or "question", @@ -488,7 +594,7 @@ def _backend_pending_from_turn(turn: Mapping[str, Any]) -> dict[str, Any] | None {"choice_id": choice.choice_id, "label": choice.label} for choice in observation.choices ], - "meta": {"source": "backend"}, + "meta": meta, } diff --git a/src/tendwire/cli.py b/src/tendwire/cli.py index 63bf6a6..bf542a7 100644 --- a/src/tendwire/cli.py +++ b/src/tendwire/cli.py @@ -1214,7 +1214,7 @@ def command_envelope_from_payload(config: Config, payload: str) -> CommandEnvelo if validation_error is not None: return CommandEnvelope.from_error(request, validation_error) - if request.action in {"send_instruction", "answer_pending"}: + if request.action in {"send_instruction", "answer_pending", "answer_decision"}: from .command_submission import submit_command return submit_command(config, payload) @@ -1267,7 +1267,10 @@ def _requires_daemon_for_mutating_command(config: Config, payload: str) -> Any | validation_error = validate_request(request) if validation_error is not None: return None - if request.action in {"send_instruction", "answer_pending"} and not request.dry_run: + if ( + request.action in {"send_instruction", "answer_pending", "answer_decision"} + and not request.dry_run + ): return request return None @@ -1335,7 +1338,8 @@ def cmd_command( parse_error is None and validation_error is None and parsed_request is not None - and parsed_request.action in {"send_instruction", "answer_pending"} + and parsed_request.action + in {"send_instruction", "answer_pending", "answer_decision"} and parsed_request.dry_run ) daemon_required_request = _requires_daemon_for_mutating_command(config, payload) diff --git a/src/tendwire/command_submission.py b/src/tendwire/command_submission.py index cb0401d..228b362 100644 --- a/src/tendwire/command_submission.py +++ b/src/tendwire/command_submission.py @@ -13,23 +13,29 @@ from .core.commands import ( COMMAND_ENVELOPE_SCHEMA_VERSION, DISPOSITION_IN_PROGRESS, + DISPOSITION_NO_RECEIPT, DISPOSITION_TERMINAL_ACCEPTED, DISPOSITION_TERMINAL_REJECTED, DISPOSITION_TERMINAL_UNCERTAIN, STATUS_ACCEPTED, + STATUS_ANSWER_IN_PROGRESS, STATUS_AMBIGUOUS_BACKEND_TARGET, STATUS_AMBIGUOUS_TARGET, STATUS_BACKEND_FAILED, STATUS_BACKEND_UNAVAILABLE, STATUS_BACKEND_UNSUPPORTED, STATUS_DRY_RUN, + STATUS_DECISION_NOT_PENDING, STATUS_DUPLICATE_REQUEST, + STATUS_INVALID_SELECTION, STATUS_NOT_FOUND, STATUS_PENDING, STATUS_REJECTED, STATUS_REQUEST_STATE_UNCERTAIN, STATUS_RESOLVED, STATUS_STALE_TARGET, + STATUS_UNKNOWN_WORKER, + STATUS_UNSUPPORTED_DECISION, CanonicalMutation, CommandEnvelope, CommandRequest, @@ -44,10 +50,13 @@ ) from .core.models import BackendHealth, Snapshot, Worker, WorkerBinding from .core.projector import project_from_observations +from .backends.herdr_decision import calibrate_decision_steps from .store.sqlite import ( abandon_backend_pending_choice_claim, + abandon_command_request_reservation, backend_pending_choice_terminal_effect, claim_backend_pending_choice, + claim_backend_pending_decision, command_pending_turn_terminal_effect, command_reservation_is_live, envelope_to_receipt_json, @@ -59,11 +68,14 @@ reserve_command_request, reserve_terminal_command_replay, start_backend_pending_choice_send, + start_backend_pending_decision_send, ) HERDR_BACKEND = "herdr" -_MUTATING_ACTIONS = frozenset({"send_instruction", "answer_pending"}) +_MUTATING_ACTIONS = frozenset( + {"send_instruction", "answer_pending", "answer_decision"} +) _LEGACY_V0_REPLAY_WORKER_ID = "legacy-v0-replay-only" _PENDING_CHANGED_MESSAGE = "pending interaction changed or is no longer answerable" _DISALLOWED_SEND_STATUSES = frozenset({"closed", "failed", "unknown"}) @@ -394,6 +406,27 @@ def _request_in_progress(request: CommandRequest) -> CommandEnvelope: ) +def _answer_in_progress( + request: CommandRequest, + *, + receipt_reserved: bool = False, +) -> CommandEnvelope: + return CommandEnvelope.from_result( + request, + ok=False, + status=STATUS_ANSWER_IN_PROGRESS, + disposition=( + DISPOSITION_IN_PROGRESS + if receipt_reserved + else DISPOSITION_NO_RECEIPT + ), + error=error_value( + STATUS_ANSWER_IN_PROGRESS, + "another request is currently answering this decision", + ), + ) + + def _duplicate_request(request: CommandRequest) -> CommandEnvelope: return CommandEnvelope.from_result( request, @@ -474,6 +507,82 @@ def _same_pending_route(left: Any, right: Any) -> bool: ) +def _decision_failure_envelope( + request: CommandRequest, + status: str, +) -> CommandEnvelope: + messages = { + STATUS_ANSWER_IN_PROGRESS: "another request is currently answering this decision", + STATUS_DECISION_NOT_PENDING: "decision is not the worker's current pending decision", + STATUS_UNKNOWN_WORKER: "target worker does not exist or is not open", + STATUS_INVALID_SELECTION: "selection is invalid for the current decision", + STATUS_UNSUPPORTED_DECISION: "multi-question decisions are not supported", + } + return CommandEnvelope.from_result( + request, + ok=False, + status=status, + error=error_value(status, messages[status]), + ) + + +def _decision_claim_has_exact_route(claim: Any) -> bool: + return ( + isinstance(getattr(claim, "worker_id", None), str) + and bool(claim.worker_id) + and isinstance(getattr(claim, "worker_fingerprint", None), str) + and bool(claim.worker_fingerprint) + and isinstance(getattr(claim, "binding_private_fingerprint", None), str) + and bool(claim.binding_private_fingerprint) + and isinstance(getattr(claim, "turn_target_value", None), str) + and bool(claim.turn_target_value.strip()) + and isinstance(getattr(claim, "decision_ref", None), str) + and bool(claim.decision_ref) + and getattr(claim, "decision_kind", None) in {"single", "multi", "plan"} + and isinstance(getattr(claim, "option_count", None), int) + and not isinstance(claim.option_count, bool) + and claim.option_count >= 1 + and isinstance(getattr(claim, "option_refs", None), tuple) + and ( + (claim.text is None and bool(claim.option_refs)) + or ( + isinstance(claim.text, str) + and bool(claim.text) + and not claim.option_refs + ) + ) + ) + + +def _same_decision_route(left: Any, right: Any) -> bool: + return ( + _decision_claim_has_exact_route(left) + and _decision_claim_has_exact_route(right) + and ( + left.worker_id, + left.worker_fingerprint, + left.binding_private_fingerprint, + left.turn_target_value, + left.decision_ref, + left.decision_kind, + left.option_count, + left.option_refs, + left.text, + ) + == ( + right.worker_id, + right.worker_fingerprint, + right.binding_private_fingerprint, + right.turn_target_value, + right.decision_ref, + right.decision_kind, + right.option_count, + right.option_refs, + right.text, + ) + ) + + class PreSendCertainty(Enum): """How a pre-send failure must be classified before any external mutation. @@ -537,6 +646,25 @@ def _abandon_pending_claim(config: Config, claim_token: str | None) -> bool: return False +def _abandon_request_reservation( + config: Config, + request: CommandRequest, + reservation: ReservedCommandMutation, +) -> bool: + if config.db_path is None: + return False + try: + return abandon_command_request_reservation( + config.db_path, + host_id=config.host_id, + request_id=request.request_id or "", + canonical_fingerprint=reservation.canonical.fingerprint, + owner_token=reservation.owner_token, + ) + except Exception: + return False + + def _connect_socket( config: Config, request: CommandRequest, @@ -1452,6 +1580,250 @@ def _answer_pending( ) +def _validate_pending_decision( + config: Config, + request: CommandRequest, +) -> Any | PreSendFailure: + if config.db_path is None: + return _safe_transient_pre_send( + _backend_unavailable(request, "pending state store is unavailable") + ) + params = request.params or {} + target = request.target or {} + try: + validated = claim_backend_pending_decision( + config.db_path, + config.host_id, + str(target.get("worker_id") or ""), + str(params.get("decision_ref") or ""), + params.get("selection") + if isinstance(params.get("selection"), Mapping) + else {}, + claim=False, + ) + except Exception: + return _safe_transient_pre_send( + _backend_unavailable(request, "pending state store is unavailable") + ) + if validated.status == "validated" and _decision_claim_has_exact_route(validated): + return validated + status = { + "already_claimed": STATUS_ANSWER_IN_PROGRESS, + "unknown_worker": STATUS_UNKNOWN_WORKER, + "invalid_selection": STATUS_INVALID_SELECTION, + "unsupported_decision": STATUS_UNSUPPORTED_DECISION, + }.get(validated.status, STATUS_DECISION_NOT_PENDING) + return _permanent_pre_send(_decision_failure_envelope(request, status)) + + +def _claim_pending_decision( + config: Config, + request: CommandRequest, + validated: Any, +) -> Any | CommandEnvelope: + assert config.db_path is not None + params = request.params or {} + target = request.target or {} + try: + claim = claim_backend_pending_decision( + config.db_path, + config.host_id, + str(target.get("worker_id") or ""), + str(params.get("decision_ref") or ""), + params.get("selection") + if isinstance(params.get("selection"), Mapping) + else {}, + claim=True, + ) + except Exception: + return _backend_uncertain(request, "pending decision claim state is uncertain") + if ( + claim.status == "claimed" + and isinstance(claim.claim_token, str) + and claim.claim_token + ): + if _same_decision_route(validated, claim): + return claim + status = { + "already_claimed": STATUS_ANSWER_IN_PROGRESS, + "unknown_worker": STATUS_UNKNOWN_WORKER, + "invalid_selection": STATUS_INVALID_SELECTION, + "unsupported_decision": STATUS_UNSUPPORTED_DECISION, + }.get(claim.status, STATUS_DECISION_NOT_PENDING) + return _decision_failure_envelope(request, status) + + +def _submit_decision_calibration( + client: Any, + pane_id: str, + decision: Any, + *, + timeout: float, +) -> None: + steps = calibrate_decision_steps( + kind=decision.decision_kind, + option_count=decision.option_count, + option_refs=decision.option_refs, + text=decision.text, + ) + for step in steps: + if step.operation == "keys": + _socket_request( + client, + "pane.send_keys", + {"pane_id": pane_id, "keys": list(step.keys)}, + timeout=timeout, + ) + elif step.operation == "text": + _socket_request( + client, + "pane.send_text", + {"pane_id": pane_id, "text": step.text}, + timeout=timeout, + ) + else: + _socket_request( + client, + "pane.send_input", + {"pane_id": pane_id, "text": step.text, "keys": list(step.keys)}, + timeout=timeout, + ) + + +def _decision_public_result( + request: CommandRequest, + claim: Any, +) -> dict[str, Any]: + return { + "target": {"worker_id": claim.worker_id}, + "decision": {"decision_ref": (request.params or {}).get("decision_ref")}, + "delivery_state": "submitted", + "transport_state": "submitted", + "observed_pending_state": "pending_observation", + } + + +def _answer_decision( + config: Config, + request: CommandRequest, + validated: Any, + reservation: ReservedCommandMutation, + client: Any, +) -> CommandEnvelope: + assert config.db_path is not None + claim = _claim_pending_decision(config, request, validated) + if isinstance(claim, CommandEnvelope): + _close_socket_client(client) + if claim.status == STATUS_ANSWER_IN_PROGRESS: + _abandon_request_reservation(config, request, reservation) + return _answer_in_progress(request, receipt_reserved=True) + return _finish_before_send(config, request, reservation, claim) + claim_token = claim.claim_token + + send_start_error = _mark_request_send_started( + config, + request, + reservation, + binding_fingerprint=claim.binding_private_fingerprint, + ) + if send_start_error is not None: + _close_socket_client(client) + claim_released = _abandon_pending_claim(config, claim_token) + if send_start_error.status == STATUS_PENDING and not claim_released: + return _finish_before_send( + config, + request, + reservation, + _backend_uncertain( + request, + "pending decision claim could not be safely released", + ), + ) + return send_start_error + + try: + started = start_backend_pending_decision_send( + config.db_path, + config.host_id, + claim_token, + ) + except Exception: + _close_socket_client(client) + _abandon_pending_claim(config, claim_token) + return _finish_request( + config, + request, + reservation, + _backend_uncertain(request, "pending decision start state is uncertain"), + expected_state="send_started", + terminal_state="uncertain", + terminal_effect=_uncertain_pending_effect(config, claim_token), + ) + if getattr(started, "status", None) != "started" or not _same_decision_route(claim, started): + _close_socket_client(client) + _abandon_pending_claim(config, claim_token) + return _finish_request( + config, + request, + reservation, + _backend_uncertain( + request, + "pending decision state is uncertain after send start", + ), + expected_state="send_started", + terminal_state="uncertain", + terminal_effect=_uncertain_pending_effect(config, claim_token), + ) + + try: + _submit_decision_calibration( + client, + started.turn_target_value.strip(), + started, + timeout=config.herdr_timeout_seconds, + ) + except Exception: # noqa: BLE001 + return _finish_request( + config, + request, + reservation, + _backend_uncertain( + request, + "Herdr decision input state is uncertain after send start", + ), + expected_state="send_started", + terminal_state="uncertain", + terminal_effect=_uncertain_pending_effect(config, claim_token), + ) + finally: + _close_socket_client(client) + + accepted = CommandEnvelope.from_result( + request, + ok=True, + status=STATUS_ACCEPTED, + disposition=DISPOSITION_TERMINAL_ACCEPTED, + result=_decision_public_result(request, started), + ) + try: + effect = backend_pending_choice_terminal_effect( + host_id=config.host_id, + claim_token=claim_token, + accepted=True, + ) + except Exception: + return _recover_request(config, request, reservation.canonical) + return _finish_request( + config, + request, + reservation, + accepted, + expected_state="send_started", + terminal_state="accepted", + terminal_effect=effect, + ) + + def _execute_non_mutating(config: Config, request: CommandRequest) -> CommandEnvelope: if request.action == "noop": return execute_command(request, CommandContext(host_id=config.host_id, workers=[])) @@ -1477,6 +1849,17 @@ def _mutation_dry_run(request: CommandRequest) -> CommandEnvelope: }, ) params = request.params or {} + if request.action == "answer_decision": + return CommandEnvelope.from_result( + request, + ok=True, + status=STATUS_DRY_RUN, + result={ + "target": dict(request.target or {}), + "decision": {"decision_ref": params.get("decision_ref")}, + "delivery_state": "not_submitted", + }, + ) return CommandEnvelope.from_result( request, ok=True, @@ -1569,6 +1952,11 @@ def _proven_replay_worker_id( return stored_worker_id or _LEGACY_V0_REPLAY_WORKER_ID if not stored_worker_id: return _receipt_malformed(request) + if request.action == "answer_decision": + explicit_worker_id = _direct_replay_worker_id(request) + if explicit_worker_id != stored_worker_id: + return _duplicate_request(request) + return stored_worker_id if request.action != "send_instruction": return stored_worker_id @@ -1824,6 +2212,11 @@ def submit_command( ) answer_pre_send: PreSendFailure | None = None + validate_answer = ( + _validate_pending_decision + if request.action == "answer_decision" + else _validate_pending_choice + ) if takeover is not None: # Re-driving an abandoned answer reservation: the receipt already fixed # which worker this request answers, so a pending interaction that now @@ -1833,17 +2226,17 @@ def submit_command( request, public_worker_id=existing_worker_id, ) - validated = _validate_pending_choice(config, request) + validated = validate_answer(config, request) if isinstance(validated, PreSendFailure): answer_pre_send = validated elif validated.worker_id != existing_worker_id: answer_pre_send = _permanent_pre_send(_duplicate_request(request)) else: - validated = _validate_pending_choice(config, request) + validated = validate_answer(config, request) if isinstance(validated, PreSendFailure): # No reservation exists yet, so neither a transient nor a permanent # validation failure writes a receipt here. Return it directly. - if health_error is not None: + if health_error is not None and request.action != "answer_decision": return health_error return validated.envelope canonical = build_canonical_mutation( @@ -1857,6 +2250,14 @@ def submit_command( if takeover is not None: return _request_in_progress(request) return answer_pre_send.envelope + if ( + answer_pre_send is not None + and answer_pre_send.envelope.status == STATUS_ANSWER_IN_PROGRESS + ): + # Another request owns the still-live decision claim. Keep this + # abandoned reservation nonterminal so it can take over after that + # claim is released or expires. + return _answer_in_progress(request, receipt_reserved=True) client_or_error: Any | CommandEnvelope | None = None if answer_pre_send is None and health_error is None: @@ -1888,6 +2289,14 @@ def submit_command( health_error, ) assert client_or_error is not None + if request.action == "answer_decision": + return _answer_decision( + config, + request, + validated, + reservation, + client_or_error, + ) return _answer_pending( config, request, diff --git a/src/tendwire/core/commands.py b/src/tendwire/core/commands.py index 048eba7..440c992 100644 --- a/src/tendwire/core/commands.py +++ b/src/tendwire/core/commands.py @@ -34,7 +34,14 @@ COMMAND_ENVELOPE_SCHEMA_VERSION = 2 ALLOWED_ACTIONS = frozenset( - {"noop", "read_snapshot", "resolve_target", "send_instruction", "answer_pending"} + { + "noop", + "read_snapshot", + "resolve_target", + "send_instruction", + "answer_pending", + "answer_decision", + } ) REQUEST_ALLOWED_FIELDS = frozenset( {"schema_version", "action", "request_id", "dry_run", "target", "instruction", "params"} @@ -58,6 +65,11 @@ STATUS_REQUEST_STATE_UNCERTAIN = "request_state_uncertain" STATUS_INVALID_REQUEST = "invalid_request" STATUS_PENDING = "pending" +STATUS_ANSWER_IN_PROGRESS = "answer_in_progress" +STATUS_DECISION_NOT_PENDING = "decision_not_pending" +STATUS_UNKNOWN_WORKER = "unknown_worker" +STATUS_INVALID_SELECTION = "invalid_selection" +STATUS_UNSUPPORTED_DECISION = "unsupported_decision" CommandDisposition = Literal[ "no_receipt", @@ -101,6 +113,11 @@ STATUS_REQUEST_STATE_UNCERTAIN, STATUS_INVALID_REQUEST, STATUS_PENDING, + STATUS_ANSWER_IN_PROGRESS, + STATUS_DECISION_NOT_PENDING, + STATUS_UNKNOWN_WORKER, + STATUS_INVALID_SELECTION, + STATUS_UNSUPPORTED_DECISION, } ) @@ -116,6 +133,10 @@ STATUS_AMBIGUOUS_BACKEND_TARGET, STATUS_BACKEND_FAILED, STATUS_DUPLICATE_REQUEST, + STATUS_DECISION_NOT_PENDING, + STATUS_UNKNOWN_WORKER, + STATUS_INVALID_SELECTION, + STATUS_UNSUPPORTED_DECISION, } ) @@ -133,6 +154,11 @@ STATUS_BACKEND_UNSUPPORTED, STATUS_AMBIGUOUS_BACKEND_TARGET, STATUS_BACKEND_FAILED, + STATUS_ANSWER_IN_PROGRESS, + STATUS_DECISION_NOT_PENDING, + STATUS_UNKNOWN_WORKER, + STATUS_INVALID_SELECTION, + STATUS_UNSUPPORTED_DECISION, } ) @@ -141,6 +167,7 @@ DRY_RUN_MUTATION_NO_RECEIPT_REJECTION_STATUSES = frozenset( { STATUS_INVALID_REQUEST, + STATUS_INVALID_SELECTION, STATUS_REJECTED, STATUS_NOT_FOUND, STATUS_AMBIGUOUS_TARGET, @@ -162,6 +189,7 @@ ANSWER_PENDING_PARAM_FIELDS = frozenset( {"pending_id", "pending_fingerprint", "choice_id"} ) +ANSWER_DECISION_PARAM_FIELDS = frozenset({"decision_ref", "selection"}) # Connector, low-level terminal, routing, and private fields rejected anywhere in a request. FORBIDDEN_REQUEST_FIELDS = FORBIDDEN_FIELD_NAMES @@ -432,8 +460,10 @@ def build_canonical_mutation( """ if not isinstance(request, CommandRequest): raise TypeError("request must be a CommandRequest") - if request.action not in {"send_instruction", "answer_pending"}: - raise ValueError("canonical mutations require send_instruction or answer_pending") + if request.action not in {"send_instruction", "answer_pending", "answer_decision"}: + raise ValueError( + "canonical mutations require send_instruction, answer_pending, or answer_decision" + ) if request.dry_run is not False: raise ValueError("canonical mutations require a non-dry-run request") request_error = validate_request(request) @@ -451,7 +481,7 @@ def build_canonical_mutation( "instruction": {"text": request.instruction["text"]}, "options": {}, } - else: + elif request.action == "answer_pending": assert request.params is not None canonical_payload = { "canonical_version": CANONICAL_MUTATION_VERSION, @@ -464,6 +494,28 @@ def build_canonical_mutation( }, "options": {}, } + else: + assert request.params is not None + selection = request.params["selection"] + if "option_refs" in selection: + canonical_selection: dict[str, Any] = { + "option_refs": sorted( + selection["option_refs"], + key=lambda ref: int(ref) if str(ref).isdigit() else -1, + ) + } + else: + canonical_selection = {"text": selection["text"]} + canonical_payload = { + "canonical_version": CANONICAL_MUTATION_VERSION, + "action": "answer_decision", + "target": {"worker_id": public_worker_id}, + "decision": { + "decision_ref": request.params["decision_ref"], + "selection": canonical_selection, + }, + "options": {}, + } return CanonicalMutation( canonical_version=CANONICAL_MUTATION_VERSION, @@ -510,8 +562,10 @@ def build_selector_proof(request: CommandRequest) -> str: """ if not isinstance(request, CommandRequest): raise TypeError("request must be a CommandRequest") - if request.action not in {"send_instruction", "answer_pending"}: - raise ValueError("selector proofs require send_instruction or answer_pending") + if request.action not in {"send_instruction", "answer_pending", "answer_decision"}: + raise ValueError( + "selector proofs require send_instruction, answer_pending, or answer_decision" + ) if request.dry_run is not False: raise ValueError("selector proofs require a non-dry-run request") request_error = validate_request(request) @@ -580,7 +634,7 @@ def validate_request(request: CommandRequest) -> dict[str, Any] | None: details={"field": "action", "allowed": sorted(ALLOWED_ACTIONS)}, ) if ( - request.action in {"send_instruction", "answer_pending"} + request.action in {"send_instruction", "answer_pending", "answer_decision"} and request.dry_run is False and not is_valid_request_id(request.request_id) ): @@ -669,6 +723,70 @@ def validate_request(request: CommandRequest) -> dict[str, Any] | None: details={"field": f"params.{field}"}, ) + if request.action == "answer_decision": + if request.target is None or set(request.target) != {"worker_id"}: + return error_value( + STATUS_INVALID_REQUEST, + "answer_decision requires exactly target.worker_id", + details={"field": "target"}, + ) + worker_id = request.target.get("worker_id") + if not isinstance(worker_id, str) or not worker_id.strip(): + return error_value( + STATUS_INVALID_REQUEST, + "answer_decision requires nonblank target.worker_id", + details={"field": "target.worker_id"}, + ) + if request.instruction is not None: + return error_value( + STATUS_INVALID_REQUEST, + "answer_decision does not accept an instruction", + details={"field": "instruction"}, + ) + if not isinstance(request.params, dict) or set(request.params) != ANSWER_DECISION_PARAM_FIELDS: + return error_value( + STATUS_INVALID_REQUEST, + "answer_decision params must contain exactly decision_ref and selection", + details={"field": "params"}, + ) + decision_ref = request.params.get("decision_ref") + if not isinstance(decision_ref, str) or not decision_ref.strip(): + return error_value( + STATUS_INVALID_REQUEST, + "answer_decision requires nonblank params.decision_ref", + details={"field": "params.decision_ref"}, + ) + selection = request.params.get("selection") + if not isinstance(selection, Mapping) or len(selection) != 1: + return error_value( + STATUS_INVALID_SELECTION, + "selection must contain exactly one selection form", + details={"field": "params.selection"}, + ) + if set(selection) == {"option_refs"}: + option_refs = selection.get("option_refs") + if not isinstance(option_refs, list) or not option_refs or any( + not isinstance(ref, str) or not ref.strip() for ref in option_refs + ): + return error_value( + STATUS_INVALID_SELECTION, + "selection.option_refs must be a nonempty array of strings", + details={"field": "params.selection.option_refs"}, + ) + elif set(selection) == {"text"}: + if validate_instruction_text(selection.get("text")) is not None: + return error_value( + STATUS_INVALID_SELECTION, + "selection.text must be nonempty safe text", + details={"field": "params.selection.text"}, + ) + else: + return error_value( + STATUS_INVALID_SELECTION, + "selection must contain option_refs or text", + details={"field": "params.selection"}, + ) + return None @@ -788,7 +906,11 @@ def __post_init__(self) -> None: f"command envelope schema_version must be {COMMAND_ENVELOPE_SCHEMA_VERSION}" ) - mutating = self.action in {"send_instruction", "answer_pending"} + mutating = self.action in { + "send_instruction", + "answer_pending", + "answer_decision", + } live_mutation = mutating and self.dry_run is False if live_mutation and not is_valid_request_id(self.request_id): raise ValueError("non-dry-run mutation requires a valid request_id") @@ -809,12 +931,15 @@ def __post_init__(self) -> None: if self.ok and clean_error is not None: raise ValueError("successful command envelope must not include an error") - if self.disposition == DISPOSITION_NO_RECEIPT: if live_mutation: valid_no_receipt_tuple = ( not self.ok and self.status in LIVE_MUTATION_NO_RECEIPT_REJECTION_STATUSES + and ( + self.status != STATUS_ANSWER_IN_PROGRESS + or self.action == "answer_decision" + ) ) elif mutating: valid_no_receipt_tuple = ( @@ -829,7 +954,16 @@ def __post_init__(self) -> None: if not valid_no_receipt_tuple: raise ValueError("no_receipt disposition has an inconsistent command tuple") elif self.disposition == DISPOSITION_IN_PROGRESS: - if not mutating or self.dry_run or self.ok or self.status != STATUS_PENDING: + if ( + not mutating + or self.dry_run + or self.ok + or self.status not in {STATUS_PENDING, STATUS_ANSWER_IN_PROGRESS} + or ( + self.status == STATUS_ANSWER_IN_PROGRESS + and self.action != "answer_decision" + ) + ): raise ValueError("in_progress disposition has an inconsistent command tuple") elif self.disposition == DISPOSITION_TERMINAL_ACCEPTED: if not mutating or self.dry_run or not self.ok or self.status != STATUS_ACCEPTED: @@ -850,6 +984,16 @@ def __post_init__(self) -> None: or self.status != STATUS_REQUEST_STATE_UNCERTAIN ): raise ValueError("terminal_uncertain disposition has an inconsistent command tuple") + if self.status == STATUS_ANSWER_IN_PROGRESS and not ( + self.action == "answer_decision" + and live_mutation + and not self.ok + and self.disposition + in {DISPOSITION_NO_RECEIPT, DISPOSITION_IN_PROGRESS} + ): + raise ValueError( + "answer_in_progress has an inconsistent command tuple" + ) def to_dict(self) -> dict[str, Any]: return { @@ -939,7 +1083,11 @@ def from_error(cls, request: CommandRequest | None, error: dict[str, Any]) -> "C result=None, error=error, ) - mutating = request.action in {"send_instruction", "answer_pending"} + mutating = request.action in { + "send_instruction", + "answer_pending", + "answer_decision", + } valid_mutation_id = is_valid_request_id(request.request_id) request_id = ( request.request_id diff --git a/src/tendwire/core/models.py b/src/tendwire/core/models.py index 0b85378..33fdc05 100644 --- a/src/tendwire/core/models.py +++ b/src/tendwire/core/models.py @@ -474,6 +474,7 @@ "label", "message", "name", + "prompt", "question", "raw_status", "reason", diff --git a/src/tendwire/core/turns.py b/src/tendwire/core/turns.py index c3b2d3a..089b2f4 100644 --- a/src/tendwire/core/turns.py +++ b/src/tendwire/core/turns.py @@ -1577,6 +1577,7 @@ def _is_pending_routing_meta_key(key: Any) -> bool: "open_prompt", "read_succeeded_no_prompt", "read_succeeded_invalid_prompt", + "read_succeeded_unsupported_decision", "read_failed", "worker_authoritatively_absent", ] @@ -1615,12 +1616,17 @@ class PendingObservation: pending_kind: str | None = None choices: tuple[PendingObservedChoice, ...] = () revision_digest: str | None = None + decision_kind: Literal["single", "multi", "plan"] | None = None + decision_options: tuple[str, ...] = () + decision_multi_select: bool = False + decision_question_count: int = 0 def __post_init__(self) -> None: if self.kind not in { "open_prompt", "read_succeeded_no_prompt", "read_succeeded_invalid_prompt", + "read_succeeded_unsupported_decision", "read_failed", "worker_authoritatively_absent", }: @@ -1648,11 +1654,44 @@ def __post_init__(self) -> None: or not self.pending_kind.strip() ): raise ValueError("invalid pending observation prompt kind") + if self.decision_kind is None: + if ( + self.decision_options + or self.decision_multi_select + or self.decision_question_count != 0 + ): + raise ValueError("non-decision prompt cannot carry decision data") + else: + if self.decision_kind not in {"single", "multi", "plan"}: + raise ValueError("invalid pending observation decision kind") + if ( + not isinstance(self.decision_options, tuple) + or not self.decision_options + or not all( + isinstance(label, str) and label.strip() + for label in self.decision_options + ) + ): + raise ValueError("decision prompt requires options") + if self.decision_multi_select is not ( + self.decision_kind == "multi" + ): + raise ValueError("decision multi-select flag does not match kind") + if ( + not isinstance(self.decision_question_count, int) + or isinstance(self.decision_question_count, bool) + or self.decision_question_count < 1 + ): + raise ValueError("invalid decision question count") elif ( self.question is not None or self.pending_kind is not None or self.choices or self.revision_digest is not None + or self.decision_kind is not None + or self.decision_options + or self.decision_multi_select + or self.decision_question_count != 0 ): raise ValueError("non-open pending observation cannot carry prompt data") diff --git a/src/tendwire/store/sqlite.py b/src/tendwire/store/sqlite.py index 4f3a89e..137df0e 100644 --- a/src/tendwire/store/sqlite.py +++ b/src/tendwire/store/sqlite.py @@ -54,7 +54,7 @@ verify_created_private_sqlite_replacement_at, verify_entry_identity, ) -from ..core.commands import CommandEnvelope, is_selector_proof +from ..core.commands import CommandEnvelope, is_selector_proof, validate_instruction_text from ..core.models import ( FINGERPRINT_HEX_CHARS, SCHEMA_VERSION, @@ -193,6 +193,50 @@ class BackendPendingChoiceSend: picker_ordinal: int | None = None +@dataclass(frozen=True) +class BackendPendingDecisionClaim: + status: Literal[ + "claimed", + "validated", + "unknown_worker", + "decision_not_pending", + "invalid_selection", + "unsupported_decision", + "already_claimed", + ] + claim_token: str | None = None + worker_id: str | None = None + worker_fingerprint: str | None = None + binding_private_fingerprint: str | None = None + turn_target_value: str | None = None + decision_ref: str | None = None + decision_kind: Literal["single", "multi", "plan"] | None = None + option_count: int | None = None + option_refs: tuple[str, ...] = () + text: str | None = None + + +@dataclass(frozen=True) +class BackendPendingDecisionSend: + status: Literal[ + "started", + "not_found", + "stale", + "changed", + "binding_changed", + "already_started", + ] + worker_id: str | None = None + worker_fingerprint: str | None = None + binding_private_fingerprint: str | None = None + turn_target_value: str | None = None + decision_ref: str | None = None + decision_kind: Literal["single", "multi", "plan"] | None = None + option_count: int | None = None + option_refs: tuple[str, ...] = () + text: str | None = None + + @dataclass(frozen=True) class SnapshotRetentionPolicy: """Database-wide snapshot history bounds.""" @@ -15625,6 +15669,57 @@ def _update_persisted_turn_row( return material_changed, item +def _normalize_pending_decision_meta(value: Any) -> dict[str, Any]: + """Validate the connector-facing semantic decision contract.""" + if not isinstance(value, Mapping) or set(value) != { + "decision_ref", + "kind", + "prompt", + "options", + "multi_select", + "question_count", + }: + raise ValueError("invalid backend pending decision") + decision_ref = value.get("decision_ref") + kind = value.get("kind") + prompt = value.get("prompt") + options = value.get("options") + multi_select = value.get("multi_select") + question_count = value.get("question_count") + if ( + type(decision_ref) is not str + or not decision_ref.strip() + or kind not in {"single", "multi", "plan"} + or type(prompt) is not str + or not prompt.strip() + or not isinstance(options, list) + or not options + or not isinstance(multi_select, bool) + or multi_select is not (kind == "multi") + or not isinstance(question_count, int) + or isinstance(question_count, bool) + or question_count < 1 + ): + raise ValueError("invalid backend pending decision") + normalized_options: list[dict[str, str]] = [] + for ordinal, option in enumerate(options, 1): + if not isinstance(option, Mapping) or set(option) != {"ref", "label"}: + raise ValueError("invalid backend pending decision option") + ref = option.get("ref") + label = option.get("label") + if ref != str(ordinal) or type(label) is not str or not label.strip(): + raise ValueError("invalid backend pending decision option") + normalized_options.append({"ref": ref, "label": label}) + return { + "decision_ref": decision_ref, + "kind": kind, + "prompt": prompt, + "options": normalized_options, + "multi_select": multi_select, + "question_count": question_count, + } + + def _normalize_backend_pending_payload( pending: Mapping[str, Any], choice_routes: tuple[tuple[str, int], ...], @@ -15673,11 +15768,16 @@ def _normalize_backend_pending_payload( type(ordinal) is not int or ordinal < 1 for ordinal in route_map.values() ): raise ValueError("backend pending choices do not match private routes") + normalized_meta = sanitize_public_mapping(meta) + if "decision" in normalized_meta: + normalized_meta["decision"] = _normalize_pending_decision_meta( + normalized_meta["decision"] + ) normalized = { "question": question, "kind": kind, "choices": normalized_choices, - "meta": sanitize_public_mapping(meta), + "meta": normalized_meta, } return normalized, _canonical_json(route_map) @@ -15894,6 +15994,60 @@ def _apply_backend_pending_observation_conn( ) return True + if observation.kind == "read_succeeded_unsupported_decision": + unsupported_payload = _canonical_json({"unsupported_decision": True}) + changed = row is None or ( + str(row[0]), + str(row[5]), + str(row[6]), + stored_binding, + stored_target, + ) != ( + unsupported_payload, + "invalid", + "fresh", + effective_binding, + effective_target, + ) + conn.execute( + """ + INSERT INTO backend_pending ( + host_id, worker_id, payload_json, observed_at, revision_digest, + choice_routes_json, binding_private_fingerprint, + observed_turn_target_value, observation_state, freshness, + last_success_at, last_failure_at, grace_deadline, updated_at + ) VALUES (?, ?, ?, ?, '', '{}', ?, ?, 'invalid', 'fresh', + ?, NULL, NULL, ?) + ON CONFLICT(host_id, worker_id) DO UPDATE SET + payload_json = excluded.payload_json, + observed_at = excluded.observed_at, + revision_digest = '', choice_routes_json = '{}', + binding_private_fingerprint = + excluded.binding_private_fingerprint, + observed_turn_target_value = + excluded.observed_turn_target_value, + observation_state = 'invalid', freshness = 'fresh', + last_success_at = excluded.last_success_at, + last_failure_at = NULL, grace_deadline = NULL, + updated_at = excluded.updated_at + """, + ( + *key, + unsupported_payload, + current_time, + effective_binding, + effective_target, + current_time, + current_time, + ), + ) + conn.execute( + "DELETE FROM backend_pending_claims " + "WHERE host_id = ? AND worker_id = ?", + key, + ) + return changed + if observation.kind == "read_succeeded_invalid_prompt": changed = row is None or ( str(row[5]), @@ -16004,6 +16158,19 @@ def _apply_backend_pending_observation_conn( ) for choice in observation.choices ) + public_meta: dict[str, Any] = {"source": "backend"} + if observation.decision_kind is not None: + public_meta["decision"] = { + "decision_ref": f"decision-{revision_digest}", + "kind": observation.decision_kind, + "prompt": observation.question, + "options": [ + {"ref": str(ordinal), "label": label} + for ordinal, label in enumerate(observation.decision_options, 1) + ], + "multi_select": observation.decision_multi_select, + "question_count": observation.decision_question_count, + } public_payload = { "question": observation.question, "kind": observation.pending_kind or "question", @@ -16011,7 +16178,7 @@ def _apply_backend_pending_observation_conn( {"choice_id": choice_id, "label": label} for choice_id, label, _ordinal in persisted_choices ], - "meta": {"source": "backend"}, + "meta": public_meta, } routes = tuple( (choice_id, ordinal) @@ -16116,12 +16283,27 @@ def _merge_backend_pending_conn( for ordinal, choice in enumerate(normalized_choices, 1) ) question = str(clean.get("question") or clean.get("kind") or "Pending action") + decision: dict[str, Any] | None = None + clean_meta = clean.get("meta") + if isinstance(clean_meta, Mapping) and "decision" in clean_meta: + decision = _normalize_pending_decision_meta(clean_meta["decision"]) observation = PendingObservation( "open_prompt", question=question, pending_kind=str(clean.get("kind") or "question"), choices=choices, revision_digest=stable_fingerprint({"legacy_pending_revision": clean}), + decision_kind=(decision or {}).get("kind"), + decision_options=tuple( + str(option["label"]) + for option in (decision or {}).get("options", []) + ), + decision_multi_select=bool( + (decision or {}).get("multi_select", False) + ), + decision_question_count=int( + (decision or {}).get("question_count", 0) + ), ) return _apply_backend_pending_observation_conn( conn, @@ -16453,10 +16635,22 @@ def pending_payload_from_store( updated_at, observation_state, ) in backend_rows: - if str(observation_state) == "none": + state = str(observation_state) + if state == "none": suppressed_worker_ids.add(str(worker_id)) continue - if str(observation_state) != "open": + if state == "invalid": + try: + invalid_payload = json.loads(payload_json) + except (TypeError, ValueError): + invalid_payload = None + if ( + isinstance(invalid_payload, Mapping) + and invalid_payload.get("unsupported_decision") is True + ): + suppressed_worker_ids.add(str(worker_id)) + continue + if state != "open": continue try: pending = json.loads(payload_json) @@ -16719,6 +16913,372 @@ def claim_backend_pending_choice( raise +_DECISION_CLAIM_PREFIX = "decision:" + + +def _decision_from_pending_row( + payload_json: Any, + routes_json: Any, + revision_digest: Any, +) -> dict[str, Any] | None: + try: + payload = json.loads(payload_json) + routes = json.loads(routes_json) + if not isinstance(payload, Mapping) or not isinstance(routes, Mapping): + return None + normalized, _ = _normalize_backend_pending_payload( + payload, + tuple((str(key), int(value)) for key, value in routes.items()), + ) + meta = normalized.get("meta") + decision = meta.get("decision") if isinstance(meta, Mapping) else None + if not isinstance(decision, Mapping): + return None + normalized_decision = _normalize_pending_decision_meta(decision) + if normalized_decision["decision_ref"] != f"decision-{revision_digest}": + return None + return normalized_decision + except (TypeError, ValueError): + return None + + +def _validated_decision_selection( + decision: Mapping[str, Any], + selection: Any, +) -> tuple[tuple[str, ...], str | None] | None: + if not isinstance(selection, Mapping) or len(selection) != 1: + return None + kind = decision.get("kind") + valid_refs = { + str(option.get("ref")) + for option in decision.get("options", []) + if isinstance(option, Mapping) + } + if set(selection) == {"option_refs"}: + raw_refs = selection.get("option_refs") + if not isinstance(raw_refs, list) or not raw_refs or any( + type(ref) is not str for ref in raw_refs + ): + return None + refs = tuple(raw_refs) + if len(refs) != len(set(refs)) or any(ref not in valid_refs for ref in refs): + return None + if kind in {"single", "plan"} and len(refs) != 1: + return None + if kind == "multi" and len(refs) < 1: + return None + return refs, None + if set(selection) == {"text"}: + text = selection.get("text") + if kind != "single" or validate_instruction_text(text) is not None: + return None + return (), str(text) + return None + + +def _encode_decision_claim_selection( + option_refs: tuple[str, ...], + text: str | None, +) -> str: + selection: dict[str, Any] + if text is None: + selection = {"option_refs": list(option_refs)} + else: + selection = {"text": text} + return _DECISION_CLAIM_PREFIX + _canonical_json(selection) + + +def _decode_decision_claim_selection(value: Any) -> Mapping[str, Any] | None: + if not isinstance(value, str) or not value.startswith(_DECISION_CLAIM_PREFIX): + return None + try: + decoded = json.loads(value[len(_DECISION_CLAIM_PREFIX) :]) + except (TypeError, ValueError): + return None + return decoded if isinstance(decoded, Mapping) else None + + +def claim_backend_pending_decision( + db_path: Path | str, + host_id: str, + worker_id: str, + decision_ref: str, + selection: Mapping[str, Any], + *, + claim: bool = True, + observed_at: str | None = None, + claim_lease_seconds: float = BACKEND_PENDING_CLAIM_LEASE_SECONDS, +) -> BackendPendingDecisionClaim: + """Validate and optionally claim one worker's exact current decision.""" + if not _sqlite_store_exists(db_path): + return BackendPendingDecisionClaim("decision_not_pending") + current_time, _ = _pending_observed_time(observed_at) + with _connect(db_path, isolation_level=None) as conn: + _ensure_schema(conn) + conn.execute("BEGIN IMMEDIATE" if claim else "BEGIN") + try: + snapshot_row = conn.execute( + "SELECT payload FROM snapshots WHERE host_id = ? ORDER BY id DESC LIMIT 1", + (str(host_id),), + ).fetchone() + try: + snapshot = ( + Snapshot.from_dict(json.loads(snapshot_row[0])) + if snapshot_row is not None + else None + ) + except Exception: + snapshot = None + worker = next( + ( + item + for item in (snapshot.workers if snapshot is not None else []) + if item.id == str(worker_id) + ), + None, + ) + if worker is None or worker.status in {"closed", "failed", "unknown"}: + conn.rollback() + return BackendPendingDecisionClaim("unknown_worker") + row = conn.execute( + """ + SELECT payload_json, choice_routes_json, revision_digest, + freshness, binding_private_fingerprint, + observed_turn_target_value, observation_state + FROM backend_pending + WHERE host_id = ? AND worker_id = ? + """, + (str(host_id), str(worker_id)), + ).fetchone() + if row is not None and str(row[6]) == "invalid": + try: + unsupported = json.loads(str(row[0])) + except (TypeError, ValueError): + unsupported = None + if ( + isinstance(unsupported, Mapping) + and unsupported.get("unsupported_decision") is True + ): + conn.rollback() + return BackendPendingDecisionClaim("unsupported_decision") + if row is None or str(row[6]) != "open" or str(row[3]) != "fresh": + conn.rollback() + return BackendPendingDecisionClaim("decision_not_pending") + context = _backend_pending_claim_context_conn( + conn, + str(host_id), + str(worker_id), + str(row[4]), + str(row[5]), + observed_at=current_time, + ) + decision = _decision_from_pending_row(row[0], row[1], row[2]) + if ( + context is None + or decision is None + or decision.get("decision_ref") != str(decision_ref) + ): + conn.rollback() + return BackendPendingDecisionClaim("decision_not_pending") + if int(decision["question_count"]) > 1: + conn.rollback() + return BackendPendingDecisionClaim("unsupported_decision") + validated_selection = _validated_decision_selection(decision, selection) + if validated_selection is None: + conn.rollback() + return BackendPendingDecisionClaim("invalid_selection") + option_refs, text = validated_selection + existing = conn.execute( + """ + SELECT state, claimed_at + FROM backend_pending_claims + WHERE host_id = ? AND worker_id = ? + """, + (str(host_id), str(worker_id)), + ).fetchone() + if existing is not None: + reclaimable = ( + str(existing[0]) == "claimed" + and _backend_pending_claim_expired( + existing[1], current_time, claim_lease_seconds + ) + ) + if reclaimable and claim: + conn.execute( + """ + DELETE FROM backend_pending_claims + WHERE host_id = ? AND worker_id = ? AND state = 'claimed' + AND claimed_at = ? + """, + (str(host_id), str(worker_id), str(existing[1])), + ) + elif not reclaimable: + conn.rollback() + return BackendPendingDecisionClaim("already_claimed") + option_count = len(decision["options"]) + picker_ordinal = int(option_refs[0]) if option_refs else option_count + 1 + fields = { + "worker_id": str(worker_id), + "worker_fingerprint": context[2], + "binding_private_fingerprint": context[1], + "turn_target_value": context[3], + "decision_ref": str(decision_ref), + "decision_kind": decision["kind"], + "option_count": option_count, + "option_refs": option_refs, + "text": text, + } + if not claim: + conn.rollback() + return BackendPendingDecisionClaim("validated", **fields) + token = secrets.token_urlsafe(32) + conn.execute( + """ + INSERT INTO backend_pending_claims ( + host_id, worker_id, claim_token, revision_digest, choice_id, + picker_ordinal, worker_fingerprint, + binding_private_fingerprint, turn_target_value, state, + claimed_at, send_started_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'claimed', ?, NULL) + """, + ( + str(host_id), str(worker_id), token, str(row[2]), + _encode_decision_claim_selection(option_refs, text), + picker_ordinal, context[2], context[1], context[3], current_time, + ), + ) + conn.commit() + return BackendPendingDecisionClaim( + "claimed", claim_token=token, **fields + ) + except Exception: + conn.rollback() + raise + + +def start_backend_pending_decision_send( + db_path: Path | str, + host_id: str, + claim_token: str, + *, + observed_at: str | None = None, + claim_lease_seconds: float = BACKEND_PENDING_CLAIM_LEASE_SECONDS, +) -> BackendPendingDecisionSend: + """CAS a decision claim against its current prompt and private binding.""" + if not _sqlite_store_exists(db_path): + return BackendPendingDecisionSend("not_found") + current_time, _ = _pending_observed_time(observed_at) + with _connect(db_path, isolation_level=None) as conn: + _ensure_schema(conn) + conn.execute("BEGIN IMMEDIATE") + try: + row = conn.execute( + """ + SELECT worker_id, revision_digest, choice_id, + worker_fingerprint, binding_private_fingerprint, + turn_target_value, state, claimed_at + FROM backend_pending_claims + WHERE host_id = ? AND claim_token = ? + """, + (str(host_id), str(claim_token)), + ).fetchone() + if row is None: + conn.rollback() + return BackendPendingDecisionSend("not_found") + if ( + str(row[6]) == "claimed" + and _backend_pending_claim_expired( + row[7], current_time, claim_lease_seconds + ) + ): + conn.execute( + """ + DELETE FROM backend_pending_claims + WHERE host_id = ? AND claim_token = ? AND state = 'claimed' + """, + (str(host_id), str(claim_token)), + ) + conn.commit() + return BackendPendingDecisionSend("not_found") + current = conn.execute( + """ + SELECT payload_json, choice_routes_json, revision_digest, + freshness, binding_private_fingerprint, + observed_turn_target_value + FROM backend_pending + WHERE host_id = ? AND worker_id = ? + AND observation_state = 'open' + """, + (str(host_id), str(row[0])), + ).fetchone() + if current is None: + conn.rollback() + return BackendPendingDecisionSend("changed") + if str(current[3]) != "fresh": + conn.rollback() + return BackendPendingDecisionSend("stale") + decision = _decision_from_pending_row(current[0], current[1], current[2]) + selection = _decode_decision_claim_selection(row[2]) + validated_selection = ( + _validated_decision_selection(decision, selection) + if decision is not None and selection is not None + else None + ) + if ( + str(current[2]) != str(row[1]) + or str(current[4]) != str(row[4]) + or str(current[5]) != str(row[5]) + or decision is None + or validated_selection is None + ): + conn.rollback() + return BackendPendingDecisionSend("changed") + option_refs, text = validated_selection + fields = { + "worker_id": str(row[0]), + "worker_fingerprint": str(row[3]), + "binding_private_fingerprint": str(row[4]), + "turn_target_value": str(row[5]), + "decision_ref": str(decision["decision_ref"]), + "decision_kind": decision["kind"], + "option_count": len(decision["options"]), + "option_refs": option_refs, + "text": text, + } + if str(row[6]) == "send_started": + conn.rollback() + return BackendPendingDecisionSend("already_started", **fields) + context = _backend_pending_claim_context_conn( + conn, + str(host_id), + str(row[0]), + str(row[4]), + str(row[5]), + observed_at=current_time, + ) + if context is None or (context[1], context[2], context[3]) != ( + str(row[4]), str(row[3]), str(row[5]) + ): + conn.rollback() + return BackendPendingDecisionSend("binding_changed") + cursor = conn.execute( + """ + UPDATE backend_pending_claims + SET state = 'send_started', send_started_at = ? + WHERE host_id = ? AND claim_token = ? AND state = 'claimed' + """, + (current_time, str(host_id), str(claim_token)), + ) + if cursor.rowcount != 1: + conn.rollback() + return BackendPendingDecisionSend("changed") + conn.commit() + return BackendPendingDecisionSend("started", **fields) + except Exception: + conn.rollback() + raise + + def start_backend_pending_choice_send( db_path: Path | str, host_id: str, @@ -20703,6 +21263,50 @@ def reserve_command_request( conn.close() +def abandon_command_request_reservation( + db_path: Path, + *, + host_id: str, + request_id: str, + canonical_fingerprint: str, + owner_token: str, + now: str | None = None, +) -> bool: + """Release only the caller's unsent reservation for immediate takeover.""" + if not _sqlite_store_exists(db_path): + return False + current = _command_request_now(now) + owner_hash = _owner_token_hash(owner_token) + if not owner_hash: + return False + with _connect(db_path, isolation_level=None) as conn: + _ensure_schema(conn) + conn.execute("BEGIN IMMEDIATE") + try: + updated = conn.execute( + """ + UPDATE command_receipts + SET owner_expires_at = ?, updated_at = ? + WHERE host_id = ? AND request_id = ? + AND canonical_fingerprint = ? + AND state = 'reserved' AND owner_token_hash = ? + """, + ( + current, + current, + str(host_id), + str(request_id), + str(canonical_fingerprint), + owner_hash, + ), + ) + conn.commit() + return int(updated.rowcount or 0) == 1 + except Exception: + conn.rollback() + raise + + def reserve_terminal_command_replay( db_path: Path, *, @@ -20952,6 +21556,7 @@ def finish_command_request( ) if terminal == "rejected" and str(status) in { "pending", + "answer_in_progress", "accepted", "request_state_uncertain", }: diff --git a/tests/test_answer_decision.py b/tests/test_answer_decision.py new file mode 100644 index 0000000..c57499c --- /dev/null +++ b/tests/test_answer_decision.py @@ -0,0 +1,856 @@ +"""Semantic connector answers for current backend-owned Claude decisions.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +import tendwire.command_submission as command_submission + +from tendwire.backends.herdr_decision import calibrate_decision_steps +from tendwire.backends.herdr_turns import ( + PENDING_DECISION_MAX_OPTIONS, + _pending_observation_from_turn, +) +from tendwire.command_submission import submit_command +from tendwire.core.commands import ( + DISPOSITION_IN_PROGRESS, + DISPOSITION_NO_RECEIPT, + STATUS_ACCEPTED, + STATUS_ANSWER_IN_PROGRESS, + STATUS_DECISION_NOT_PENDING, + STATUS_INVALID_SELECTION, + STATUS_PENDING, + STATUS_REQUEST_STATE_UNCERTAIN, + STATUS_UNKNOWN_WORKER, + STATUS_UNSUPPORTED_DECISION, + CommandRequest, + build_canonical_mutation, +) +from tendwire.core.models import Worker +from tendwire.store.sqlite import ( + abandon_backend_pending_choice_claim, + apply_backend_pending_observation, + claim_backend_pending_decision, + envelope_to_receipt_json, + get_command_request, + pending_payload_from_store, + reserve_command_request, +) + +from tests.test_command_submission import ( + _FakeSocketClient, + _binding, + _config, + _factory, + _seed, +) + + +def _decision_turn( + *, + prompt: str = "Choose a database", + kind: str = "AskUserQuestion", + multi_select: bool = False, + question_count: int = 1, +) -> dict[str, Any]: + return { + "pending_decision": { + "decision_id": "private-tool-use", + "kind": kind, + "question": prompt, + "options": [ + {"id": "postgres", "label": "Postgres"}, + {"id": "sqlite", "label": "SQLite"}, + {"id": "duckdb", "label": "DuckDB"}, + {"id": "mysql", "label": "MySQL"}, + ], + "multi_select": multi_select, + "question_count": question_count, + } + } + + +def _seed_pending_decision( + tmp_path: Path, + *, + turn: dict[str, Any] | None = None, +) -> tuple[Any, Worker, str]: + config = _config(tmp_path) + worker = Worker(id="w-1", name="Alpha", status="active") + binding = _binding( + worker, + private_fingerprint="decision-binding-private", + turn_target_value="decision-pane-private", + ) + _seed(config, [worker], [binding]) + assert config.db_path is not None + observation = _pending_observation_from_turn(turn or _decision_turn()) + assert apply_backend_pending_observation( + config.db_path, + config.host_id, + worker.id, + observation, + binding_private_fingerprint=binding.private_fingerprint, + observed_turn_target_value=binding.turn_target_value, + ) + payload = pending_payload_from_store(config.db_path, config.host_id) + row = next(item for item in payload["pending_interactions"] if item["worker_id"] == worker.id) + return config, worker, row["meta"]["decision"]["decision_ref"] + + +def _answer_request( + decision_ref: str, + *, + request_id: str = "decision-request-1", + worker_id: str = "w-1", + selection: dict[str, Any] | None = None, +) -> dict[str, Any]: + return { + "schema_version": 1, + "action": "answer_decision", + "request_id": request_id, + "dry_run": False, + "target": {"worker_id": worker_id}, + "params": { + "decision_ref": decision_ref, + "selection": selection or {"option_refs": ["2"]}, + }, + } + + +def test_pending_payload_carries_stable_structured_decision_and_rotates_ref( + tmp_path: Path, +) -> None: + config, worker, first_ref = _seed_pending_decision(tmp_path) + assert config.db_path is not None + first = pending_payload_from_store(config.db_path, config.host_id) + first_row = next(item for item in first["pending_interactions"] if item["worker_id"] == worker.id) + assert first_row["meta"]["decision"] == { + "decision_ref": first_ref, + "kind": "single", + "prompt": "Choose a database", + "options": [ + {"ref": "1", "label": "Postgres"}, + {"ref": "2", "label": "SQLite"}, + {"ref": "3", "label": "DuckDB"}, + {"ref": "4", "label": "MySQL"}, + ], + "multi_select": False, + "question_count": 1, + } + + binding = _binding( + worker, + private_fingerprint="decision-binding-private", + turn_target_value="decision-pane-private", + ) + changed = _pending_observation_from_turn( + _decision_turn(prompt="Choose a durable database") + ) + assert apply_backend_pending_observation( + config.db_path, + config.host_id, + worker.id, + changed, + binding_private_fingerprint=binding.private_fingerprint, + observed_turn_target_value=binding.turn_target_value, + ) + second = pending_payload_from_store(config.db_path, config.host_id) + second_row = next(item for item in second["pending_interactions"] if item["worker_id"] == worker.id) + assert second_row["meta"]["decision"]["decision_ref"] != first_ref + + +def test_answer_decision_stale_ref_fails_before_pane_io(tmp_path: Path) -> None: + config, _worker, _decision_ref = _seed_pending_decision(tmp_path) + calls: list[dict[str, Any]] = [] + result = submit_command( + config, + _answer_request("decision-stale"), + socket_client_factory=_factory(calls), + ) + assert result.ok is False + assert result.status == STATUS_DECISION_NOT_PENDING + assert calls == [] + + +def test_answer_decision_omitted_dry_run_is_preview_only(tmp_path: Path) -> None: + config, _worker, decision_ref = _seed_pending_decision(tmp_path) + calls: list[dict[str, Any]] = [] + request = _answer_request(decision_ref) + request.pop("dry_run") + + result = submit_command( + config, + request, + socket_client_factory=_factory(calls), + ) + + assert result.ok is True + assert result.status == "dry_run" + assert result.dry_run is True + assert calls == [] + + +def test_answer_decision_unknown_worker_fails_before_pane_io(tmp_path: Path) -> None: + config = _config(tmp_path) + worker = Worker(id="w-1", name="Alpha", status="active") + _seed(config, [worker], [_binding(worker)]) + calls: list[dict[str, Any]] = [] + result = submit_command( + config, + _answer_request("decision-any", worker_id="worker-missing"), + socket_client_factory=_factory(calls), + ) + assert result.ok is False + assert result.status == STATUS_UNKNOWN_WORKER + assert calls == [] + + +@pytest.mark.parametrize( + "selection", + [ + {"option_refs": ["9"]}, + {"option_refs": ["1", "2"]}, + {"option_refs": ["2", "2"]}, + {"text": ""}, + {"option_refs": ["1"], "text": "both"}, + ], +) +def test_answer_decision_invalid_selection_fails_before_pane_io( + tmp_path: Path, + selection: dict[str, Any], +) -> None: + config, _worker, decision_ref = _seed_pending_decision(tmp_path) + calls: list[dict[str, Any]] = [] + result = submit_command( + config, + _answer_request(decision_ref, selection=selection), + socket_client_factory=_factory(calls), + ) + assert result.ok is False + assert result.status == STATUS_INVALID_SELECTION + assert calls == [] + + +def test_answer_decision_refuses_multi_question_before_pane_io(tmp_path: Path) -> None: + config, _worker, decision_ref = _seed_pending_decision( + tmp_path, + turn=_decision_turn(question_count=2), + ) + calls: list[dict[str, Any]] = [] + result = submit_command( + config, + _answer_request(decision_ref), + socket_client_factory=_factory(calls), + ) + assert result.ok is False + assert result.status == STATUS_UNSUPPORTED_DECISION + assert calls == [] + + +def test_answer_decision_rejects_plan_write_in_before_pane_io(tmp_path: Path) -> None: + config, _worker, decision_ref = _seed_pending_decision( + tmp_path, + turn=_decision_turn(kind="ExitPlanMode"), + ) + calls: list[dict[str, Any]] = [] + result = submit_command( + config, + _answer_request(decision_ref, selection={"text": "Revise this plan"}), + socket_client_factory=_factory(calls), + ) + assert result.ok is False + assert result.status == STATUS_INVALID_SELECTION + assert calls == [] + + +def test_decision_calibration_steps_cover_single_plan_write_in_and_multi() -> None: + single = calibrate_decision_steps( + kind="single", option_count=4, option_refs=("2",) + ) + assert [(item.operation, item.keys, item.text) for item in single] == [ + ("keys", ("2", "Enter"), None) + ] + + plan = calibrate_decision_steps( + kind="plan", option_count=2, option_refs=("1",) + ) + assert [(item.operation, item.keys, item.text) for item in plan] == [ + ("keys", ("1", "Enter"), None) + ] + + write_in = calibrate_decision_steps( + kind="single", option_count=4, text="Use another database" + ) + assert [(item.operation, item.keys, item.text) for item in write_in] == [ + ("keys", ("5",), None), + ("input", ("Enter",), "Use another database"), + ] + + multi = calibrate_decision_steps( + kind="multi", option_count=4, option_refs=("3", "1") + ) + assert [(item.operation, item.keys, item.text) for item in multi] == [ + ("keys", ("Down", "Down"), None), + ("text", (), " "), + ("keys", ("Up", "Up"), None), + ("text", (), " "), + ("keys", ("Down", "Down", "Down", "Down", "Enter"), None), + ] + + +def test_production_shaped_multi_decision_end_to_end(tmp_path: Path) -> None: + tool_input = { + "questions": [ + { + "question": "Which databases should we support?", + "header": "Database support", + "options": [ + {"label": "Postgres", "description": "Primary database"}, + {"label": "SQLite", "description": "Local database"}, + {"label": "DuckDB", "description": "Analytics database"}, + {"label": "MySQL", "description": "Compatibility database"}, + ], + "multiSelect": True, + } + ] + } + question = tool_input["questions"][0] + adapter_pending_decision = { + "decision_id": "toolu_multi_123", + "prompt": f'{question["header"]}: {question["question"]}', + "mode": "multi", + "multi_select": question["multiSelect"], + "options": [ + {"id": str(ordinal), "label": option["label"]} + for ordinal, option in enumerate(question["options"], 1) + ], + } + assert adapter_pending_decision == { + "decision_id": "toolu_multi_123", + "prompt": "Database support: Which databases should we support?", + "mode": "multi", + "multi_select": True, + "options": [ + {"id": "1", "label": "Postgres"}, + {"id": "2", "label": "SQLite"}, + {"id": "3", "label": "DuckDB"}, + {"id": "4", "label": "MySQL"}, + ], + } + + config, worker, decision_ref = _seed_pending_decision( + tmp_path, + turn={"pending_decision": adapter_pending_decision}, + ) + assert config.db_path is not None + payload = pending_payload_from_store(config.db_path, config.host_id) + pending = next( + row for row in payload["pending_interactions"] + if row["worker_id"] == worker.id + ) + assert pending["meta"]["decision"] == { + "decision_ref": decision_ref, + "kind": "multi", + "prompt": "Database support: Which databases should we support?", + "options": [ + {"ref": "1", "label": "Postgres"}, + {"ref": "2", "label": "SQLite"}, + {"ref": "3", "label": "DuckDB"}, + {"ref": "4", "label": "MySQL"}, + ], + "multi_select": True, + "question_count": 1, + } + + calls: list[dict[str, Any]] = [] + result = submit_command( + config, + _answer_request( + decision_ref, + request_id="multi-production-answer", + selection={"option_refs": ["3", "1"]}, + ), + socket_client_factory=_factory(calls), + ) + + assert result.ok is True + assert result.status == STATUS_ACCEPTED + assert calls == [ + { + "method": "pane.send_keys", + "params": { + "pane_id": "decision-pane-private", + "keys": ["Down", "Down"], + }, + }, + { + "method": "pane.send_text", + "params": {"pane_id": "decision-pane-private", "text": " "}, + }, + { + "method": "pane.send_keys", + "params": { + "pane_id": "decision-pane-private", + "keys": ["Up", "Up"], + }, + }, + { + "method": "pane.send_text", + "params": {"pane_id": "decision-pane-private", "text": " "}, + }, + { + "method": "pane.send_keys", + "params": { + "pane_id": "decision-pane-private", + "keys": ["Down", "Down", "Down", "Down", "Enter"], + }, + }, + ] + + +def _bounded_decision_turn(option_count: int, *, custom_last: bool = False) -> dict[str, Any]: + options = [ + {"id": str(ordinal), "label": f"Option {ordinal}"} + for ordinal in range(1, option_count + 1) + ] + if custom_last: + options.append({"id": "custom", "label": "Type something"}) + return { + "pending_decision": { + "decision_id": "toolu_bound", + "kind": "AskUserQuestion", + "prompt": "Choose one", + "multi_select": False, + "options": options, + } + } + + +def _unknown_decision_turn() -> dict[str, Any]: + turn = _bounded_decision_turn(2) + turn["pending_decision"]["kind"] = "FutureDecisionKind" + return turn + + +def test_decision_option_bound_accepts_exactly_eleven(tmp_path: Path) -> None: + config, worker, _decision_ref = _seed_pending_decision( + tmp_path, + turn=_bounded_decision_turn(PENDING_DECISION_MAX_OPTIONS), + ) + assert config.db_path is not None + payload = pending_payload_from_store(config.db_path, config.host_id) + row = next( + item for item in payload["pending_interactions"] + if item["worker_id"] == worker.id + ) + assert len(row["meta"]["decision"]["options"]) == 11 + + +@pytest.mark.parametrize( + "turn", + [ + _bounded_decision_turn(PENDING_DECISION_MAX_OPTIONS + 1), + _bounded_decision_turn(PENDING_DECISION_MAX_OPTIONS, custom_last=True), + _unknown_decision_turn(), + ], + ids=[ + "twelve-real-options", + "custom-row-after-eleven-real-options", + "unknown-kind", + ], +) +def test_over_bound_decision_fails_closed_without_pane_io( + tmp_path: Path, + turn: dict[str, Any], +) -> None: + config = _config(tmp_path) + worker = Worker(id="w-1", name="Alpha", status="active") + binding = _binding( + worker, + private_fingerprint="decision-binding-private", + turn_target_value="decision-pane-private", + ) + _seed(config, [worker], [binding]) + observation = _pending_observation_from_turn(turn) + assert observation.kind == "read_succeeded_unsupported_decision" + assert config.db_path is not None + assert apply_backend_pending_observation( + config.db_path, + config.host_id, + worker.id, + observation, + binding_private_fingerprint=binding.private_fingerprint, + observed_turn_target_value=binding.turn_target_value, + ) + payload = pending_payload_from_store(config.db_path, config.host_id) + assert not any( + item["worker_id"] == worker.id + for item in payload["pending_interactions"] + ) + + calls: list[dict[str, Any]] = [] + result = submit_command( + config, + _answer_request("decision-unsupported-bound"), + socket_client_factory=_factory(calls), + ) + assert result.ok is False + assert result.status == STATUS_UNSUPPORTED_DECISION + assert calls == [] + + +def test_answer_decision_request_id_replay_does_not_resend_keys(tmp_path: Path) -> None: + config, _worker, decision_ref = _seed_pending_decision(tmp_path) + calls: list[dict[str, Any]] = [] + request = _answer_request(decision_ref) + + first = submit_command(config, request, socket_client_factory=_factory(calls)) + replay = submit_command(config, request, socket_client_factory=_factory(calls)) + + assert first.ok is True + assert first.status == STATUS_ACCEPTED + assert replay.to_dict() == first.to_dict() + assert calls == [ + { + "method": "pane.send_keys", + "params": { + "pane_id": "decision-pane-private", + "keys": ["2", "Enter"], + }, + } + ] + + +def test_two_requests_race_one_decision_and_only_first_claimant_sends( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, worker, decision_ref = _seed_pending_decision(tmp_path) + assert config.db_path is not None + winner = _answer_request(decision_ref, request_id="decision-winner") + loser = _answer_request(decision_ref, request_id="decision-loser") + calls: list[dict[str, Any]] = [] + raced: dict[str, Any] = {} + real_mark = command_submission._mark_request_send_started + + def race_after_claim(*args: Any, **kwargs: Any) -> Any: + pending = pending_payload_from_store(config.db_path, config.host_id) + assert any( + item["worker_id"] == worker.id + and item["meta"]["decision"]["decision_ref"] == decision_ref + for item in pending["pending_interactions"] + ) + raced["loser"] = submit_command( + config, + loser, + socket_client_factory=_factory(calls), + ) + return real_mark(*args, **kwargs) + + monkeypatch.setattr( + command_submission, + "_mark_request_send_started", + race_after_claim, + ) + first = submit_command( + config, + winner, + socket_client_factory=_factory(calls), + ) + + assert first.ok is True + assert first.status == STATUS_ACCEPTED + assert raced["loser"].ok is False + assert raced["loser"].status == STATUS_ANSWER_IN_PROGRESS + assert raced["loser"].disposition == DISPOSITION_NO_RECEIPT + assert calls == [ + { + "method": "pane.send_keys", + "params": { + "pane_id": "decision-pane-private", + "keys": ["2", "Enter"], + }, + } + ] + + +def test_winner_safe_presend_failure_releases_claim_for_loser_retry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, _worker, decision_ref = _seed_pending_decision(tmp_path) + winner = _answer_request(decision_ref, request_id="decision-safe-failure") + loser = _answer_request(decision_ref, request_id="decision-safe-retry") + calls: list[dict[str, Any]] = [] + raced: dict[str, Any] = {} + real_mark = command_submission._mark_request_send_started + first_mark = True + + def fail_winner_before_send(*args: Any, **kwargs: Any) -> Any: + nonlocal first_mark + if first_mark: + first_mark = False + raced["loser"] = submit_command( + config, + loser, + socket_client_factory=_factory(calls), + ) + return command_submission._request_in_progress(args[1]) + return real_mark(*args, **kwargs) + + monkeypatch.setattr( + command_submission, + "_mark_request_send_started", + fail_winner_before_send, + ) + failed_winner = submit_command( + config, + winner, + socket_client_factory=_factory(calls), + ) + retried_loser = submit_command( + config, + loser, + socket_client_factory=_factory(calls), + ) + + assert failed_winner.status == STATUS_PENDING + assert raced["loser"].status == STATUS_ANSWER_IN_PROGRESS + assert raced["loser"].disposition == DISPOSITION_NO_RECEIPT + assert retried_loser.ok is True + assert retried_loser.status == STATUS_ACCEPTED + assert calls == [ + { + "method": "pane.send_keys", + "params": { + "pane_id": "decision-pane-private", + "keys": ["2", "Enter"], + }, + } + ] + + +def test_claim_time_loser_releases_unsent_reservation_for_takeover( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, worker, decision_ref = _seed_pending_decision(tmp_path) + assert config.db_path is not None + request = _answer_request(decision_ref, request_id="claim-time-loser") + calls: list[dict[str, Any]] = [] + real_claim = command_submission._claim_pending_decision + injected_claim: dict[str, Any] = {} + + def lose_during_claim( + claim_config: Any, + claim_request: CommandRequest, + validated: Any, + ) -> Any: + if "value" not in injected_claim: + injected_claim["value"] = claim_backend_pending_decision( + config.db_path, + config.host_id, + worker.id, + decision_ref, + {"option_refs": ["2"]}, + claim=True, + ) + assert injected_claim["value"].status == "claimed" + return real_claim(claim_config, claim_request, validated) + + monkeypatch.setattr( + command_submission, + "_claim_pending_decision", + lose_during_claim, + ) + first = submit_command( + config, + request, + socket_client_factory=_factory(calls), + ) + receipt = get_command_request( + config.db_path, + config.host_id, + "claim-time-loser", + ) + + assert first.status == STATUS_ANSWER_IN_PROGRESS + assert first.disposition == DISPOSITION_IN_PROGRESS + assert receipt is not None + assert receipt["state"] == "reserved" + assert receipt["status"] == STATUS_PENDING + assert calls == [] + + assert abandon_backend_pending_choice_claim( + config.db_path, + config.host_id, + injected_claim["value"].claim_token, + ) + retry = submit_command( + config, + request, + socket_client_factory=_factory(calls), + ) + assert retry.ok is True + assert retry.status == STATUS_ACCEPTED + assert len(calls) == 1 + + +def test_abandoned_reservation_is_not_terminalized_by_live_claim( + tmp_path: Path, +) -> None: + config, worker, decision_ref = _seed_pending_decision(tmp_path) + assert config.db_path is not None + payload = _answer_request(decision_ref, request_id="abandoned-loser") + request = CommandRequest.from_dict(payload) + canonical = build_canonical_mutation(request, public_worker_id=worker.id) + initial = reserve_command_request( + config.db_path, + host_id=config.host_id, + request_id=request.request_id or "", + action=canonical.action, + canonical_version=canonical.canonical_version, + canonical_fingerprint=canonical.fingerprint, + canonical_request_json=canonical.canonical_json, + public_worker_id=canonical.public_worker_id, + pending_result_json=envelope_to_receipt_json( + command_submission._request_in_progress(request) + ), + legacy_raw_payload_fingerprint=request.payload_fingerprint(), + owner_lease_seconds=1, + now="2020-01-01T00:00:00+00:00", + ) + assert initial["status"] == "reserved" + competing = claim_backend_pending_decision( + config.db_path, + config.host_id, + worker.id, + decision_ref, + {"option_refs": ["1"]}, + claim=True, + ) + assert competing.status == "claimed" + + calls: list[dict[str, Any]] = [] + result = submit_command( + config, + payload, + socket_client_factory=_factory(calls), + ) + receipt = get_command_request( + config.db_path, + config.host_id, + request.request_id or "", + ) + + assert result.status == STATUS_ANSWER_IN_PROGRESS + assert result.disposition == DISPOSITION_IN_PROGRESS + assert receipt is not None + assert receipt["state"] == "reserved" + assert receipt["status"] == STATUS_PENDING + assert calls == [] + + +def test_process_loss_before_send_started_is_recovered_after_lease_expiry( + tmp_path: Path, +) -> None: + config, worker, decision_ref = _seed_pending_decision(tmp_path) + assert config.db_path is not None + payload = _answer_request(decision_ref, request_id="crashed-before-send") + request = CommandRequest.from_dict(payload) + canonical = build_canonical_mutation(request, public_worker_id=worker.id) + reservation = reserve_command_request( + config.db_path, + host_id=config.host_id, + request_id=request.request_id or "", + action=canonical.action, + canonical_version=canonical.canonical_version, + canonical_fingerprint=canonical.fingerprint, + canonical_request_json=canonical.canonical_json, + public_worker_id=canonical.public_worker_id, + pending_result_json=envelope_to_receipt_json( + command_submission._request_in_progress(request) + ), + legacy_raw_payload_fingerprint=request.payload_fingerprint(), + owner_lease_seconds=1, + now="2020-01-01T00:00:00+00:00", + ) + assert reservation["status"] == "reserved" + abandoned_claim = claim_backend_pending_decision( + config.db_path, + config.host_id, + worker.id, + decision_ref, + {"option_refs": ["2"]}, + claim=True, + observed_at="2020-01-01T00:00:00+00:00", + claim_lease_seconds=1, + ) + assert abandoned_claim.status == "claimed" + + calls: list[dict[str, Any]] = [] + recovered = submit_command( + config, + payload, + socket_client_factory=_factory(calls), + ) + + assert recovered.ok is True + assert recovered.status == STATUS_ACCEPTED + assert calls == [ + { + "method": "pane.send_keys", + "params": { + "pane_id": "decision-pane-private", + "keys": ["2", "Enter"], + }, + } + ] + + +def test_send_uncertainty_is_durable_and_never_resends( + tmp_path: Path, +) -> None: + config, _worker, decision_ref = _seed_pending_decision(tmp_path) + request = _answer_request(decision_ref, request_id="uncertain-decision") + calls: list[dict[str, Any]] = [] + + class FailingKeyClient(_FakeSocketClient): + def request( + self, + method: str, + params: dict[str, Any], + *, + timeout: float | None = None, + ) -> dict[str, Any]: + self.calls.append({"method": method, "params": dict(params)}) + if method == "pane.send_keys": + raise RuntimeError("response lost after key send") + return {"accepted": True} + + first = submit_command( + config, + request, + socket_client_factory=lambda _config: FailingKeyClient(calls), + ) + replay = submit_command( + config, + request, + socket_client_factory=lambda _config: FailingKeyClient(calls), + ) + + assert first.ok is False + assert first.status == STATUS_REQUEST_STATE_UNCERTAIN + assert replay.status == STATUS_REQUEST_STATE_UNCERTAIN + assert calls == [ + { + "method": "pane.send_keys", + "params": { + "pane_id": "decision-pane-private", + "keys": ["2", "Enter"], + }, + } + ] diff --git a/tests/test_backend_pending.py b/tests/test_backend_pending.py index e39d109..4283a7f 100644 --- a/tests/test_backend_pending.py +++ b/tests/test_backend_pending.py @@ -65,7 +65,19 @@ def test_extract_pending_decision(): assert all("value" not in c for c in pending["choices"]) # decision_id (internal tool_use_id) is not published in public pending. assert "decision_id" not in pending["meta"] - assert pending["meta"] == {"source": "backend"} + decision = pending["meta"]["decision"] + assert decision == { + "decision_ref": decision["decision_ref"], + "kind": "single", + "prompt": "Which database should we use?", + "options": [ + {"ref": "1", "label": "Postgres"}, + {"ref": "2", "label": "SQLite"}, + ], + "multi_select": False, + "question_count": 1, + } + assert decision["decision_ref"].startswith("decision-") def test_extract_plan_approval_kind(): diff --git a/tests/test_command_submission.py b/tests/test_command_submission.py index de474bb..a60c444 100644 --- a/tests/test_command_submission.py +++ b/tests/test_command_submission.py @@ -251,7 +251,9 @@ def _expected_private_clear_calls(pane_id: str = "pane-secret") -> list[dict[str {"method": "pane.send_keys", "params": {"pane_id": pane_id, "keys": ["ctrl+a", "backspace"]}}, ] -@pytest.mark.parametrize("action", ["send_instruction", "answer_pending"]) +@pytest.mark.parametrize( + "action", ["send_instruction", "answer_pending", "answer_decision"] +) @pytest.mark.parametrize( ("request_id", "include_request_id"), [ @@ -312,6 +314,18 @@ def guarded_socket_factory(config: Config) -> _FakeSocketClient: "choice_id": "choice-public", }, } + elif action == "answer_decision": + payload = { + "schema_version": 1, + "action": "answer_decision", + "request_id": request_id, + "dry_run": False, + "target": {"worker_id": "w-1"}, + "params": { + "decision_ref": "decision-public", + "selection": {"option_refs": ["1"]}, + }, + } if include_request_id: payload["request_id"] = request_id else: diff --git a/tests/test_commands.py b/tests/test_commands.py index c52ae3f..f31dace 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -21,6 +21,7 @@ TERMINAL_MUTATION_REJECTION_STATUSES, VALID_DISPOSITIONS, STATUS_ACCEPTED, + STATUS_ANSWER_IN_PROGRESS, STATUS_BACKEND_UNAVAILABLE, STATUS_PENDING, STATUS_REQUEST_STATE_UNCERTAIN, @@ -191,6 +192,7 @@ def test_allowed_actions_frozen() -> None: "resolve_target", "send_instruction", "answer_pending", + "answer_decision", } @@ -557,7 +559,9 @@ def test_command_envelope_rejects_inconsistent_receipt_tuples( ) -@pytest.mark.parametrize("action", ["send_instruction", "answer_pending"]) +@pytest.mark.parametrize( + "action", ["send_instruction", "answer_pending", "answer_decision"] +) @pytest.mark.parametrize("request_id", [None, "", "not canonical"]) def test_command_envelope_live_mutations_require_canonical_request_ids( action: str, @@ -567,7 +571,7 @@ def test_command_envelope_live_mutations_require_canonical_request_ids( action=action, request_id=request_id, dry_run=False, - target={"worker_id": "w-1"} if action == "send_instruction" else None, + target={"worker_id": "w-1"} if action != "answer_pending" else None, instruction={"text": "hello"} if action == "send_instruction" else None, params=( { @@ -576,7 +580,14 @@ def test_command_envelope_live_mutations_require_canonical_request_ids( "choice_id": "choice-1", } if action == "answer_pending" - else None + else ( + { + "decision_ref": "decision-1", + "selection": {"option_refs": ["1"]}, + } + if action == "answer_decision" + else None + ) ), ) @@ -638,6 +649,10 @@ def test_mutation_disposition_status_sets_are_explicit_and_fail_closed() -> None "ambiguous_backend_target", "backend_failed", "duplicate_request", + "decision_not_pending", + "unknown_worker", + "invalid_selection", + "unsupported_decision", } assert LIVE_MUTATION_NO_RECEIPT_REJECTION_STATUSES == { "invalid_request", @@ -649,9 +664,15 @@ def test_mutation_disposition_status_sets_are_explicit_and_fail_closed() -> None "backend_unsupported", "ambiguous_backend_target", "backend_failed", + "answer_in_progress", + "decision_not_pending", + "unknown_worker", + "invalid_selection", + "unsupported_decision", } assert DRY_RUN_MUTATION_NO_RECEIPT_REJECTION_STATUSES == { "invalid_request", + "invalid_selection", "rejected", "not_found", "ambiguous_target", @@ -659,7 +680,44 @@ def test_mutation_disposition_status_sets_are_explicit_and_fail_closed() -> None } -@pytest.mark.parametrize("action", ["send_instruction", "answer_pending"]) +def test_answer_in_progress_allows_only_retryable_dispositions() -> None: + request = CommandRequest( + action="answer_decision", + request_id="answer-race", + dry_run=False, + target={"worker_id": "w-1"}, + params={ + "decision_ref": "decision-1", + "selection": {"option_refs": ["1"]}, + }, + ) + for disposition in (DISPOSITION_NO_RECEIPT, DISPOSITION_IN_PROGRESS): + envelope = CommandEnvelope.from_result( + request, + ok=False, + status=STATUS_ANSWER_IN_PROGRESS, + disposition=disposition, + error={"code": STATUS_ANSWER_IN_PROGRESS, "message": "racing"}, + ) + assert envelope.disposition == disposition + for disposition in ( + DISPOSITION_TERMINAL_ACCEPTED, + DISPOSITION_TERMINAL_REJECTED, + DISPOSITION_TERMINAL_UNCERTAIN, + ): + with pytest.raises(ValueError): + CommandEnvelope.from_result( + request, + ok=False, + status=STATUS_ANSWER_IN_PROGRESS, + disposition=disposition, + error={"code": STATUS_ANSWER_IN_PROGRESS, "message": "racing"}, + ) + + +@pytest.mark.parametrize( + "action", ["send_instruction", "answer_pending", "answer_decision"] +) @pytest.mark.parametrize("ok", [False, True]) @pytest.mark.parametrize("status", sorted(VALID_STATUSES)) def test_command_envelope_from_dict_enforces_terminal_rejected_matrix( @@ -688,7 +746,9 @@ def test_command_envelope_from_dict_enforces_terminal_rejected_matrix( CommandEnvelope.from_dict(payload) -@pytest.mark.parametrize("action", ["send_instruction", "answer_pending"]) +@pytest.mark.parametrize( + "action", ["send_instruction", "answer_pending", "answer_decision"] +) @pytest.mark.parametrize("dry_run", [False, True], ids=["live", "dry-run"]) @pytest.mark.parametrize("ok", [False, True]) @pytest.mark.parametrize("status", sorted(VALID_STATUSES)) @@ -721,6 +781,10 @@ def test_command_envelope_from_dict_enforces_mutation_no_receipt_matrix( allowed = ( not ok and status in LIVE_MUTATION_NO_RECEIPT_REJECTION_STATUSES + and ( + status != STATUS_ANSWER_IN_PROGRESS + or action == "answer_decision" + ) ) if allowed: diff --git a/tests/test_turns.py b/tests/test_turns.py index fc0ab08..bf217c9 100644 --- a/tests/test_turns.py +++ b/tests/test_turns.py @@ -491,6 +491,7 @@ def test_pending_observation_models_explicit_outcomes_and_private_picker_routes( for kind in ( "read_succeeded_no_prompt", "read_succeeded_invalid_prompt", + "read_succeeded_unsupported_decision", "read_failed", "worker_authoritatively_absent", ): From 9774083d885fb8cda7d06a6783d0a9b2cc8228fc Mon Sep 17 00:00:00 2001 From: Jerry the Martian <38183696+jerryfane@users.noreply.github.com> Date: Sun, 19 Jul 2026 07:12:02 +0200 Subject: [PATCH 2/7] Gitmoot merge adhoc-f154331b Merged by Gitmoot after policy gate passed. --- .env.example | 1 + INSTALL.md | 3 + README.md | 1 + src/tendwire/backends/herdr_turns.py | 12 + src/tendwire/cli.py | 2 + src/tendwire/command_submission.py | 119 ++- src/tendwire/config.py | 17 + src/tendwire/core/turns.py | 16 + src/tendwire/daemon.py | 2 + src/tendwire/store/sqlite.py | 784 +++++++++++++++++--- tests/test_backend_pending.py | 2 +- tests/test_cli.py | 4 + tests/test_command_submission.py | 301 +++++++- tests/test_config.py | 8 + tests/test_connector_outbox.py | 2 +- tests/test_daemon.py | 2 + tests/test_delivery_retention.py | 4 + tests/test_delivery_retention_migration.py | 2 +- tests/test_delivery_retention_projection.py | 2 +- tests/test_delivery_retention_recovery.py | 2 +- tests/test_store.py | 412 +++++++++- tests/test_turn_ingestion.py | 30 + 22 files changed, 1592 insertions(+), 136 deletions(-) diff --git a/.env.example b/.env.example index e2a78fd..4ed8c4d 100644 --- a/.env.example +++ b/.env.example @@ -81,6 +81,7 @@ TENDWIRE_HERDR_TIMEOUT_SECONDS=1.0 # TENDWIRE_MAX_WORKERS. The internal refresh queue is fixed at 64. TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS=2.0 TENDWIRE_TURN_REFRESH_WORKERS=4 +TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS=86400 # Backend prompts remain visible as stale after a transient adapter failure for # one fixed, non-sliding grace window. Repeated failures do not extend it. diff --git a/INSTALL.md b/INSTALL.md index 4a8ace7..d7a7443 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -45,6 +45,7 @@ Environment=TENDWIRE_HERDR_BACKEND=socket Environment=TENDWIRE_DB_PATH=%h/.local/share/tendwire/tendwire.db Environment=TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS=2.0 Environment=TENDWIRE_TURN_REFRESH_WORKERS=4 +Environment=TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS=86400 ExecStart=%h/.local/bin/tendwire daemon --db-path %h/.local/share/tendwire/tendwire.db Restart=always RestartSec=5s @@ -71,6 +72,8 @@ reconciles. `TENDWIRE_TURN_REFRESH_WORKERS` defaults to `4`, must be from 1 through 32, and cannot exceed `TENDWIRE_MAX_WORKERS`. Every adapter uses `TENDWIRE_HERDR_TIMEOUT_SECONDS`; the queue is fixed at 64. One private target is serialized with itself while distinct targets can use the worker pool. +Unobserved command-turn claims become terminal tombstones after +`TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS` (default `86400`). OMP JSONL cache/IPC state is coordinate-only: parse/EOF and replay offsets, observed file identity/size/timestamps, an open-turn flag, and validated project diff --git a/README.md b/README.md index 5581acd..55627a8 100644 --- a/README.md +++ b/README.md @@ -545,6 +545,7 @@ variables: | `store_maintenance_cadence_seconds` | `TENDWIRE_STORE_MAINTENANCE_CADENCE_SECONDS` | `3600` | positive integer | | `turn_refresh_interval_seconds` | `TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS` | `2.0` | finite positive float | | `turn_refresh_workers` | `TENDWIRE_TURN_REFRESH_WORKERS` | `4` | integer from 1 through 32 and no greater than `max_workers` | +| `turn_claim_hard_ttl_seconds` | `TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS` | `86400` | positive integer; unobserved command claims become terminal after this interval | The socket/event backend uses `event_debounce_seconds` for event batching and `reconcile_interval_seconds` for bounded periodic full reconciles. Set diff --git a/src/tendwire/backends/herdr_turns.py b/src/tendwire/backends/herdr_turns.py index b22cc49..1346527 100644 --- a/src/tendwire/backends/herdr_turns.py +++ b/src/tendwire/backends/herdr_turns.py @@ -39,6 +39,7 @@ apply_turn_refresh, list_worker_bindings, prune_backend_pending, + sweep_turn_claims, ) @@ -4398,6 +4399,17 @@ def _scan_bindings(self) -> None: except Exception: pass + try: + sweep_turn_claims( + self.config.db_path, + self.config.host_id, + grace_seconds=2.0 * self.refresh_interval_seconds, + hard_ttl_seconds=self.config.turn_claim_hard_ttl_seconds, + now=self._utc_clock(), + ) + except Exception: + pass + def _binding_for_item(self, item: _TurnRefreshItem) -> WorkerBinding | None: if self.config.db_path is None: return None diff --git a/src/tendwire/cli.py b/src/tendwire/cli.py index bf542a7..a07619a 100644 --- a/src/tendwire/cli.py +++ b/src/tendwire/cli.py @@ -1032,6 +1032,8 @@ def cmd_turns( limit=limit, cursor=cursor, since=since, + turn_refresh_interval_seconds=config.turn_refresh_interval_seconds, + claim_hard_ttl_seconds=config.turn_claim_hard_ttl_seconds, ) elif daemon_attempt.error_kind == "timeout": payload = { diff --git a/src/tendwire/command_submission.py b/src/tendwire/command_submission.py index 228b362..b1b7405 100644 --- a/src/tendwire/command_submission.py +++ b/src/tendwire/command_submission.py @@ -58,6 +58,7 @@ claim_backend_pending_choice, claim_backend_pending_decision, command_pending_turn_terminal_effect, + delete_command_pending_turn_claim_effect, command_reservation_is_live, envelope_to_receipt_json, finish_command_request, @@ -69,6 +70,7 @@ reserve_terminal_command_replay, start_backend_pending_choice_send, start_backend_pending_decision_send, + upsert_command_pending_turn, ) @@ -349,13 +351,16 @@ def _submit_private_pane_input(client: Any, pane_id: str, instruction_text: str, # A single ctrl+u is not reliable across all foreground TUIs. Clear stale # input first, then submit text and Enter in one Herdr operation so the # foreground application cannot observe a staged prompt between requests. - for keys in _PRIVATE_PANE_CLEAR_KEY_SEQUENCES: - _socket_request( - client, - "pane.send_keys", - {"pane_id": pane_id, "keys": list(keys)}, - timeout=timeout, - ) + try: + for keys in _PRIVATE_PANE_CLEAR_KEY_SEQUENCES: + _socket_request( + client, + "pane.send_keys", + {"pane_id": pane_id, "keys": list(keys)}, + timeout=timeout, + ) + except Exception as exc: + raise _PaneInputNotStartedError from exc _socket_request( client, "pane.send_input", @@ -364,6 +369,10 @@ def _submit_private_pane_input(client: Any, pane_id: str, instruction_text: str, ) +class _PaneInputNotStartedError(RuntimeError): + """The instruction input operation was never attempted.""" + + def _target_state_at_send(worker: Worker) -> str: status = str(worker.status or "").strip().lower().replace("-", "_") return status or "unknown" @@ -1253,10 +1262,22 @@ def _mark_request_send_started( reservation: ReservedCommandMutation, *, binding_fingerprint: str, -) -> CommandEnvelope | None: + worker: Worker | None = None, + instruction_text: str | None = None, +) -> CommandEnvelope | Mapping[str, Any] | None: if config.db_path is None: return _backend_uncertain(request, "command receipt store is unavailable") try: + send_started_effect = ( + command_pending_turn_terminal_effect( + host_id=config.host_id, + worker=worker, + request_id=request.request_id or "", + instruction_text=instruction_text, + ) + if worker is not None and instruction_text is not None + else None + ) started = mark_command_send_started( config.db_path, host_id=config.host_id, @@ -1264,6 +1285,7 @@ def _mark_request_send_started( canonical_fingerprint=reservation.canonical.fingerprint, owner_token=reservation.owner_token, binding_fingerprint=binding_fingerprint, + send_started_effect=send_started_effect, event_payload=_transition_payload( request, worker_id=reservation.canonical.public_worker_id, @@ -1283,7 +1305,12 @@ def _mark_request_send_started( and started["receipt"].get("state") == "send_started" and _receipt_is_canonical(request, reservation.canonical, started["receipt"]) ): - return None + if send_started_effect is None: + return None + effect_result = started.get("effect_result") + if isinstance(effect_result, Mapping): + return effect_result + return _recover_request(config, request, reservation.canonical) if isinstance(started, Mapping) and isinstance(started.get("receipt"), Mapping): embedded = _envelope_from_receipt( request, @@ -1304,7 +1331,13 @@ def _mark_request_send_started( def _accepted_send_envelope( request: CommandRequest, worker: Worker, + turn: Mapping[str, Any], ) -> CommandEnvelope: + observed_turn_state = "pending_observation" + if str(turn.get("source_turn_id") or "").strip(): + observed_turn_state = ( + "complete" if turn.get("complete") is True else "observed" + ) return CommandEnvelope.from_result( request, ok=True, @@ -1315,7 +1348,8 @@ def _accepted_send_envelope( "delivery_state": "submitted", "transport_state": "submitted", "target_state_at_send": _target_state_at_send(worker), - "observed_turn_state": "pending_observation", + "turn_id": str(turn.get("id") or ""), + "observed_turn_state": observed_turn_state, }, ) @@ -1329,14 +1363,23 @@ def _submit_instruction( ) -> CommandEnvelope: assert config.db_path is not None try: - send_start_error = _mark_request_send_started( + send_started = _mark_request_send_started( config, request, reservation, binding_fingerprint=prepared.binding_fingerprint, + worker=worker, + instruction_text=_instruction_text(request), ) - if send_start_error is not None: - return send_start_error + if isinstance(send_started, CommandEnvelope): + return send_started + if not isinstance(send_started, Mapping): + return _recover_request( + config, + request, + reservation.canonical, + ) + pending_turn = send_started try: _submit_private_pane_input( @@ -1345,6 +1388,23 @@ def _submit_instruction( _instruction_text(request), timeout=config.herdr_timeout_seconds, ) + except _PaneInputNotStartedError: + envelope = _backend_uncertain( + request, + "Herdr socket pane input did not start after send start", + ) + return _finish_request( + config, + request, + reservation, + envelope, + expected_state="send_started", + terminal_state="uncertain", + terminal_effect=delete_command_pending_turn_claim_effect( + host_id=config.host_id, + request_id=request.request_id or "", + ), + ) except Exception: # noqa: BLE001 envelope = _backend_uncertain( request, @@ -1358,23 +1418,27 @@ def _submit_instruction( expected_state="send_started", terminal_state="uncertain", ) + + # The ingestion scheduler can adopt the write-early claim while the + # pane call is in flight. Re-read through the idempotent upsert so the + # accepted envelope reports the canonical row's observed state rather + # than the pre-send snapshot of that row. + try: + refreshed_turn = upsert_command_pending_turn( + config.db_path, + config.host_id, + worker, + request_id=request.request_id or "", + instruction_text=_instruction_text(request), + ) + except Exception: # noqa: BLE001 + refreshed_turn = None + if isinstance(refreshed_turn, Mapping): + pending_turn = refreshed_turn finally: _close_socket_client(prepared.client) - accepted = _accepted_send_envelope(request, worker) - try: - effect = command_pending_turn_terminal_effect( - host_id=config.host_id, - worker=worker, - request_id=request.request_id or "", - instruction_text=_instruction_text(request), - ) - except Exception: - return _recover_request( - config, - request, - reservation.canonical, - ) + accepted = _accepted_send_envelope(request, worker, pending_turn) return _finish_request( config, request, @@ -1382,7 +1446,6 @@ def _submit_instruction( accepted, expected_state="send_started", terminal_state="accepted", - terminal_effect=effect, ) diff --git a/src/tendwire/config.py b/src/tendwire/config.py index 52ea3e6..adc8f40 100644 --- a/src/tendwire/config.py +++ b/src/tendwire/config.py @@ -21,6 +21,7 @@ DEFAULT_MAX_WORKERS = 512 DEFAULT_TURN_REFRESH_INTERVAL_SECONDS = 2.0 DEFAULT_TURN_REFRESH_WORKERS = 4 +DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS = 86_400 DEFAULT_PENDING_STALE_GRACE_SECONDS = 30.0 DEFAULT_MAX_OUTBOX_ATTEMPTS = 10 DEFAULT_CONNECTOR_CLAIM_TTL_SECONDS = 60 @@ -59,6 +60,7 @@ class Config: max_workers: int = DEFAULT_MAX_WORKERS turn_refresh_interval_seconds: float = DEFAULT_TURN_REFRESH_INTERVAL_SECONDS turn_refresh_workers: int = DEFAULT_TURN_REFRESH_WORKERS + turn_claim_hard_ttl_seconds: int = DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS pending_stale_grace_seconds: float = DEFAULT_PENDING_STALE_GRACE_SECONDS max_outbox_attempts: int = DEFAULT_MAX_OUTBOX_ATTEMPTS connector_claim_ttl_seconds: int = DEFAULT_CONNECTOR_CLAIM_TTL_SECONDS @@ -145,6 +147,15 @@ def __post_init__(self) -> None: ) if self.turn_refresh_workers > self.max_workers: raise ValueError("turn_refresh_workers must be <= max_workers") + object.__setattr__( + self, + "turn_claim_hard_ttl_seconds", + _bounded_positive_int( + self.turn_claim_hard_ttl_seconds, + "turn_claim_hard_ttl_seconds", + maximum=MAX_MAINTENANCE_CADENCE_SECONDS, + ), + ) object.__setattr__( self, "pending_stale_grace_seconds", @@ -353,6 +364,7 @@ def load_config( max_workers: int | str | None = None, turn_refresh_interval_seconds: float | str | None = None, turn_refresh_workers: int | str | None = None, + turn_claim_hard_ttl_seconds: int | str | None = None, pending_stale_grace_seconds: float | str | None = None, max_outbox_attempts: int | str | None = None, connector_claim_ttl_seconds: int | str | None = None, @@ -464,6 +476,11 @@ def load_config( "TENDWIRE_TURN_REFRESH_WORKERS", DEFAULT_TURN_REFRESH_WORKERS, ), + turn_claim_hard_ttl_seconds=_resolve_value( + turn_claim_hard_ttl_seconds, + "TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", + DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS, + ), pending_stale_grace_seconds=_resolve_value( pending_stale_grace_seconds, "TENDWIRE_PENDING_STALE_GRACE_SECONDS", diff --git a/src/tendwire/core/turns.py b/src/tendwire/core/turns.py index 089b2f4..cda5cfe 100644 --- a/src/tendwire/core/turns.py +++ b/src/tendwire/core/turns.py @@ -1784,6 +1784,8 @@ class Turn: completed_at: str | None = None origin_command_id: str | None = None source_turn_id: str | None = None + superseded_by_turn_id: str | None = None + superseded_at: str | None = None meta: dict[str, Any] = field(default_factory=dict) id: str = "" fingerprint: str = "" @@ -1808,6 +1810,10 @@ def __post_init__(self) -> None: completed_at = _optional_timestamp(self.completed_at) origin_command_id = _optional_public_text(self.origin_command_id) raw_source_turn_id = _optional_public_text(self.source_turn_id) + superseded_by_turn_id = _optional_public_text( + self.superseded_by_turn_id + ) + superseded_at = _optional_timestamp(self.superseded_at) meta = _clean_meta(self.meta) source_id_candidates = turn_source_id_candidates( raw_source_turn_id, @@ -1891,6 +1897,12 @@ def __post_init__(self) -> None: object.__setattr__(self, "source", source) object.__setattr__(self, "origin_command_id", origin_command_id) object.__setattr__(self, "source_turn_id", source_turn_id) + object.__setattr__( + self, + "superseded_by_turn_id", + superseded_by_turn_id, + ) + object.__setattr__(self, "superseded_at", superseded_at) object.__setattr__(self, "fingerprint", fingerprint) object.__setattr__(self, "meta", meta) @@ -1921,6 +1933,8 @@ def to_dict(self) -> dict[str, Any]: "source": self.source, "origin_command_id": self.origin_command_id, "source_turn_id": self.source_turn_id, + "superseded_by_turn_id": self.superseded_by_turn_id, + "superseded_at": self.superseded_at, "fingerprint": self.fingerprint, "meta": _clean_meta(self.meta), } @@ -1955,6 +1969,8 @@ def from_dict(cls, data: "Turn | Mapping[str, Any]") -> "Turn": source=clean.get("source", "snapshot"), origin_command_id=clean.get("origin_command_id"), source_turn_id=clean.get("source_turn_id"), + superseded_by_turn_id=clean.get("superseded_by_turn_id"), + superseded_at=clean.get("superseded_at"), fingerprint=_string_value(clean.get("fingerprint")), meta=clean.get("meta", {}), ) diff --git a/src/tendwire/daemon.py b/src/tendwire/daemon.py index fb8a16d..2a90dbe 100644 --- a/src/tendwire/daemon.py +++ b/src/tendwire/daemon.py @@ -1021,6 +1021,8 @@ def get_turns( limit=limit, cursor=cursor, since=since, + turn_refresh_interval_seconds=self.config.turn_refresh_interval_seconds, + claim_hard_ttl_seconds=self.config.turn_claim_hard_ttl_seconds, ) def get_turn_content(self, params: Mapping[str, Any]) -> Mapping[str, Any]: diff --git a/src/tendwire/store/sqlite.py b/src/tendwire/store/sqlite.py index 137df0e..37fc3ac 100644 --- a/src/tendwire/store/sqlite.py +++ b/src/tendwire/store/sqlite.py @@ -107,7 +107,8 @@ FINGERPRINT_HEX_LENGTH = FINGERPRINT_HEX_CHARS -STORE_SCHEMA_VERSION = 14 +STORE_SCHEMA_VERSION = 15 +TURN_CLAIM_HARD_TTL_SECONDS = 86_400 ACKNOWLEDGED_FINAL_RETENTION_DAYS = 30 ACKNOWLEDGED_FINAL_RETENTION_COUNT = 4096 COMMAND_RETRY_HORIZON_SECONDS = 604_800 @@ -10821,6 +10822,110 @@ def _migrate_v13_to_v14_conn(conn: sqlite3.Connection) -> None: conn.execute(statement) +def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: + """Tombstone duplicate and stale command claims without rekeying turns.""" + if not _table_columns(conn, "turns"): + return + now = utc_timestamp() + now_dt = datetime.fromisoformat(now) + rows = conn.execute( + """ + SELECT turns.host_id, turns.turn_id, turns.worker_id, + turns.payload_json, turns.observed_at, + revisions.user_text, revisions.assistant_final_text, + revisions.user_state, revisions.final_state + FROM turns + LEFT JOIN turn_content_revisions AS revisions + ON revisions.host_id = turns.host_id + AND revisions.turn_id = turns.turn_id + AND revisions.is_current = 1 + """ + ).fetchall() + decoded = [] + for row in rows: + payload = _json_object(row[3]) + current = ( + { + "user_text": row[5], + "assistant_final_text": row[6], + "user_state": str(row[7]), + "final_state": str(row[8]), + } + if row[7] is not None + else None + ) + decoded.append( + ( + str(row[0]), + str(row[1]), + str(row[2]), + payload, + current, + str(row[4] or ""), + ) + ) + claims = [ + row + for row in decoded + if str(row[3].get("source") or "") == "command" + and not str(row[3].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[3]) + and row[3].get("complete") is not True + ] + done = [ + row + for row in decoded + if str(row[3].get("source_turn_id") or "").strip() + and ( + row[3].get("complete") is True + or row[4] is not None + and str(row[4].get("final_state") or "") == "complete" + ) + ] + affected_hosts: set[str] = set() + for claim in claims: + claim_view = _turn_with_current_content(claim[3], claim[4]) + matches = [ + observed + for observed in done + if observed[0] == claim[0] + and observed[2] == claim[2] + and _turn_content_matches_origin( + _turn_with_current_content(observed[3], observed[4]), + claim_view, + ) + ] + if len(matches) == 1 and _tombstone_turn_conn( + conn, + claim[0], + claim[1], + superseded_by_turn_id=matches[0][1], + superseded_at=now, + ): + affected_hosts.add(claim[0]) + + for claim in claims: + stored = conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (claim[0], claim[1]), + ).fetchone() + if stored is None or _turn_is_tombstoned(_json_object(stored[0])): + continue + claim_dt = _turn_row_time(claim[3], claim[5]) + if claim_dt is None or (now_dt - claim_dt).total_seconds() < TURN_CLAIM_HARD_TTL_SECONDS: + continue + if _tombstone_turn_conn( + conn, + claim[0], + claim[1], + superseded_by_turn_id=None, + superseded_at=now, + ): + affected_hosts.add(claim[0]) + for host_id in sorted(affected_hosts): + _increment_turn_list_generation_conn(conn, host_id) + + MIGRATIONS: tuple[Migration, ...] = ( Migration(0, 1, _migrate_v0_to_v1_conn), Migration(1, 2, _migrate_v1_to_v2_conn), @@ -10836,6 +10941,7 @@ def _migrate_v13_to_v14_conn(conn: sqlite3.Connection) -> None: Migration(11, 12, _migrate_v11_to_v12_conn), Migration(12, 13, _migrate_v12_to_v13_conn), Migration(13, 14, _migrate_v13_to_v14_conn), + Migration(14, 15, _migrate_v14_to_v15_conn), ) @@ -15532,6 +15638,9 @@ def _normalized_persisted_turn_payload( turn_id: str, payload: Mapping[str, Any], stored_payload: Mapping[str, Any], + *, + adopt_source_turn_id: bool = False, + adopt_origin_command_id: bool = False, ) -> dict[str, Any]: """Sanitize a row update while retaining every published identity.""" stored_source_turn_id = str( @@ -15540,26 +15649,38 @@ def _normalized_persisted_turn_payload( stored_origin_command_id = str( stored_payload.get("origin_command_id") or "" ).strip() + incoming_origin_command_id = str( + payload.get("origin_command_id") or "" + ).strip() + origin_command_id = ( + stored_origin_command_id + or (incoming_origin_command_id if adopt_origin_command_id else "") + ) stored_kind = str(stored_payload.get("kind") or "").strip() normalized = Turn.from_dict(payload).to_dict() - if stored_source_turn_id: - normalized["source_turn_id"] = stored_source_turn_id + source_turn_id = stored_source_turn_id or ( + str(normalized.get("source_turn_id") or "").strip() + if adopt_source_turn_id + else "" + ) + if source_turn_id: + normalized["source_turn_id"] = source_turn_id else: normalized.pop("source_turn_id", None) - if stored_origin_command_id: - normalized["origin_command_id"] = stored_origin_command_id + if origin_command_id: + normalized["origin_command_id"] = origin_command_id else: normalized.pop("origin_command_id", None) # Recompute the fingerprint from the preserved compatibility token and # provenance, then restore the published row identity after normalization. normalized = Turn.from_dict(normalized).to_dict() normalized["id"] = str(turn_id) - if stored_source_turn_id: - normalized["source_turn_id"] = stored_source_turn_id + if source_turn_id: + normalized["source_turn_id"] = source_turn_id else: normalized.pop("source_turn_id", None) - if stored_origin_command_id: - normalized["origin_command_id"] = stored_origin_command_id + if origin_command_id: + normalized["origin_command_id"] = origin_command_id else: normalized.pop("origin_command_id", None) if stored_kind: @@ -15579,12 +15700,16 @@ def _update_persisted_turn_row( current_time: str, *, snapshot_content_fingerprint: str | None = None, + adopt_source_turn_id: bool = False, + adopt_origin_command_id: bool = False, ) -> tuple[bool, dict[str, Any]]: """Update a current projection without rekeying its persisted turn.""" item = _normalized_persisted_turn_payload( str(turn_id), payload, stored_payload, + adopt_source_turn_id=adopt_source_turn_id, + adopt_origin_command_id=adopt_origin_command_id, ) encoded = _canonical_json(item) row = conn.execute( @@ -17724,6 +17849,318 @@ def _turn_with_current_content( return merged +def _turn_is_tombstoned(payload: Mapping[str, Any]) -> bool: + return bool(str(payload.get("superseded_at") or "").strip()) + + +def _tombstone_turn_conn( + conn: sqlite3.Connection, + host_id: str, + turn_id: str, + *, + superseded_by_turn_id: str | None, + superseded_at: str, +) -> bool: + row = conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (str(host_id), str(turn_id)), + ).fetchone() + if row is None: + return False + stored = _json_object(row[0]) + if _turn_is_tombstoned(stored): + return False + payload = dict(stored) + payload.update( + { + "status": "closed", + "complete": True, + "has_open_turn": False, + "assistant_stream_text": None, + "completed_at": payload.get("completed_at") or superseded_at, + "updated_at": superseded_at, + "superseded_by_turn_id": ( + str(superseded_by_turn_id) + if str(superseded_by_turn_id or "").strip() + else None + ), + "superseded_at": superseded_at, + } + ) + _update_persisted_turn_row( + conn, + str(host_id), + str(turn_id), + payload, + stored, + superseded_at, + ) + return True + + +def _tombstone_matching_command_sibling_conn( + conn: sqlite3.Connection, + host_id: str, + worker_id: str, + completing_turn_id: str, + *, + observed_at: str, +) -> bool: + rows = _current_turn_content_rows_conn(conn, host_id, worker_id) + completing = next( + (row for row in rows if str(row[0]) == str(completing_turn_id)), + None, + ) + if completing is None: + return False + completing_view = _turn_with_current_content(completing[1], completing[2]) + if not ( + completing_view.get("complete") is True + or completing[2] is not None + and str(completing[2].get("final_state") or "") == "complete" + ): + return False + candidates = [ + row + for row in rows + if str(row[0]) != str(completing_turn_id) + and str(row[1].get("source") or "") == "command" + and not str(row[1].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[1]) + and row[1].get("has_open_turn") is True + and _turn_content_matches_origin( + _turn_with_current_content(row[1], row[2]), + completing_view, + ) + ] + if len(candidates) != 1: + return False + return _tombstone_turn_conn( + conn, + str(host_id), + str(candidates[0][0]), + superseded_by_turn_id=str(completing_turn_id), + superseded_at=str(observed_at), + ) + + +def _turn_row_time(payload: Mapping[str, Any], observed_at: str) -> datetime | None: + for value in ( + payload.get("started_at"), + payload.get("updated_at"), + observed_at, + ): + timestamp = _strict_utc_timestamp(value) + if timestamp is not None: + return datetime.fromisoformat(timestamp) + return None + + +def _sweep_turn_claims_conn( + conn: sqlite3.Connection, + host_id: str, + *, + grace_seconds: float, + hard_ttl_seconds: float, + now: str, +) -> int: + current_timestamp = _strict_utc_timestamp(now) + if current_timestamp is None: + raise ValueError("invalid sweep timestamp") + current_dt = datetime.fromisoformat(current_timestamp) + has_claim = conn.execute( + """ + SELECT 1 + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.source') = 'command' + AND COALESCE(json_extract(payload_json, '$.origin_command_id'), '') != '' + AND COALESCE(json_extract(payload_json, '$.source_turn_id'), '') = '' + AND COALESCE(json_extract(payload_json, '$.superseded_at'), '') = '' + AND json_extract(payload_json, '$.has_open_turn') = 1 + LIMIT 1 + """, + (str(host_id),), + ).fetchone() + if has_claim is None: + return 0 + rows = conn.execute( + """ + SELECT turns.turn_id, turns.worker_id, turns.payload_json, + turns.observed_at, revisions.user_text, + revisions.assistant_final_text, revisions.user_state, + revisions.final_state + FROM turns + LEFT JOIN turn_content_revisions AS revisions + ON revisions.host_id = turns.host_id + AND revisions.turn_id = turns.turn_id + AND revisions.is_current = 1 + WHERE turns.host_id = ? + """, + (str(host_id),), + ).fetchall() + decoded: list[tuple[str, str, dict[str, Any], dict[str, Any] | None, str]] = [] + for row in rows: + payload = _json_object(row[2]) + current = ( + { + "user_text": row[4], + "assistant_final_text": row[5], + "user_state": str(row[6]), + "final_state": str(row[7]), + } + if row[6] is not None + else None + ) + decoded.append((str(row[0]), str(row[1]), payload, current, str(row[3] or ""))) + + claims = [ + row + for row in decoded + if str(row[2].get("source") or "") == "command" + and str(row[2].get("origin_command_id") or "").strip() + and not str(row[2].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[2]) + and row[2].get("has_open_turn") is True + ] + claims.sort( + key=lambda row: ( + _turn_row_time(row[2], row[4]) or current_dt, + row[0], + ) + ) + done = [ + row + for row in decoded + if str(row[2].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[2]) + and ( + row[2].get("complete") is True + or row[3] is not None + and str(row[3].get("final_state") or "") == "complete" + ) + ] + used_done = { + str(row[2].get("superseded_by_turn_id") or "") + for row in decoded + if _turn_is_tombstoned(row[2]) + } + changed = 0 + unresolved: list[tuple[str, str, dict[str, Any], dict[str, Any] | None, str]] = [] + for claim in claims: + claim_dt = _turn_row_time(claim[2], claim[4]) + if claim_dt is None or (current_dt - claim_dt).total_seconds() < grace_seconds: + unresolved.append(claim) + continue + claim_view = _turn_with_current_content(claim[2], claim[3]) + matches = [ + candidate + for candidate in done + if candidate[1] == claim[1] + and candidate[0] not in used_done + and _turn_content_matches_origin( + _turn_with_current_content(candidate[2], candidate[3]), + claim_view, + ) + ] + if len(matches) == 1: + if _tombstone_turn_conn( + conn, + str(host_id), + claim[0], + superseded_by_turn_id=matches[0][0], + superseded_at=current_timestamp, + ): + changed += 1 + used_done.add(matches[0][0]) + continue + unresolved.append(claim) + + for worker_id in sorted({row[1] for row in unresolved}): + worker_claims = sorted( + (row for row in unresolved if row[1] == worker_id), + key=lambda row: (_turn_row_time(row[2], row[4]) or current_dt, row[0]), + ) + if not worker_claims: + continue + claim = worker_claims[0] + claim_dt = _turn_row_time(claim[2], claim[4]) + if claim_dt is None or (current_dt - claim_dt).total_seconds() < grace_seconds: + continue + newer_done = sorted( + ( + row + for row in done + if row[1] == worker_id + and row[0] not in used_done + and (_turn_row_time(row[2], row[4]) or current_dt) > claim_dt + ), + key=lambda row: (_turn_row_time(row[2], row[4]) or current_dt, row[0]), + ) + if newer_done and _tombstone_turn_conn( + conn, + str(host_id), + claim[0], + superseded_by_turn_id=newer_done[0][0], + superseded_at=current_timestamp, + ): + changed += 1 + used_done.add(newer_done[0][0]) + + for claim in claims: + row = conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (str(host_id), claim[0]), + ).fetchone() + if row is None or _turn_is_tombstoned(_json_object(row[0])): + continue + claim_dt = _turn_row_time(claim[2], claim[4]) + if claim_dt is None or (current_dt - claim_dt).total_seconds() < hard_ttl_seconds: + continue + if _tombstone_turn_conn( + conn, + str(host_id), + claim[0], + superseded_by_turn_id=None, + superseded_at=current_timestamp, + ): + changed += 1 + return changed + + +def sweep_turn_claims( + db_path: Path | str, + host_id: str, + *, + grace_seconds: float, + hard_ttl_seconds: float = TURN_CLAIM_HARD_TTL_SECONDS, + now: str | None = None, +) -> int: + """Resolve or expire durable command claims without deleting referenced rows.""" + if grace_seconds <= 0 or hard_ttl_seconds <= 0: + raise ValueError("turn claim TTL values must be positive") + if not _sqlite_store_exists(db_path): + return 0 + current = now or utc_timestamp() + with _connect(db_path, isolation_level=None) as conn: + _ensure_schema(conn) + conn.execute("PRAGMA busy_timeout=50") + conn.execute("BEGIN IMMEDIATE") + try: + changed = _sweep_turn_claims_conn( + conn, + str(host_id), + grace_seconds=float(grace_seconds), + hard_ttl_seconds=float(hard_ttl_seconds), + now=current, + ) + conn.commit() + return changed + except Exception: + conn.rollback() + raise + + def _source_turn_matches(payload: Mapping[str, Any], incoming_source_turn: str) -> bool: stored = str(payload.get("source_turn_id") or "").strip() if not stored or not incoming_source_turn: @@ -18102,9 +18539,7 @@ def _merge_owned_turn_content_conn( command_predecessor_turn_id = predecessor_id else: command_rows = _owned_command_candidates(rows, automation_probe) - if len(command_rows) > 1: - raise StoreSchemaError("turn_owner_command_ambiguous") - if command_rows: + if len(command_rows) == 1: base_row = command_rows[0] else: placeholder_rows = _owned_placeholder_candidates(rows) @@ -18129,80 +18564,126 @@ def _merge_owned_turn_content_conn( ) if incoming_source_turn: - seed = { - key: base_payload.get(key) - for key in _TURN_IDENTITY_SEED_FIELDS - if base_payload.get(key) is not None - } - if base_row is None or base_row not in command_rows: + if len(command_rows) == 1: + ( + command_turn_id, + command_payload, + command_current, + _command_observed_at, + ) = command_rows[0] + payload = _adopt_turn_projection( + command_payload, + current_projection, + ) + payload.update( + _retain_authoritative_completion( + clean_content, + command_current, + payload, + incoming_final=incoming_final, + observed_at=observed_at, + ) + ) + payload["source_turn_id"] = incoming_source_turn + metadata_changed, persisted_item = _update_persisted_turn_row( + conn, + str(host_id), + str(command_turn_id), + payload, + command_payload, + observed_at, + snapshot_content_fingerprint=snapshot_content_fingerprint, + adopt_source_turn_id=True, + ) + revision_changed = _replace_current_turn_content_conn( + conn, + host_id=str(host_id), + turn_id=str(command_turn_id), + current=command_current, + incoming_user=incoming_user, + incoming_final=incoming_final, + current_time=observed_at, + ) + revision_repaired = _ensure_absent_turn_content_revision_conn( + conn, + host_id=str(host_id), + turn_id=str(command_turn_id), + observed_at=observed_at, + ) + selected_turn_id = str(command_turn_id) + changed = ( + metadata_changed + or revision_changed + or revision_repaired + ) + else: + seed = { + key: base_payload.get(key) + for key in _TURN_IDENTITY_SEED_FIELDS + if base_payload.get(key) is not None + } seed.pop("origin_command_id", None) if str(seed.get("source") or "") == "command": seed["source"] = "snapshot" - seed.update( - _retain_authoritative_completion( - clean_content, - None, - {}, + seed.update( + _retain_authoritative_completion( + clean_content, + None, + {}, + incoming_final=incoming_final, + observed_at=observed_at, + ) + ) + item = _strip_canonical_turn_payload( + Turn.from_dict(seed).to_dict() + ) + turn_id = str(item.get("id") or "") + collision = conn.execute( + """ + SELECT payload_json + FROM turns + WHERE host_id = ? AND turn_id = ? + """, + (str(host_id), turn_id), + ).fetchone() + if collision is not None: + collision_payload = _json_object(collision[0]) + if ( + _turn_continuity_identity(collision_payload) + != _turn_continuity_identity(item) + or not _owned_source_turn_matches( + collision_payload, + incoming_source_turn, + ) + ): + raise StoreSchemaError( + "turn_owner_source_identity_conflict" + ) + raise StoreSchemaError("turn_owner_source_ambiguous") + turn_id = _insert_owned_turn_conn( + conn, + host_id=str(host_id), + item=item, + snapshot_content_fingerprint=snapshot_content_fingerprint, + observed_at=observed_at, + ) + _replace_current_turn_content_conn( + conn, + host_id=str(host_id), + turn_id=turn_id, + current=None, + incoming_user=incoming_user, incoming_final=incoming_final, + current_time=observed_at, + ) + _ensure_absent_turn_content_revision_conn( + conn, + host_id=str(host_id), + turn_id=turn_id, observed_at=observed_at, ) - ) - item = _strip_canonical_turn_payload( - Turn.from_dict(seed).to_dict() - ) - turn_id = str(item.get("id") or "") - collision = conn.execute( - """ - SELECT payload_json - FROM turns - WHERE host_id = ? AND turn_id = ? - """, - (str(host_id), turn_id), - ).fetchone() - if collision is not None: - collision_payload = _json_object(collision[0]) - if ( - _turn_continuity_identity(collision_payload) - != _turn_continuity_identity(item) - or not _owned_source_turn_matches( - collision_payload, - incoming_source_turn, - ) - ): - raise StoreSchemaError( - "turn_owner_source_identity_conflict" - ) - # A matching physical row would have appeared in the exact - # tier above. Treat its absence as corruption, never overwrite. - raise StoreSchemaError("turn_owner_source_ambiguous") - turn_id = _insert_owned_turn_conn( - conn, - host_id=str(host_id), - item=item, - snapshot_content_fingerprint=snapshot_content_fingerprint, - observed_at=observed_at, - ) - _replace_current_turn_content_conn( - conn, - host_id=str(host_id), - turn_id=turn_id, - current=None, - incoming_user=incoming_user, - incoming_final=incoming_final, - current_time=observed_at, - ) - _ensure_absent_turn_content_revision_conn( - conn, - host_id=str(host_id), - turn_id=turn_id, - observed_at=observed_at, - ) - selected_turn_id = turn_id - if command_rows: - predecessor_id = str(command_rows[0][0]) - if predecessor_id != selected_turn_id: - command_predecessor_turn_id = predecessor_id - changed = True + selected_turn_id = turn_id + changed = True elif base_row is not None: authoritative_older = ( _turn_has_authoritative_observation( @@ -18323,14 +18804,20 @@ def _merge_owned_turn_content_conn( if current_revision is not None else None ) - if ( - anchor_id is not None - and command_predecessor_turn_id is not None - ): - _delete_turn_if_unreferenced_conn( + _tombstone_matching_command_sibling_conn( + conn, + str(host_id), + str(current_projection.get("worker_id") or ""), + selected_turn_id, + observed_at=str(observed_at), + ) + if anchor_id is not None and command_predecessor_turn_id is not None: + _tombstone_turn_conn( conn, str(host_id), command_predecessor_turn_id, + superseded_by_turn_id=selected_turn_id, + superseded_at=str(observed_at), ) return int(changed) @@ -18650,11 +19137,20 @@ def _merge_turn_content_conn( if current_revision is not None else None ) + _tombstone_matching_command_sibling_conn( + conn, + str(host_id), + str(worker_id), + selected_turn_id, + observed_at=str(observed_at), + ) if anchor_id is not None and command_predecessor_turn_id is not None: - _delete_turn_if_unreferenced_conn( + _tombstone_turn_conn( conn, str(host_id), command_predecessor_turn_id, + superseded_by_turn_id=selected_turn_id, + superseded_at=str(observed_at), ) return int(changed) @@ -19050,6 +19546,51 @@ def _upsert_command_pending_turn_impl( if owns_transaction: conn.commit() return sanitize_public_mapping(existing_item) + observation_rows = [ + row + for row in owned_rows + if str(row[1].get("source_turn_id") or "").strip() + and not str( + row[1].get("origin_command_id") or "" + ).strip() + and not _turn_is_tombstoned(row[1]) + and _turn_content_matches_origin( + _turn_with_current_content(row[1], row[2]), + {"user_text": clean_text}, + ) + ] + if len(observation_rows) == 1: + ( + persisted_turn_id, + stored_payload, + current, + _stored_observed_at, + ) = observation_rows[0] + adopted_payload = _adopt_turn_projection( + stored_payload, + item, + ) + adopted_payload["origin_command_id"] = clean_request_id + adopted_payload["source"] = "command" + _metadata_changed, persisted_item = ( + _update_persisted_turn_row( + conn, + str(host_id), + str(persisted_turn_id), + adopted_payload, + stored_payload, + current_time, + snapshot_content_fingerprint=content_fingerprint, + adopt_origin_command_id=True, + ) + ) + existing_item = _turn_with_current_content( + persisted_item, + current, + ) + if owns_transaction: + conn.commit() + return sanitize_public_mapping(existing_item) command_rows = [ row for row in owned_rows @@ -19270,9 +19811,9 @@ def command_pending_turn_terminal_effect( worker: Any, request_id: str, instruction_text: str, -) -> Callable[[sqlite3.Connection], None]: - """Build a pending-turn effect for a command terminal transaction.""" - def effect(conn: sqlite3.Connection) -> None: +) -> Callable[[sqlite3.Connection], dict[str, Any]]: + """Build a transactional pending-turn upsert effect.""" + def effect(conn: sqlite3.Connection) -> dict[str, Any]: item = _upsert_command_pending_turn_conn( conn, host_id, @@ -19283,6 +19824,44 @@ def effect(conn: sqlite3.Connection) -> None: ) if item is None: raise StoreSchemaError("command_pending_turn_terminal_effect_failed") + return item + + return effect + + +def delete_command_pending_turn_claim_effect( + *, + host_id: str, + request_id: str, +) -> Callable[[sqlite3.Connection], bool]: + """Build a transactional delete for a claim proven not to have been sent.""" + clean_request_id = str(request_id or "").strip() + + def effect(conn: sqlite3.Connection) -> bool: + rows = conn.execute( + """ + SELECT turn_id, payload_json + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.source') = 'command' + AND json_extract(payload_json, '$.origin_command_id') = ? + AND COALESCE( + json_extract(payload_json, '$.source_turn_id'), + '' + ) = '' + """, + (str(host_id), clean_request_id), + ).fetchall() + if not clean_request_id or len(rows) != 1: + return False + payload = _json_object(rows[0][1]) + if _turn_is_tombstoned(payload): + return False + return _delete_turn_if_unreferenced_conn( + conn, + str(host_id), + str(rows[0][0]), + ) return effect @@ -19298,6 +19877,8 @@ def turns_payload_from_store( since: str | None = None, now: float | int | None = None, work_counters: TurnContentWorkCounters | None = None, + turn_refresh_interval_seconds: float = 2.0, + claim_hard_ttl_seconds: float = TURN_CLAIM_HARD_TTL_SECONDS, ) -> dict[str, Any]: """Return one insertion-stable, byte-bounded turn-list page.""" requested_schema = int(schema_version) @@ -19328,6 +19909,16 @@ def turns_payload_from_store( "ok": False, "status": "store_unavailable", } + try: + sweep_turn_claims( + db_path, + str(host_id), + grace_seconds=2.0 * float(turn_refresh_interval_seconds), + hard_ttl_seconds=float(claim_hard_ttl_seconds), + ) + except Exception: + # Listing remains available if opportunistic maintenance is contended. + pass current_clock = time.time() if now is None else float(now) try: decoded_cursor = ( @@ -19529,6 +20120,10 @@ def turns_payload_from_store( WHERE turns.host_id = ? AND turns.list_sequence > ? AND turns.list_sequence <= ? + AND COALESCE( + json_extract(turns.payload_json, '$.superseded_at'), + '' + ) = '' {continuation} ORDER BY turns.worker_id ASC, @@ -21450,6 +22045,7 @@ def mark_command_send_started( owner_token: str, binding_fingerprint: str, event_payload: Mapping[str, Any] | None = None, + send_started_effect: Callable[[sqlite3.Connection], Any] | None = None, now: str | None = None, ) -> dict[str, Any]: """CAS the exact reserved owner to send_started before external mutation.""" @@ -21502,6 +22098,11 @@ def mark_command_send_started( row = _command_request_row(conn, host_id, request_id) conn.commit() return _command_request_response("not_owner", row) + effect_result = ( + send_started_effect(conn) + if send_started_effect is not None + else None + ) row = _command_request_row(conn, host_id, request_id) if row is None: raise RuntimeError("command request send-start disappeared") @@ -21513,9 +22114,12 @@ def mark_command_send_started( event_payload=event_payload, ) conn.commit() - return _command_request_response( + response = _command_request_response( "send_started", row, owner_token=str(owner_token) ) + if send_started_effect is not None: + response["effect_result"] = effect_result + return response except Exception: if conn.in_transaction: conn.rollback() diff --git a/tests/test_backend_pending.py b/tests/test_backend_pending.py index 4283a7f..c160cb5 100644 --- a/tests/test_backend_pending.py +++ b/tests/test_backend_pending.py @@ -1449,7 +1449,7 @@ def test_current_schema_creation_has_exact_binding_and_claim_state(tmp_path: Pat db = tmp_path / "current-schema.db" init_store(db) with sqlite3.connect(db) as conn: - assert conn.execute("PRAGMA user_version").fetchone()[0] == STORE_SCHEMA_VERSION == 14 + assert conn.execute("PRAGMA user_version").fetchone()[0] == STORE_SCHEMA_VERSION == 15 columns = { str(row[1]) for row in conn.execute("PRAGMA table_info(backend_pending)").fetchall() diff --git a/tests/test_cli.py b/tests/test_cli.py index 263aa97..f841af0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -480,6 +480,8 @@ def forbidden_snapshot(*_args: Any, **_kwargs: Any) -> Any: "limit": 7, "cursor": None, "since": None, + "turn_refresh_interval_seconds": 2.0, + "claim_hard_ttl_seconds": 86400, }, ) ] @@ -555,6 +557,8 @@ def read_page(_db_path: Path, host_id: str, **kwargs: Any) -> dict[str, Any]: "limit": 9, "cursor": position_value if position_flag == "--cursor" else None, "since": position_value if position_flag == "--since" else None, + "turn_refresh_interval_seconds": 2.0, + "claim_hard_ttl_seconds": 86400, } ] diff --git a/tests/test_command_submission.py b/tests/test_command_submission.py index a60c444..7cc2696 100644 --- a/tests/test_command_submission.py +++ b/tests/test_command_submission.py @@ -478,12 +478,14 @@ def test_submit_command_uses_socket_pane_input_once_and_caches_result(tmp_path: assert first.status == STATUS_ACCEPTED assert first.disposition == DISPOSITION_TERMINAL_ACCEPTED + assert str(first.result["turn_id"]).startswith("turn-") assert first.result == { "target": {"worker_id": "w-1"}, "delivery_state": "submitted", "transport_state": "submitted", "target_state_at_send": "active", "observed_turn_state": "pending_observation", + "turn_id": first.result["turn_id"], } assert second.to_dict() == first.to_dict() assert duplicate.status == STATUS_DUPLICATE_REQUEST @@ -580,13 +582,251 @@ def test_submit_command_uses_socket_pane_input_once_and_caches_result(tmp_path: _assert_no_private_json(surface) -def test_submit_command_sends_identical_100_character_instructions_with_distinct_ids( +def test_submit_command_writes_turn_claim_before_pane_send(tmp_path: Path) -> None: + config = _config(tmp_path) + assert config.db_path is not None + worker = Worker( + id="w-1", + name="Alpha", + status="active", + meta={ + "stable_key": "wsk1_" + ("a" * 64), + "stable_key_version": 1, + }, + ) + _seed(config, [worker], [_binding(worker)]) + calls: list[dict[str, Any]] = [] + claim_ids: list[str] = [] + + class InspectingClient(_FakeSocketClient): + def request(self, method, params, *, timeout=None): + if method == "pane.send_input": + with sqlite3.connect(str(config.db_path)) as conn: + claim_ids.extend( + str(row[0]) + for row in conn.execute( + """ + SELECT turn_id FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.origin_command_id') = ? + """, + (config.host_id, "write-early-request"), + ).fetchall() + ) + return super().request(method, params, timeout=timeout) + + accepted = submit_command( + config, + _request(request_id="write-early-request"), + socket_client_factory=lambda _config: InspectingClient(calls), + ) + + assert accepted.status == STATUS_ACCEPTED + assert claim_ids == [accepted.result["turn_id"]] + + +def test_submit_command_removes_claim_when_pane_input_never_starts( tmp_path: Path, ) -> None: config = _config(tmp_path) + assert config.db_path is not None worker = Worker(id="w-1", name="Alpha", status="active") _seed(config, [worker], [_binding(worker)]) calls: list[dict[str, Any]] = [] + + class ClearFailsClient(_FakeSocketClient): + def request(self, method, params, *, timeout=None): + if method == "pane.send_keys": + calls.append({"method": method, "params": dict(params)}) + raise HerdrSocketDisconnectedError("clear failed before input") + return super().request(method, params, timeout=timeout) + + envelope = submit_command( + config, + _request(request_id="clear-failure-request"), + socket_client_factory=lambda _config: ClearFailsClient(calls), + ) + + assert envelope.status == STATUS_REQUEST_STATE_UNCERTAIN + assert not any(call["method"] == "pane.send_input" for call in calls) + receipt = get_command_request( + config.db_path, + config.host_id, + "clear-failure-request", + ) + assert receipt is not None + assert receipt["state"] == "uncertain" + with sqlite3.connect(str(config.db_path)) as conn: + claims = conn.execute( + """ + SELECT turn_id + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.origin_command_id') = ? + """, + (config.host_id, "clear-failure-request"), + ).fetchall() + assert claims == [] + + +def test_observation_first_submission_adopts_source_row_in_place( + tmp_path: Path, +) -> None: + config = _config(tmp_path) + assert config.db_path is not None + worker = Worker( + id="w-1", + name="Alpha", + status="active", + meta={ + "stable_key": "wsk1_" + ("b" * 64), + "stable_key_version": 1, + }, + ) + _seed(config, [worker], [_binding(worker)]) + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "observation-first-source", + "user_text": "hello", + "assistant_final_text": "already observed", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T10:00:00+00:00", + ) == 1 + observed = next( + turn + for turn in turns_payload_from_store( + config.db_path, + config.host_id, + schema_version=2, + )["turns"] + if turn.get("assistant_final_text") == "already observed" + ) + + calls: list[dict[str, Any]] = [] + accepted = submit_command( + config, + _request(request_id="observation-first-request"), + socket_client_factory=_factory(calls), + ) + + assert accepted.status == STATUS_ACCEPTED + assert accepted.result["turn_id"] == observed["id"] + assert accepted.result["observed_turn_state"] == "complete" + assert calls == _expected_submit_calls() + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "observation-first-source", + "user_text": "hello", + "assistant_final_text": "tier one refresh", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T10:00:02+00:00", + ) == 1 + with sqlite3.connect(str(config.db_path)) as conn: + matching = conn.execute( + """ + SELECT turns.turn_id, turns.list_sequence + FROM turns + JOIN turn_content_revisions AS revisions + ON revisions.host_id = turns.host_id + AND revisions.turn_id = turns.turn_id + AND revisions.is_current = 1 + WHERE turns.host_id = ? AND revisions.user_text = 'hello' + """, + (config.host_id,), + ).fetchall() + assert len(matching) == 1 + assert matching[0][0] == observed["id"] + refreshed = turns_payload_from_store( + config.db_path, + config.host_id, + schema_version=2, + )["turns"] + assert len([turn for turn in refreshed if turn.get("source_turn_id")]) == 1 + assert next( + turn for turn in refreshed if turn.get("source_turn_id") + )["assistant_final_text"] == "tier one refresh" + + +def test_submission_first_envelope_reflects_observation_during_send( + tmp_path: Path, +) -> None: + config = _config(tmp_path) + assert config.db_path is not None + worker = Worker( + id="w-1", + name="Alpha", + status="active", + meta={ + "stable_key": "wsk1_" + ("c" * 64), + "stable_key_version": 1, + }, + ) + _seed(config, [worker], [_binding(worker)]) + calls: list[dict[str, Any]] = [] + + class ObservingClient(_FakeSocketClient): + def request(self, method, params, *, timeout=None): + result = super().request(method, params, timeout=timeout) + if method == "pane.send_input": + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "submission-first-source", + "user_text": "hello", + "assistant_final_text": "observed during send", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T10:00:00+00:00", + ) == 1 + return result + + accepted = submit_command( + config, + _request(request_id="submission-first-request"), + socket_client_factory=lambda _config: ObservingClient(calls), + ) + + assert accepted.status == STATUS_ACCEPTED + assert accepted.result["observed_turn_state"] == "complete" + turns = turns_payload_from_store( + config.db_path, + config.host_id, + schema_version=2, + )["turns"] + matching = [turn for turn in turns if turn.get("user_text") == "hello"] + assert len(matching) == 1 + assert matching[0]["id"] == accepted.result["turn_id"] + assert matching[0]["assistant_final_text"] == "observed during send" + + +def test_submit_command_sends_identical_100_character_instructions_with_distinct_ids( + tmp_path: Path, +) -> None: + config = _config(tmp_path) + worker = Worker( + id="w-1", + name="Alpha", + status="active", + meta={ + "stable_key": "wsk1_" + ("d" * 64), + "stable_key_version": 1, + }, + ) + _seed(config, [worker], [_binding(worker)]) + calls: list[dict[str, Any]] = [] text = "x" * 100 first = submit_command( @@ -602,6 +842,7 @@ def test_submit_command_sends_identical_100_character_instructions_with_distinct assert first.status == STATUS_ACCEPTED assert second.status == STATUS_ACCEPTED + assert first.result["turn_id"] != second.result["turn_id"] expected_send = [ {"method": "agent.get", "params": {"target": "agent-secret"}}, *_expected_private_clear_calls(), @@ -617,12 +858,64 @@ def test_submit_command_sends_identical_100_character_instructions_with_distinct assert second_receipt is not None assert second_receipt["state"] == "accepted" with sqlite3.connect(str(config.db_path)) as conn: + turn_count = conn.execute( + """ + SELECT COUNT(*) + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.source') = 'command' + AND json_extract(payload_json, '$.has_open_turn') = 1 + """, + (config.host_id,), + ).fetchone()[0] events = [ row[0] for row in conn.execute( "SELECT event_type FROM events WHERE aggregate_type = 'command_request' ORDER BY id" ).fetchall() ] + assert turn_count == 2 + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "ambiguous-identical-source", + "user_text": text, + "assistant_final_text": "independent observation", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T10:00:00+00:00", + ) == 1 + with sqlite3.connect(str(config.db_path)) as conn: + stored = [ + (str(turn_id), json.loads(payload_json)) + for turn_id, payload_json in conn.execute( + "SELECT turn_id, payload_json FROM turns WHERE host_id = ?", + (config.host_id,), + ).fetchall() + ] + relevant = [ + (turn_id, payload) + for turn_id, payload in stored + if payload.get("origin_command_id") + or payload.get("source_turn_id") + ] + assert len(relevant) == 3 + assert {first.result["turn_id"], second.result["turn_id"]}.issubset( + {turn_id for turn_id, _payload in relevant} + ) + observed = [ + (turn_id, payload) + for turn_id, payload in relevant + if payload.get("source_turn_id") + ] + assert len(observed) == 1 + assert observed[0][0] not in { + first.result["turn_id"], + second.result["turn_id"], + } assert events == [ "command.request.reserved", "command.request.send_started", @@ -807,12 +1100,14 @@ def test_submit_command_reports_submitted_transport_and_worker_state(tmp_path: P envelope = submit_command(config, _request(), socket_client_factory=_factory(calls)) assert envelope.status == STATUS_ACCEPTED + assert str(envelope.result["turn_id"]).startswith("turn-") assert envelope.result == { "target": {"worker_id": "w-1"}, "delivery_state": "submitted", "transport_state": "submitted", "target_state_at_send": "active", "observed_turn_state": "pending_observation", + "turn_id": envelope.result["turn_id"], } @@ -825,12 +1120,14 @@ def test_submit_command_marks_idle_worker_delivery_as_submitted(tmp_path: Path) envelope = submit_command(config, _request(), socket_client_factory=_factory(calls)) assert envelope.status == STATUS_ACCEPTED + assert str(envelope.result["turn_id"]).startswith("turn-") assert envelope.result == { "target": {"worker_id": "w-1"}, "delivery_state": "submitted", "transport_state": "submitted", "target_state_at_send": "idle", "observed_turn_state": "pending_observation", + "turn_id": envelope.result["turn_id"], } @@ -3325,7 +3622,7 @@ def test_stable_owner_pending_command_survives_worker_churn_and_source_wins( for turn in completed_payload["turns"] if turn.get("assistant_final_text") == "durable owner answer" ) - assert completed_source["id"] != command_before["id"] + assert completed_source["id"] == command_before["id"] assert completed_source["origin_command_id"] == request_id assert completed_source["source_turn_id"].startswith("turnsrc-") assert completed_source["source_turn_id"] != raw_source diff --git a/tests/test_config.py b/tests/test_config.py index 26c4620..5ff7cec 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -12,6 +12,7 @@ DEFAULT_COMMAND_RECEIPT_RETENTION_COUNT, DEFAULT_COMMAND_RECEIPT_RETENTION_SECONDS, DEFAULT_COMMAND_RETRY_HORIZON_SECONDS, + DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS, MAX_COMMAND_RETRY_HORIZON_SECONDS, MIN_COMMAND_RECEIPT_RETENTION_SECONDS, DEFAULT_TURN_REFRESH_INTERVAL_SECONDS, @@ -307,6 +308,7 @@ def test_command_receipt_retention_must_strictly_exceed_retry_horizon( TURN_REFRESH_ENV_NAMES = ( + "TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS", "TENDWIRE_TURN_REFRESH_WORKERS", ) @@ -320,25 +322,31 @@ def test_turn_refresh_knobs_have_documented_defaults(monkeypatch) -> None: assert DEFAULT_TURN_REFRESH_INTERVAL_SECONDS == 2.0 assert DEFAULT_TURN_REFRESH_WORKERS == 4 + assert DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS == 86_400 assert config.turn_refresh_interval_seconds == 2.0 assert config.turn_refresh_workers == 4 + assert config.turn_claim_hard_ttl_seconds == 86_400 def test_turn_refresh_knobs_use_explicit_before_environment(monkeypatch) -> None: monkeypatch.setenv("TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS", "3.5") monkeypatch.setenv("TENDWIRE_TURN_REFRESH_WORKERS", "8") + monkeypatch.setenv("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "7200") env_config = load_config(max_workers=16) explicit = load_config( max_workers=16, turn_refresh_interval_seconds="0.25", turn_refresh_workers="6", + turn_claim_hard_ttl_seconds="3600", ) assert env_config.turn_refresh_interval_seconds == 3.5 assert env_config.turn_refresh_workers == 8 + assert env_config.turn_claim_hard_ttl_seconds == 7200 assert explicit.turn_refresh_interval_seconds == 0.25 assert explicit.turn_refresh_workers == 6 + assert explicit.turn_claim_hard_ttl_seconds == 3600 @pytest.mark.parametrize("value", [0, -0.01, "nan", "inf", "-inf"]) diff --git a/tests/test_connector_outbox.py b/tests/test_connector_outbox.py index 54c770e..e9cbf69 100644 --- a/tests/test_connector_outbox.py +++ b/tests/test_connector_outbox.py @@ -1069,7 +1069,7 @@ def test_v6_to_current_plan_migration_is_bounded_atomic_and_preserves_jobs( ).fetchall() } foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall() - assert version == store_sqlite.STORE_SCHEMA_VERSION == 14 + assert version == store_sqlite.STORE_SCHEMA_VERSION == 15 assert plan_row == (plan["plan_token"], 1, None, "active") assert job_count == 2 assert outbox_count == 3 diff --git a/tests/test_daemon.py b/tests/test_daemon.py index 8f412c5..9f01105 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -932,6 +932,8 @@ def project( "limit": 17, "cursor": "twlist1.public", "since": None, + "turn_refresh_interval_seconds": 2.0, + "claim_hard_ttl_seconds": 86400, } for call in projection_calls ) diff --git a/tests/test_delivery_retention.py b/tests/test_delivery_retention.py index 7d3eb7d..3274524 100644 --- a/tests/test_delivery_retention.py +++ b/tests/test_delivery_retention.py @@ -84,6 +84,8 @@ def _source_turns(db_path: Path, snapshot: Any, *, host_id: str = HOST_ID) -> li snapshot=snapshot, schema_version=2, limit=250, + turn_refresh_interval_seconds=1_000_000_000, + claim_hard_ttl_seconds=1_000_000_000, ) return [ turn @@ -791,6 +793,8 @@ def test_completed_command_receipt_does_not_release_command_linked_pending_turn( snapshot=snapshot, schema_version=2, limit=250, + turn_refresh_interval_seconds=1_000_000_000, + claim_hard_ttl_seconds=1_000_000_000, )["turns"] protected = [ turn diff --git a/tests/test_delivery_retention_migration.py b/tests/test_delivery_retention_migration.py index 394b521..7d7d134 100644 --- a/tests/test_delivery_retention_migration.py +++ b/tests/test_delivery_retention_migration.py @@ -875,7 +875,7 @@ def test_v10_to_v12_migration_retains_finals_without_reposting_or_leaking( finals = _seed_v10_finals(db_path) init_store(db_path) - assert store_sqlite.STORE_SCHEMA_VERSION == 14 + assert store_sqlite.STORE_SCHEMA_VERSION == 15 delivered_key = _final_key(*finals["delivered"]) hold_keys = {_final_key(*finals[label]) for label in ("hold-a", "hold-b")} diff --git a/tests/test_delivery_retention_projection.py b/tests/test_delivery_retention_projection.py index 3ce5841..c0e5eee 100644 --- a/tests/test_delivery_retention_projection.py +++ b/tests/test_delivery_retention_projection.py @@ -165,7 +165,7 @@ def _turn_rows_snapshot(db_path: Path) -> tuple[tuple[Any, ...], ...]: def _assert_continuity_integrity(db_path: Path) -> None: with sqlite3.connect(str(db_path)) as conn: - assert conn.execute("PRAGMA user_version").fetchone() == (store_sqlite.STORE_SCHEMA_VERSION,) == (14,) + assert conn.execute("PRAGMA user_version").fetchone() == (store_sqlite.STORE_SCHEMA_VERSION,) == (15,) assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] current_counts = conn.execute( """ diff --git a/tests/test_delivery_retention_recovery.py b/tests/test_delivery_retention_recovery.py index 81f0ea7..37e2c7e 100644 --- a/tests/test_delivery_retention_recovery.py +++ b/tests/test_delivery_retention_recovery.py @@ -973,7 +973,7 @@ def test_v12_migration_uses_effective_recovery_lineage_without_repost_or_hold( api = ConnectorOutboxAPI(db_path, HOST_ID) assert api.poll({"name": FINAL_NAME, "limit": 100})["items"] == [] with sqlite3.connect(str(db_path)) as conn: - assert conn.execute("PRAGMA user_version").fetchone()[0] == store_sqlite.STORE_SCHEMA_VERSION == 14 + assert conn.execute("PRAGMA user_version").fetchone()[0] == store_sqlite.STORE_SCHEMA_VERSION == 15 anchor = conn.execute( """ SELECT delivery_kind, status diff --git a/tests/test_store.py b/tests/test_store.py index bee9f48..364c8ee 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -4953,6 +4953,66 @@ def test_command_pending_turn_terminal_effect_is_atomic_with_acceptance( assert turns[0]["user_text"] == "Continue safely." +def test_command_pending_turn_effect_is_atomic_with_send_start( + tmp_path: Path, +) -> None: + db_path = tmp_path / "pending-turn-send-start-effect.db" + init_store(db_path) + reservation = _reserve_test_request(db_path, request_id="turn-send-start") + invalid_effect = command_pending_turn_terminal_effect( + host_id="host-a", + worker={"id": "worker-public", "name": "Worker"}, + request_id="turn-send-start", + instruction_text="", + ) + with pytest.raises( + store_sqlite.StoreSchemaError, + match="command_pending_turn_terminal_effect_failed", + ): + mark_command_send_started( + db_path, + host_id="host-a", + request_id="turn-send-start", + canonical_fingerprint="send_instruction:turn-send-start", + owner_token=reservation["owner_token"], + binding_fingerprint="private-binding", + send_started_effect=invalid_effect, + now="2026-01-01T00:00:01+00:00", + ) + assert get_command_request( + db_path, + "host-a", + "turn-send-start", + )["state"] == "reserved" + assert turns_payload_from_store(db_path, "host-a")["turns"] == [] + + valid_effect = command_pending_turn_terminal_effect( + host_id="host-a", + worker={"id": "worker-public", "name": "Worker"}, + request_id="turn-send-start", + instruction_text="Continue safely.", + ) + started = mark_command_send_started( + db_path, + host_id="host-a", + request_id="turn-send-start", + canonical_fingerprint="send_instruction:turn-send-start", + owner_token=reservation["owner_token"], + binding_fingerprint="private-binding", + send_started_effect=valid_effect, + now="2026-01-01T00:00:01+00:00", + ) + assert started["status"] == "send_started" + assert started["effect_result"]["origin_command_id"] == "turn-send-start" + turns = turns_payload_from_store( + db_path, + "host-a", + claim_hard_ttl_seconds=1_000_000_000, + )["turns"] + assert len(turns) == 1 + assert turns[0]["id"] == started["effect_result"]["id"] + + def test_backend_pending_choice_terminal_effect_is_atomic_with_acceptance( tmp_path: Path, ) -> None: @@ -6384,6 +6444,321 @@ def test_twenty_offline_source_finals_are_retained_as_unique_ready_anchors( assert "source_turn_id" not in encoded_anchors +def test_turn_claim_sweeper_expires_never_observed_claim(tmp_path: Path) -> None: + db_path = tmp_path / "turn-claim-expiry.db" + host_id = "turn-claim-expiry-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("c" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="never-observed-request", + instruction_text="never observed", + observed_at="2026-07-19T00:00:00+00:00", + ) + assert claim is not None + + assert store_sqlite.sweep_turn_claims( + db_path, + host_id, + grace_seconds=1, + hard_ttl_seconds=60, + now="2026-07-19T00:02:00+00:00", + ) == 1 + + with sqlite3.connect(str(db_path)) as conn: + raw = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall() + assert raw["complete"] is True + assert raw["has_open_turn"] is False + assert raw["status"] == "closed" + assert raw["superseded_at"] == "2026-07-19T00:02:00+00:00" + assert raw["superseded_by_turn_id"] is None + assert claim["id"] not in { + turn["id"] + for turn in turns_payload_from_store(db_path, host_id)["turns"] + } + assert foreign_keys == [] + + +def test_turn_claim_sweeper_resolves_only_one_identical_claim_per_done_turn( + tmp_path: Path, +) -> None: + db_path = tmp_path / "turn-claim-identical.db" + host_id = "turn-claim-identical-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[{"id": "worker-1", "name": "Worker", "status": "active"}], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "identical-observed-source", + "user_text": "same prompt", + "assistant_final_text": "one answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T00:00:00+00:00", + ) == 1 + observed = next( + turn + for turn in turns_payload_from_store(db_path, host_id)["turns"] + if turn.get("source_turn_id") + ) + first = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="identical-first", + instruction_text="same prompt", + observed_at="2099-07-19T00:00:01+00:00", + ) + second = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="identical-second", + instruction_text="same prompt", + observed_at="2099-07-19T00:00:02+00:00", + ) + assert first is not None and second is not None + + assert store_sqlite.sweep_turn_claims( + db_path, + host_id, + grace_seconds=1, + now="2099-07-19T00:01:00+00:00", + ) == 1 + + with sqlite3.connect(str(db_path)) as conn: + stored = { + turn_id: json.loads(payload_json) + for turn_id, payload_json in conn.execute( + """ + SELECT turn_id, payload_json + FROM turns + WHERE host_id = ? AND turn_id IN (?, ?) + """, + (host_id, first["id"], second["id"]), + ).fetchall() + } + assert stored[first["id"]]["superseded_by_turn_id"] == observed["id"] + assert stored[second["id"]].get("superseded_at") is None + assert stored[second["id"]]["has_open_turn"] is True + + +def test_text_drift_tombstone_preserves_cursor_and_since_continuity( + tmp_path: Path, +) -> None: + db_path = tmp_path / "turn-claim-cursor.db" + host_id = "turn-claim-cursor-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("e" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + baseline = turns_payload_from_store(db_path, host_id, schema_version=2) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="drift-request", + instruction_text="final normalized prompt", + observed_at="2099-07-19T00:00:00+00:00", + ) + assert claim is not None + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "drift-source", + "user_text": "draft prompt", + "complete": False, + "has_open_turn": True, + }, + observed_at="2099-07-19T00:00:01+00:00", + ) == 1 + first_page = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + limit=1, + ) + source_id = first_page["turns"][0]["id"] + assert source_id != claim["id"] + assert first_page["has_more"] is True + assert first_page["next_cursor"] is not None + + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "drift-source", + "user_text": "final normalized prompt", + "assistant_final_text": "done", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T00:00:02+00:00", + ) == 1 + + continuation = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + limit=1, + cursor=first_page["next_cursor"], + ) + assert continuation.get("status") != "cursor_expired" + assert claim["id"] not in {turn["id"] for turn in continuation["turns"]} + since_poll = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + since=baseline["since"], + ) + assert {turn["id"] for turn in since_poll["turns"]} == {source_id} + with sqlite3.connect(str(db_path)) as conn: + physical = conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone() + assert physical is not None + claim_payload = json.loads(physical[0]) + assert claim_payload["superseded_by_turn_id"] == source_id + assert claim_payload["superseded_at"] == "2099-07-19T00:00:02+00:00" + + +def test_v14_migration_tombstones_production_shape_duplicate_claim( + tmp_path: Path, +) -> None: + db_path = tmp_path / "turn-claim-v14.db" + host_id = "turn-claim-v14-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("d" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "migration-observed-source", + "user_text": "same migration prompt", + "assistant_final_text": "migration answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:00:00+00:00", + ) == 1 + observed = next( + turn + for turn in turns_payload_from_store(db_path, host_id, schema_version=2)["turns"] + if turn.get("assistant_final_text") == "migration answer" + ) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="migration-command-request", + instruction_text="temporary different prompt", + observed_at="2026-07-19T00:00:01+00:00", + ) + assert claim is not None + assert claim["id"] != observed["id"] + with sqlite3.connect(str(db_path)) as conn: + conn.execute( + """ + UPDATE turn_content_revisions + SET user_text = 'same migration prompt' + WHERE host_id = ? AND turn_id = ? AND is_current = 1 + """, + (host_id, claim["id"]), + ) + generation_before = conn.execute( + "SELECT traversal_generation FROM turn_list_hosts WHERE host_id = ?", + (host_id,), + ).fetchone()[0] + conn.execute("PRAGMA user_version = 14") + + init_store(db_path) + + with sqlite3.connect(str(db_path)) as conn: + claim_payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + generation_after = conn.execute( + "SELECT traversal_generation FROM turn_list_hosts WHERE host_id = ?", + (host_id,), + ).fetchone()[0] + physical_rows = conn.execute( + "SELECT COUNT(*) FROM turns WHERE host_id = ? AND turn_id IN (?, ?)", + (host_id, observed["id"], claim["id"]), + ).fetchone()[0] + foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall() + public = turns_payload_from_store(db_path, host_id, schema_version=2)["turns"] + assert claim_payload["superseded_by_turn_id"] == observed["id"] + assert claim_payload["complete"] is True + assert generation_after == generation_before + 1 + assert physical_rows == 2 + assert public[0]["id"] == observed["id"] + assert claim["id"] not in {turn["id"] for turn in public} + assert foreign_keys == [] + + def _reset_store_to_v5_with_legacy_turn( db_path: Path, *, @@ -6785,7 +7160,7 @@ def test_v6_to_v7_repairs_mixed_turns_with_absent_content_descriptors( ) for row in first_rows ) - assert len(first_v2["turns"]) >= 3 + assert len(first_v2["turns"]) >= 2 for turn in first_v2["turns"]: assert turn["content"]["schema_version"] == 1 if turn["id"] not in missing_turn_ids: @@ -10221,7 +10596,7 @@ def test_v13_migration_repairs_nonpositive_turn_sequences_and_blocks_recurrence( init_store(db_path) with sqlite3.connect(str(db_path)) as conn: - assert _user_version(conn) == store_sqlite.STORE_SCHEMA_VERSION == 14 + assert _user_version(conn) == store_sqlite.STORE_SCHEMA_VERSION == 15 assert conn.execute( """ SELECT turn_id, list_sequence @@ -13289,7 +13664,12 @@ def test_same_source_completion_is_observation_monotonic_and_never_reopens( }, observed_at="2030-01-01T00:00:01+00:00", ) == 1 - payload = turns_payload_from_store(db_path, host_id, schema_version=2) + payload = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + claim_hard_ttl_seconds=1_000_000_000, + ) source_turn = next(turn for turn in payload["turns"] if turn.get("source_turn_id")) assert source_turn["assistant_final_text"] == "revised final" assert source_turn["assistant_stream_text"] is None @@ -14272,8 +14652,9 @@ def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( source_turn_id=raw_matching_source, meta=worker.meta, ) - assert exact_source["id"] == source_with_origin.id == source_without_origin.id - assert exact_source["id"] != command["id"] + assert exact_source["id"] == command["id"] + assert exact_source["id"] != source_with_origin.id + assert source_with_origin.id == source_without_origin.id assert exact_source["origin_command_id"] == "request-exact" assert exact_source["source_turn_id"] == source_with_origin.source_turn_id assert raw_matching_source not in json.dumps(payload, sort_keys=True) @@ -14301,18 +14682,27 @@ def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( }, observed_at="2026-07-13T02:00:04+00:00", ) == 1 - refreshed = turns_payload_from_store(db_path, host_id, schema_version=2) + refreshed = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + claim_hard_ttl_seconds=1_000_000_000, + ) unmatched_source = next( turn for turn in refreshed["turns"] if turn.get("assistant_final_text") == "independent final" ) assert unmatched_source.get("origin_command_id") is None assert unmatched_source["id"] != stale_command["id"] - assert any( - turn.get("id") == stale_command["id"] - and turn.get("origin_command_id") == "request-stale" - for turn in refreshed["turns"] - ) + assert stale_command["id"] not in {turn["id"] for turn in refreshed["turns"]} + with sqlite3.connect(str(db_path)) as conn: + stale_payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, stale_command["id"]), + ).fetchone()[0] + ) + assert stale_payload["superseded_by_turn_id"] == unmatched_source["id"] encoded = json.dumps(refreshed, sort_keys=True) assert raw_matching_source not in encoded assert raw_unmatched_source not in encoded diff --git a/tests/test_turn_ingestion.py b/tests/test_turn_ingestion.py index 4cc85f4..8c40b61 100644 --- a/tests/test_turn_ingestion.py +++ b/tests/test_turn_ingestion.py @@ -394,6 +394,36 @@ def reader(_config, _binding, *, adapter_timeout_seconds): scheduler.stop() +def test_scheduler_scan_sweeps_turn_claims_with_configured_ttls( + tmp_path: Path, + monkeypatch, +) -> None: + config, _snapshot, _bindings = _scheduler_store(tmp_path, 1) + calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] + + monkeypatch.setattr( + herdr_turns, + "prune_backend_pending", + lambda *_args, **_kwargs: 0, + ) + + def observed_sweep(*args: Any, **kwargs: Any) -> int: + calls.append((args, kwargs)) + return 0 + + monkeypatch.setattr(herdr_turns, "sweep_turn_claims", observed_sweep) + scheduler = TurnIngestionScheduler(config, refresh_interval_seconds=7.5) + + scheduler._scan_bindings() + + assert len(calls) == 1 + args, kwargs = calls[0] + assert args == (config.db_path, config.host_id) + assert kwargs["grace_seconds"] == 15.0 + assert kwargs["hard_ttl_seconds"] == config.turn_claim_hard_ttl_seconds + assert isinstance(kwargs["now"], str) + + def test_transient_initial_binding_scan_retries_once_and_dispatches( tmp_path: Path, monkeypatch, From c02079fc3a7843f790b1cd3df6b077731b68cf28 Mon Sep 17 00:00:00 2001 From: Jerry the Martian <38183696+jerryfane@users.noreply.github.com> Date: Sun, 19 Jul 2026 08:52:47 +0200 Subject: [PATCH 3/7] Gitmoot implement adhoc-01dabb6b (#4) --- src/tendwire/backends/herdr_turns.py | 6 +- src/tendwire/store/sqlite.py | 259 +++++++++++- tests/test_command_submission.py | 76 +++- tests/test_store.py | 612 ++++++++++++++++++++++++++- tests/test_turn_ingestion.py | 2 +- 5 files changed, 924 insertions(+), 31 deletions(-) diff --git a/src/tendwire/backends/herdr_turns.py b/src/tendwire/backends/herdr_turns.py index 1346527..128870a 100644 --- a/src/tendwire/backends/herdr_turns.py +++ b/src/tendwire/backends/herdr_turns.py @@ -36,6 +36,7 @@ redact_private_prompt_text, ) from ..store.sqlite import ( + TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS, apply_turn_refresh, list_worker_bindings, prune_backend_pending, @@ -4403,7 +4404,10 @@ def _scan_bindings(self) -> None: sweep_turn_claims( self.config.db_path, self.config.host_id, - grace_seconds=2.0 * self.refresh_interval_seconds, + grace_seconds=max( + TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS, + 10.0 * self.refresh_interval_seconds, + ), hard_ttl_seconds=self.config.turn_claim_hard_ttl_seconds, now=self._utc_clock(), ) diff --git a/src/tendwire/store/sqlite.py b/src/tendwire/store/sqlite.py index 37fc3ac..1fe6266 100644 --- a/src/tendwire/store/sqlite.py +++ b/src/tendwire/store/sqlite.py @@ -9,6 +9,7 @@ import fcntl import hashlib import json +import logging import math import os import secrets @@ -25,7 +26,10 @@ from typing import Any, Literal from urllib.parse import parse_qsl, quote, urlsplit -from ..config import DEFAULT_PENDING_STALE_GRACE_SECONDS +from ..config import ( + DEFAULT_PENDING_STALE_GRACE_SECONDS, + DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS, +) from ..local_state import ( EntryIdentity, EntryType, @@ -108,7 +112,9 @@ FINGERPRINT_HEX_LENGTH = FINGERPRINT_HEX_CHARS STORE_SCHEMA_VERSION = 15 -TURN_CLAIM_HARD_TTL_SECONDS = 86_400 +TURN_CLAIM_HARD_TTL_SECONDS = DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS +TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS = 60.0 +TURN_SUBMISSION_OBSERVATION_ADOPTION_WINDOW_SECONDS = 60.0 ACKNOWLEDGED_FINAL_RETENTION_DAYS = 30 ACKNOWLEDGED_FINAL_RETENTION_COUNT = 4096 COMMAND_RETRY_HORIZON_SECONDS = 604_800 @@ -138,6 +144,9 @@ ATTENTION_MISSING_REQUIRED = 2 ATTENTION_MISSING_GRACE_SECONDS = 120 _ATTENTION_SEVERITY_RANK = {"info": 0, "warning": 1, "critical": 2} +_LOGGER = logging.getLogger(__name__) +_TURN_CLAIM_SWEEP_LAST_AT: dict[tuple[str, str], float] = {} +_TURN_CLAIM_SWEEP_LOCK = threading.Lock() class StoreSchemaError(RuntimeError): @@ -148,6 +157,19 @@ def __init__(self, status: str) -> None: super().__init__(self.status) +def _configured_turn_claim_hard_ttl_seconds() -> int: + raw = os.environ.get("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS") + if raw is None: + return TURN_CLAIM_HARD_TTL_SECONDS + try: + configured = int(raw) + except (TypeError, ValueError) as exc: + raise StoreSchemaError("turn_claim_hard_ttl_invalid") from exc + if configured <= 0: + raise StoreSchemaError("turn_claim_hard_ttl_invalid") + return configured + + @dataclass(frozen=True) class Migration: """One exact, transaction-external schema transition.""" @@ -10876,6 +10898,7 @@ def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: row for row in decoded if str(row[3].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[3]) and ( row[3].get("complete") is True or row[4] is not None @@ -10883,6 +10906,12 @@ def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: ) ] affected_hosts: set[str] = set() + used_done = { + str(row[3].get("superseded_by_turn_id") or "") + for row in decoded + if _turn_is_tombstoned(row[3]) + and str(row[3].get("superseded_by_turn_id") or "").strip() + } for claim in claims: claim_view = _turn_with_current_content(claim[3], claim[4]) matches = [ @@ -10890,12 +10919,27 @@ def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: for observed in done if observed[0] == claim[0] and observed[2] == claim[2] + and observed[1] not in used_done and _turn_content_matches_origin( _turn_with_current_content(observed[3], observed[4]), claim_view, ) ] - if len(matches) == 1 and _tombstone_turn_conn( + if len(matches) != 1: + continue + matching_claims = [ + candidate + for candidate in claims + if candidate[0] == claim[0] + and candidate[2] == claim[2] + and _turn_content_matches_origin( + _turn_with_current_content(matches[0][3], matches[0][4]), + _turn_with_current_content(candidate[3], candidate[4]), + ) + ] + if len(matching_claims) != 1: + continue + if _tombstone_turn_conn( conn, claim[0], claim[1], @@ -10903,7 +10947,9 @@ def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: superseded_at=now, ): affected_hosts.add(claim[0]) + used_done.add(matches[0][1]) + configured_hard_ttl = _configured_turn_claim_hard_ttl_seconds() for claim in claims: stored = conn.execute( "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", @@ -10912,7 +10958,7 @@ def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: if stored is None or _turn_is_tombstoned(_json_object(stored[0])): continue claim_dt = _turn_row_time(claim[3], claim[5]) - if claim_dt is None or (now_dt - claim_dt).total_seconds() < TURN_CLAIM_HARD_TTL_SECONDS: + if claim_dt is None or (now_dt - claim_dt).total_seconds() < configured_hard_ttl: continue if _tombstone_turn_conn( conn, @@ -15557,7 +15603,8 @@ def _snapshot_owned_turn_candidate( source_rows = [ row for row in rows - if str(row[1].get("source_turn_id") or "").strip() + if not _turn_is_tombstoned(row[1]) + and str(row[1].get("source_turn_id") or "").strip() and str(row[1].get("origin_command_id") or "").strip() == origin_command_id ] @@ -15568,7 +15615,8 @@ def _snapshot_owned_turn_candidate( command_rows = [ row for row in rows - if not str(row[1].get("source_turn_id") or "").strip() + if not _turn_is_tombstoned(row[1]) + and not str(row[1].get("source_turn_id") or "").strip() and str(row[1].get("origin_command_id") or "").strip() == origin_command_id ] @@ -17801,7 +17849,11 @@ def _current_turn_content_rows_conn( """, (str(host_id), str(worker_id)), ).fetchall() - return _decode_turn_content_rows(rows) + return [ + row + for row in _decode_turn_content_rows(rows) + if not _turn_is_tombstoned(row[1]) + ] def _current_owned_turn_content_rows_conn( @@ -17834,7 +17886,8 @@ def _current_owned_turn_content_rows_conn( return [ row for row in _decode_turn_content_rows(rows) - if _turn_continuity_identity(row[1]) == owner_identity + if not _turn_is_tombstoned(row[1]) + and _turn_continuity_identity(row[1]) == owner_identity ] @@ -17898,6 +17951,57 @@ def _tombstone_turn_conn( return True +def _superseding_turn_content_conn( + conn: sqlite3.Connection, + host_id: str, + turn_id: str, +) -> dict[str, Any] | None: + """Follow a tombstone chain to the current list-visible successor.""" + current_turn_id = str(turn_id) + seen: set[str] = set() + first = True + while current_turn_id and current_turn_id not in seen: + seen.add(current_turn_id) + row = conn.execute( + """ + SELECT turns.payload_json, + revisions.user_text, + revisions.assistant_final_text, + revisions.user_state, + revisions.final_state + FROM turns + LEFT JOIN turn_content_revisions AS revisions + ON revisions.host_id = turns.host_id + AND revisions.turn_id = turns.turn_id + AND revisions.is_current = 1 + WHERE turns.host_id = ? AND turns.turn_id = ? + """, + (str(host_id), current_turn_id), + ).fetchone() + if row is None: + return None + payload = _json_object(row[0]) + if not _turn_is_tombstoned(payload): + if first: + return None + current = ( + { + "user_text": row[1], + "assistant_final_text": row[2], + "user_state": str(row[3]), + "final_state": str(row[4]), + } + if row[3] is not None + else None + ) + return _turn_with_current_content(payload, current) + first = False + current_turn_id = str( + payload.get("superseded_by_turn_id") or "" + ).strip() + return None + + def _tombstone_matching_command_sibling_conn( conn: sqlite3.Connection, host_id: str, @@ -17956,6 +18060,35 @@ def _turn_row_time(payload: Mapping[str, Any], observed_at: str) -> datetime | N return None +def _turn_is_open_or_incomplete( + payload: Mapping[str, Any], + current: Mapping[str, Any] | None, +) -> bool: + return bool( + payload.get("has_open_turn") is True + or payload.get("complete") is not True + or current is not None + and str(current.get("final_state") or "") != "complete" + ) + + +def _turn_observed_near_submission( + observed_at: str, + send_started_at: str, +) -> bool: + observed = _strict_utc_timestamp(observed_at) + started = _strict_utc_timestamp(send_started_at) + if observed is None or started is None: + return False + delta = abs( + ( + datetime.fromisoformat(started) + - datetime.fromisoformat(observed) + ).total_seconds() + ) + return delta <= TURN_SUBMISSION_OBSERVATION_ADOPTION_WINDOW_SECONDS + + def _sweep_turn_claims_conn( conn: sqlite3.Connection, host_id: str, @@ -18081,7 +18214,7 @@ def _sweep_turn_claims_conn( (row for row in unresolved if row[1] == worker_id), key=lambda row: (_turn_row_time(row[2], row[4]) or current_dt, row[0]), ) - if not worker_claims: + if len(worker_claims) != 1: continue claim = worker_claims[0] claim_dt = _turn_row_time(claim[2], claim[4]) @@ -18093,11 +18226,12 @@ def _sweep_turn_claims_conn( for row in done if row[1] == worker_id and row[0] not in used_done + and not str(row[2].get("origin_command_id") or "").strip() and (_turn_row_time(row[2], row[4]) or current_dt) > claim_dt ), key=lambda row: (_turn_row_time(row[2], row[4]) or current_dt, row[0]), ) - if newer_done and _tombstone_turn_conn( + if len(newer_done) == 1 and _tombstone_turn_conn( conn, str(host_id), claim[0], @@ -18161,6 +18295,36 @@ def sweep_turn_claims( raise +def _reserve_lazy_turn_claim_sweep( + db_path: Path | str, + host_id: str, + *, + current_clock: float, + refresh_interval_seconds: float, +) -> tuple[tuple[str, str], bool]: + key = (str(Path(db_path).absolute()), str(host_id)) + with _TURN_CLAIM_SWEEP_LOCK: + last = _TURN_CLAIM_SWEEP_LAST_AT.get(key) + if ( + last is not None + and current_clock >= last + and current_clock - last < refresh_interval_seconds + ): + return key, False + _TURN_CLAIM_SWEEP_LAST_AT[key] = current_clock + return key, True + + +def _release_failed_lazy_turn_claim_sweep( + key: tuple[str, str], + *, + current_clock: float, +) -> None: + with _TURN_CLAIM_SWEEP_LOCK: + if _TURN_CLAIM_SWEEP_LAST_AT.get(key) == current_clock: + _TURN_CLAIM_SWEEP_LAST_AT.pop(key, None) + + def _source_turn_matches(payload: Mapping[str, Any], incoming_source_turn: str) -> bool: stored = str(payload.get("source_turn_id") or "").strip() if not stored or not incoming_source_turn: @@ -18351,7 +18515,8 @@ def _owned_command_candidates( payload = row[1] row_origin = str(payload.get("origin_command_id") or "").strip() if ( - str(payload.get("source_turn_id") or "").strip() + _turn_is_tombstoned(payload) + or str(payload.get("source_turn_id") or "").strip() or not row_origin or expected_origin and row_origin != expected_origin or not _turn_content_matches_origin( @@ -18372,7 +18537,8 @@ def _owned_placeholder_candidates( return [ row for row in rows - if not str(row[1].get("source_turn_id") or "").strip() + if not _turn_is_tombstoned(row[1]) + and not str(row[1].get("source_turn_id") or "").strip() and not str(row[1].get("origin_command_id") or "").strip() ] @@ -18539,6 +18705,20 @@ def _merge_owned_turn_content_conn( command_predecessor_turn_id = predecessor_id else: command_rows = _owned_command_candidates(rows, automation_probe) + if len(command_rows) > 1: + _LOGGER.warning( + "turn_ingestion_ambiguity_fallthrough", + extra={ + "tendwire_diagnostic": { + "code": "turn_ingestion_ambiguity_fallthrough", + "host_id": str(host_id), + "worker_id": str( + current_projection.get("worker_id") or "" + ), + "candidate_count": len(command_rows), + } + }, + ) if len(command_rows) == 1: base_row = command_rows[0] else: @@ -19491,6 +19671,15 @@ def _upsert_command_pending_turn_impl( _ensure_schema(conn) conn.execute("BEGIN IMMEDIATE") try: + superseding_turn = _superseding_turn_content_conn( + conn, + str(host_id), + turn_id, + ) + if superseding_turn is not None: + if owns_transaction: + conn.commit() + return sanitize_public_mapping(superseding_turn) owner_identity = _turn_continuity_identity(item) if owner_identity is not None: owned_rows = _current_owned_turn_content_rows_conn( @@ -19554,6 +19743,13 @@ def _upsert_command_pending_turn_impl( row[1].get("origin_command_id") or "" ).strip() and not _turn_is_tombstoned(row[1]) + and ( + _turn_is_open_or_incomplete(row[1], row[2]) + or _turn_observed_near_submission( + row[3], + current_time, + ) + ) and _turn_content_matches_origin( _turn_with_current_content(row[1], row[2]), {"user_text": clean_text}, @@ -19909,17 +20105,36 @@ def turns_payload_from_store( "ok": False, "status": "store_unavailable", } - try: - sweep_turn_claims( - db_path, - str(host_id), - grace_seconds=2.0 * float(turn_refresh_interval_seconds), - hard_ttl_seconds=float(claim_hard_ttl_seconds), - ) - except Exception: - # Listing remains available if opportunistic maintenance is contended. - pass current_clock = time.time() if now is None else float(now) + refresh_interval = float(turn_refresh_interval_seconds) + sweep_key, sweep_due = _reserve_lazy_turn_claim_sweep( + db_path, + str(host_id), + current_clock=current_clock, + refresh_interval_seconds=refresh_interval, + ) + if sweep_due: + try: + sweep_turn_claims( + db_path, + str(host_id), + grace_seconds=max( + TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS, + 10.0 * refresh_interval, + ), + hard_ttl_seconds=float(claim_hard_ttl_seconds), + now=datetime.fromtimestamp( + current_clock, + tz=timezone.utc, + ).isoformat(), + ) + except Exception: + _release_failed_lazy_turn_claim_sweep( + sweep_key, + current_clock=current_clock, + ) + # Listing remains available if opportunistic maintenance is contended. + pass try: decoded_cursor = ( decode_turn_list_cursor( diff --git a/tests/test_command_submission.py b/tests/test_command_submission.py index 7cc2696..648604c 100644 --- a/tests/test_command_submission.py +++ b/tests/test_command_submission.py @@ -684,6 +684,7 @@ def test_observation_first_submission_adopts_source_row_in_place( }, ) _seed(config, [worker], [_binding(worker)]) + observed_at = store_sqlite.utc_timestamp() assert merge_turn_content( config.db_path, config.host_id, @@ -695,7 +696,7 @@ def test_observation_first_submission_adopts_source_row_in_place( "complete": True, "has_open_turn": False, }, - observed_at="2099-07-19T10:00:00+00:00", + observed_at=observed_at, ) == 1 observed = next( turn @@ -706,6 +707,11 @@ def test_observation_first_submission_adopts_source_row_in_place( )["turns"] if turn.get("assistant_final_text") == "already observed" ) + with sqlite3.connect(str(config.db_path)) as conn: + observed_list_sequence = conn.execute( + "SELECT list_sequence FROM turns WHERE host_id = ? AND turn_id = ?", + (config.host_id, observed["id"]), + ).fetchone()[0] calls: list[dict[str, Any]] = [] accepted = submit_command( @@ -729,7 +735,7 @@ def test_observation_first_submission_adopts_source_row_in_place( "complete": True, "has_open_turn": False, }, - observed_at="2099-07-19T10:00:02+00:00", + observed_at=store_sqlite.utc_timestamp(), ) == 1 with sqlite3.connect(str(config.db_path)) as conn: matching = conn.execute( @@ -746,6 +752,7 @@ def test_observation_first_submission_adopts_source_row_in_place( ).fetchall() assert len(matching) == 1 assert matching[0][0] == observed["id"] + assert matching[0][1] == observed_list_sequence refreshed = turns_payload_from_store( config.db_path, config.host_id, @@ -812,6 +819,71 @@ def request(self, method, params, *, timeout=None): assert matching[0]["assistant_final_text"] == "observed during send" +def test_legacy_submission_envelope_follows_tombstoned_claim_successor( + tmp_path: Path, +) -> None: + config = _config(tmp_path) + assert config.db_path is not None + worker = Worker(id="w-1", name="Alpha", status="active") + _seed(config, [worker], [_binding(worker)]) + calls: list[dict[str, Any]] = [] + + class LegacyObservingClient(_FakeSocketClient): + def request(self, method, params, *, timeout=None): + result = super().request(method, params, timeout=timeout) + if method == "pane.send_input": + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "legacy-envelope-source", + "user_text": "hello", + "assistant_final_text": "legacy observed during send", + "complete": True, + "has_open_turn": False, + }, + observed_at=store_sqlite.utc_timestamp(), + ) == 1 + return result + + accepted = submit_command( + config, + _request(request_id="legacy-envelope-request"), + socket_client_factory=lambda _config: LegacyObservingClient(calls), + ) + + assert accepted.status == STATUS_ACCEPTED + assert accepted.result["observed_turn_state"] == "complete" + turns = turns_payload_from_store( + config.db_path, + config.host_id, + schema_version=2, + )["turns"] + successor = next( + turn + for turn in turns + if turn.get("assistant_final_text") == "legacy observed during send" + ) + assert accepted.result["turn_id"] == successor["id"] + with sqlite3.connect(str(config.db_path)) as conn: + tombstones = [ + json.loads(row[0]) + for row in conn.execute( + """ + SELECT payload_json + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.origin_command_id') = ? + AND COALESCE(json_extract(payload_json, '$.superseded_at'), '') != '' + """, + (config.host_id, "legacy-envelope-request"), + ).fetchall() + ] + assert len(tombstones) == 1 + assert tombstones[0]["superseded_by_turn_id"] == successor["id"] + + def test_submit_command_sends_identical_100_character_instructions_with_distinct_ids( tmp_path: Path, ) -> None: diff --git a/tests/test_store.py b/tests/test_store.py index 364c8ee..e0f1123 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -12,7 +12,7 @@ import stat import threading import time -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -6572,6 +6572,237 @@ def test_turn_claim_sweeper_resolves_only_one_identical_claim_per_done_turn( assert stored[second["id"]]["has_open_turn"] is True +@pytest.mark.parametrize(("claim_count", "done_count"), [(1, 2), (2, 1)]) +def test_turn_claim_sweeper_fifo_requires_unambiguous_claim_and_done_sets( + tmp_path: Path, + claim_count: int, + done_count: int, +) -> None: + db_path = tmp_path / f"turn-fifo-{claim_count}-{done_count}.db" + host_id = "turn-fifo-unambiguous-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("7" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claims = [ + store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id=f"fifo-claim-{index}", + instruction_text=f"claim prompt {index}", + observed_at=f"2026-07-19T00:00:0{index}+00:00", + ) + for index in range(claim_count) + ] + assert all(claim is not None for claim in claims) + for index in range(done_count): + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": f"fifo-done-{index}", + "user_text": f"unrelated done {index}", + "assistant_final_text": f"answer {index}", + "complete": True, + "has_open_turn": False, + }, + observed_at=f"2026-07-19T00:01:0{index}+00:00", + ) == 1 + + assert store_sqlite.sweep_turn_claims( + db_path, + host_id, + grace_seconds=1, + hard_ttl_seconds=1_000_000, + now="2026-07-19T00:02:00+00:00", + ) == 0 + with sqlite3.connect(str(db_path)) as conn: + stored = [ + json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + for claim in claims + if claim is not None + ] + assert all(payload.get("superseded_at") is None for payload in stored) + + +def test_turn_list_lazy_sweep_uses_the_callers_now(tmp_path: Path) -> None: + db_path = tmp_path / "turn-list-caller-now.db" + host_id = "turn-list-caller-now-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[{"id": "worker-1", "name": "Worker", "status": "active"}], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="caller-now-claim", + instruction_text="caller clock", + observed_at="2000-01-01T00:00:00+00:00", + ) + assert claim is not None + + before_ttl = turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2000-01-01T00:00:30+00:00").timestamp(), + claim_hard_ttl_seconds=60, + ) + assert claim["id"] in {turn["id"] for turn in before_ttl["turns"]} + after_ttl = turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2000-01-01T00:02:00+00:00").timestamp(), + claim_hard_ttl_seconds=60, + ) + assert claim["id"] not in {turn["id"] for turn in after_ttl["turns"]} + + +def test_late_real_completion_never_adopts_tombstoned_claim(tmp_path: Path) -> None: + db_path = tmp_path / "turn-late-real-completion.db" + host_id = "turn-late-real-completion-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("8" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="late-real-claim", + instruction_text="late real prompt", + observed_at="2026-07-19T00:00:00+00:00", + ) + assert claim is not None + assert store_sqlite.sweep_turn_claims( + db_path, + host_id, + grace_seconds=1, + hard_ttl_seconds=60, + now="2026-07-19T00:02:00+00:00", + ) == 1 + + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "late-real-source", + "user_text": "late real prompt", + "assistant_final_text": "late real answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:02:01+00:00", + ) == 1 + listed = turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2026-07-19T00:02:02+00:00").timestamp(), + )["turns"] + real = next(turn for turn in listed if turn.get("assistant_final_text") == "late real answer") + assert real["id"] != claim["id"] + with sqlite3.connect(str(db_path)) as conn: + claim_payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + assert claim_payload.get("source_turn_id") is None + assert claim_payload["superseded_at"] == "2026-07-19T00:02:00+00:00" + + +def test_submission_adoption_rejects_old_completed_matching_text(tmp_path: Path) -> None: + db_path = tmp_path / "turn-old-observation-adoption.db" + host_id = "turn-old-observation-adoption-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("9" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "old-yes-source", + "user_text": "yes", + "assistant_final_text": "old answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:00:00+00:00", + ) == 1 + old = next( + turn + for turn in turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2026-07-19T00:00:01+00:00").timestamp(), + )["turns"] + if turn.get("assistant_final_text") == "old answer" + ) + + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="new-yes-request", + instruction_text="yes", + observed_at="2026-07-19T00:05:00+00:00", + ) + assert claim is not None + assert claim["id"] != old["id"] + assert claim["complete"] is False + assert claim.get("assistant_final_text") in {None, ""} + assert old.get("origin_command_id") is None + + def test_text_drift_tombstone_preserves_cursor_and_since_continuity( tmp_path: Path, ) -> None: @@ -6759,6 +6990,370 @@ def test_v14_migration_tombstones_production_shape_duplicate_claim( assert foreign_keys == [] +def test_v14_migration_skips_ambiguous_claim_pair(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "turn-v14-ambiguous-pair.db" + host_id = "turn-v14-ambiguous-pair-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("a" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "ambiguous-migration-source", + "user_text": "ambiguous migration prompt", + "assistant_final_text": "one observed answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:00:00+00:00", + ) == 1 + claims = [ + store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id=f"ambiguous-migration-{index}", + instruction_text=f"temporary migration prompt {index}", + observed_at=f"2026-07-19T00:02:0{index}+00:00", + ) + for index in range(2) + ] + assert all(claim is not None for claim in claims) + with sqlite3.connect(str(db_path)) as conn: + for claim in claims: + assert claim is not None + conn.execute( + """ + UPDATE turn_content_revisions + SET user_text = 'ambiguous migration prompt' + WHERE host_id = ? AND turn_id = ? AND is_current = 1 + """, + (host_id, claim["id"]), + ) + conn.execute("PRAGMA user_version = 14") + monkeypatch.setenv("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "999999999") + + init_store(db_path) + + with sqlite3.connect(str(db_path)) as conn: + payloads = [ + json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + for claim in claims + if claim is not None + ] + assert len(payloads) == 2 + assert all(payload.get("superseded_at") is None for payload in payloads) + + +def test_v14_migration_does_not_reuse_already_claimed_done_row( + tmp_path: Path, + monkeypatch, +) -> None: + db_path = tmp_path / "turn-v14-used-done.db" + host_id = "turn-v14-used-done-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("b" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "used-done-source", + "user_text": "used done prompt", + "assistant_final_text": "used done answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:00:00+00:00", + ) == 1 + done = next( + turn + for turn in turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2026-07-19T00:00:01+00:00").timestamp(), + )["turns"] + if turn.get("assistant_final_text") == "used done answer" + ) + claims = [ + store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id=f"used-done-claim-{index}", + instruction_text=f"temporary used prompt {index}", + observed_at=f"2026-07-19T00:02:0{index}+00:00", + ) + for index in range(2) + ] + assert all(claim is not None for claim in claims) + with sqlite3.connect(str(db_path)) as conn: + for claim in claims: + assert claim is not None + conn.execute( + """ + UPDATE turn_content_revisions + SET user_text = 'used done prompt' + WHERE host_id = ? AND turn_id = ? AND is_current = 1 + """, + (host_id, claim["id"]), + ) + assert store_sqlite._tombstone_turn_conn( + conn, + host_id, + claims[0]["id"], + superseded_by_turn_id=done["id"], + superseded_at="2026-07-19T00:03:00+00:00", + ) + conn.execute("PRAGMA user_version = 14") + monkeypatch.setenv("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "999999999") + + init_store(db_path) + + with sqlite3.connect(str(db_path)) as conn: + second_payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claims[1]["id"]), + ).fetchone()[0] + ) + assert second_payload.get("superseded_at") is None + assert second_payload["has_open_turn"] is True + + +def test_v14_migration_uses_configured_claim_hard_ttl( + tmp_path: Path, + monkeypatch, +) -> None: + db_path = tmp_path / "turn-v14-configured-ttl.db" + host_id = "turn-v14-configured-ttl-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[{"id": "worker-1", "name": "Worker", "status": "active"}], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="configured-ttl-claim", + instruction_text="expire from configured ttl", + observed_at=(datetime.now(timezone.utc) - timedelta(seconds=120)).isoformat(), + ) + assert claim is not None + with sqlite3.connect(str(db_path)) as conn: + conn.execute("PRAGMA user_version = 14") + monkeypatch.setenv("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "60") + + init_store(db_path) + + with sqlite3.connect(str(db_path)) as conn: + payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + assert payload.get("superseded_at") is not None + assert payload.get("superseded_by_turn_id") is None + + +def test_turn_cursor_continues_across_interior_tombstone(tmp_path: Path) -> None: + db_path = tmp_path / "turn-interior-tombstone-cursor.db" + host_id = "turn-interior-tombstone-cursor-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[{"id": "worker-1", "name": "Worker", "status": "active"}], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + for index in range(3): + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": f"cursor-source-{index}", + "user_text": f"cursor prompt {index}", + "assistant_final_text": f"cursor answer {index}", + "complete": True, + "has_open_turn": False, + }, + observed_at=f"2026-07-19T00:0{index}:00+00:00", + ) == 1 + with sqlite3.connect(str(db_path)) as conn: + visible = conn.execute( + """ + SELECT turn_id + FROM turns + WHERE host_id = ? + AND COALESCE(json_extract(payload_json, '$.source_turn_id'), '') != '' + ORDER BY worker_id ASC, list_sequence DESC, turn_id ASC + """, + (host_id,), + ).fetchall() + ordered_ids = [str(row[0]) for row in visible] + assert len(ordered_ids) == 3 + first = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + limit=1, + now=datetime.fromisoformat("2026-07-19T00:03:00+00:00").timestamp(), + ) + assert first["turns"][0]["id"] == ordered_ids[0] + assert first["next_cursor"] is not None + with sqlite3.connect(str(db_path)) as conn: + assert store_sqlite._tombstone_turn_conn( + conn, + host_id, + ordered_ids[1], + superseded_by_turn_id=ordered_ids[0], + superseded_at="2026-07-19T00:03:01+00:00", + ) + + continuation = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + limit=1, + cursor=first["next_cursor"], + now=datetime.fromisoformat("2026-07-19T00:03:02+00:00").timestamp(), + ) + assert continuation.get("status") != "cursor_expired" + assert [turn["id"] for turn in continuation["turns"]] == [ordered_ids[2]] + + +def test_turn_list_lazy_sweep_is_rate_limited( + tmp_path: Path, + monkeypatch, +) -> None: + db_path = tmp_path / "turn-list-sweep-rate-limit.db" + host_id = "turn-list-sweep-rate-limit-host" + init_store(db_path) + calls: list[tuple[float | None, str | None]] = [] + + def capture_sweep(*_args, grace_seconds=None, now=None, **_kwargs): + calls.append((grace_seconds, now)) + return 0 + + monkeypatch.setattr(store_sqlite, "sweep_turn_claims", capture_sweep) + for current in (1_000.0, 1_001.0, 1_002.0): + turns_payload_from_store( + db_path, + host_id, + now=current, + turn_refresh_interval_seconds=2.0, + ) + assert calls == [ + ( + 60.0, + datetime.fromtimestamp(1_000.0, tz=timezone.utc).isoformat(), + ), + ( + 60.0, + datetime.fromtimestamp(1_002.0, tz=timezone.utc).isoformat(), + ), + ] + + +def test_ingestion_ambiguity_fallthrough_emits_structured_diagnostic( + tmp_path: Path, + caplog, +) -> None: + db_path = tmp_path / "turn-ingestion-ambiguity-diagnostic.db" + host_id = "turn-ingestion-ambiguity-diagnostic-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("c" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + for index in range(2): + assert store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id=f"diagnostic-claim-{index}", + instruction_text="same ambiguous prompt", + observed_at=f"2026-07-19T00:00:0{index}+00:00", + ) is not None + + with caplog.at_level("WARNING", logger="tendwire.store.sqlite"): + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "diagnostic-source", + "user_text": "same ambiguous prompt", + "assistant_final_text": "independent answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:01:00+00:00", + ) == 1 + + diagnostics = [ + getattr(record, "tendwire_diagnostic", None) + for record in caplog.records + if record.getMessage() == "turn_ingestion_ambiguity_fallthrough" + ] + assert diagnostics == [ + { + "code": "turn_ingestion_ambiguity_fallthrough", + "host_id": host_id, + "worker_id": "worker-1", + "candidate_count": 2, + } + ] + + def _reset_store_to_v5_with_legacy_turn( db_path: Path, *, @@ -14579,7 +15174,7 @@ def test_stable_owner_snapshot_ambiguity_rolls_back_without_allocating_sequence( assert foreign_keys == [] -def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( +def test_stable_owner_exact_content_adoption_does_not_misattribute_unmatched_turn( tmp_path: Path, ) -> None: db_path = tmp_path / "stable-owner-command-provenance.db" @@ -14624,7 +15219,12 @@ def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( }, observed_at="2026-07-13T02:00:02+00:00", ) == 1 - payload = turns_payload_from_store(db_path, host_id, schema_version=2) + payload = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + now=datetime.fromisoformat("2026-07-13T02:00:02+00:00").timestamp(), + ) exact_source = next( turn for turn in payload["turns"] if turn.get("assistant_final_text") == "exact final" @@ -14687,6 +15287,7 @@ def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( host_id, schema_version=2, claim_hard_ttl_seconds=1_000_000_000, + now=datetime.fromisoformat("2026-07-13T02:00:05+00:00").timestamp(), ) unmatched_source = next( turn for turn in refreshed["turns"] @@ -14694,7 +15295,7 @@ def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( ) assert unmatched_source.get("origin_command_id") is None assert unmatched_source["id"] != stale_command["id"] - assert stale_command["id"] not in {turn["id"] for turn in refreshed["turns"]} + assert stale_command["id"] in {turn["id"] for turn in refreshed["turns"]} with sqlite3.connect(str(db_path)) as conn: stale_payload = json.loads( conn.execute( @@ -14702,7 +15303,8 @@ def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( (host_id, stale_command["id"]), ).fetchone()[0] ) - assert stale_payload["superseded_by_turn_id"] == unmatched_source["id"] + assert stale_payload.get("superseded_at") is None + assert stale_payload.get("superseded_by_turn_id") is None encoded = json.dumps(refreshed, sort_keys=True) assert raw_matching_source not in encoded assert raw_unmatched_source not in encoded diff --git a/tests/test_turn_ingestion.py b/tests/test_turn_ingestion.py index 8c40b61..dae09a2 100644 --- a/tests/test_turn_ingestion.py +++ b/tests/test_turn_ingestion.py @@ -419,7 +419,7 @@ def observed_sweep(*args: Any, **kwargs: Any) -> int: assert len(calls) == 1 args, kwargs = calls[0] assert args == (config.db_path, config.host_id) - assert kwargs["grace_seconds"] == 15.0 + assert kwargs["grace_seconds"] == 75.0 assert kwargs["hard_ttl_seconds"] == config.turn_claim_hard_ttl_seconds assert isinstance(kwargs["now"], str) From b6f86215213a2a646f6f9b3952aecf66149c1c77 Mon Sep 17 00:00:00 2001 From: Jerry the Martian <38183696+jerryfane@users.noreply.github.com> Date: Sun, 19 Jul 2026 10:51:56 +0200 Subject: [PATCH 4/7] Gitmoot merge adhoc-d9548fe4 Merged by Gitmoot after policy gate passed. --- .env.example | 2 + INSTALL.md | 9 +- README.md | 39 +- src/tendwire/cli.py | 15 +- src/tendwire/config.py | 34 ++ src/tendwire/connectors/outbox.py | 64 +- src/tendwire/core/models.py | 2 + src/tendwire/daemon.py | 25 + src/tendwire/daemon_api.py | 22 + src/tendwire/store/sqlite.py | 639 +++++++++++++++++++- tests/test_backend_pending.py | 2 +- tests/test_config.py | 18 + tests/test_connector_daemon_cli.py | 141 +++++ tests/test_connector_outbox.py | 536 +++++++++++++++- tests/test_daemon.py | 4 + tests/test_delivery_retention_migration.py | 2 +- tests/test_delivery_retention_projection.py | 2 +- tests/test_delivery_retention_recovery.py | 2 +- tests/test_store.py | 2 +- 19 files changed, 1526 insertions(+), 34 deletions(-) diff --git a/.env.example b/.env.example index 4ed8c4d..c192f93 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,8 @@ TENDWIRE_ACKNOWLEDGED_FINAL_RETENTION_COUNT=4096 # malformed-owner, known-incomplete, and automation holds are inspection-only. TENDWIRE_MAX_OUTBOX_ATTEMPTS=10 TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS=60 +TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS=300 +TENDWIRE_CONNECTOR_ACK_TTL_SECONDS=300 # Automatic snapshot cleanup defaults to 14 days and 4096 rows per host. # Each host's latest row is always retained. At a 5-minute snapshot interval, diff --git a/INSTALL.md b/INSTALL.md index d7a7443..1d213ae 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -631,10 +631,11 @@ following together: - `turn.list` schema v2 with descriptor schema v1, 1,000-character previews and insertion-stable paging; schema-v1 `turn.content.get` with a 49,152-byte UTF-8 page ceiling; -- range-only schema-v1 `connector.prepare` begin/part/commit/recover, root-wide - leases, independent part ACKs, and immutable schema-v2 source-less route - lineage. Cleanup retains bounded delivered tombstones so repeated snapshots - cannot recreate/repost removed acknowledged roots; and +- range-only schema-v1 `connector.prepare` begin/part/commit/recover, per-worker + ordered roots and parts, bounded renewable/releasable leases, deadline-based + awaiting-ACK recovery, independent part ACKs, and immutable schema-v2 + source-less route lineage. Cleanup retains bounded delivered tombstones so + repeated snapshots cannot recreate/repost removed acknowledged roots; and - fair dead-letter inspection, exact root/failed-plan retry including source-less recovery, retained ACK prefix, cumulative attempts, request-ID idempotency, immutable recovery audit, and no provider-perfect exactly-once diff --git a/README.md b/README.md index 55627a8..de3baf9 100644 --- a/README.md +++ b/README.md @@ -534,6 +534,8 @@ variables: | `max_workers` | `TENDWIRE_MAX_WORKERS` | `512` | integer >= 1 | | `max_outbox_attempts` | `TENDWIRE_MAX_OUTBOX_ATTEMPTS` | `10` | integer >= 1 | | `connector_claim_ttl_seconds` | `TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS` | `60` | integer >= 1 | +| `connector_max_claim_ttl_seconds` | `TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS` | `300` | integer >= 1; caps `turn-final` poll and renew requests | +| `connector_ack_ttl_seconds` | `TENDWIRE_CONNECTOR_ACK_TTL_SECONDS` | `300` | integer >= 1; grace period for committed plans awaiting ACK completion | | `command_retry_horizon_seconds` | `TENDWIRE_COMMAND_RETRY_HORIZON_SECONDS` | `604800` | positive integer no greater than 604800 | | `command_receipt_retention_seconds` | `TENDWIRE_COMMAND_RECEIPT_RETENTION_SECONDS` | `2592000` | integer at least 691200 and strictly greater than the retry horizon | | `command_receipt_retention_count` | `TENDWIRE_COMMAND_RECEIPT_RETENTION_COUNT` | `4096` | positive integer; newest bounded inactive receipts per host | @@ -1135,15 +1137,17 @@ See `INSTALL.md` for the required maintenance and rollback order. Tendwire exposes a Tendwire-only connector delivery boundary above the SQLite store. The public daemon methods are `connector.prepare`, `connector.poll`, -`connector.ack`, `connector.fail`, `connector.defer`, `connector.inspect`, -`connector.retry`, and the operational helper `connector.reclaim`. Matching -JSON-only CLI hooks include: +`connector.ack`, `connector.fail`, `connector.defer`, `connector.renew`, +`connector.release`, `connector.inspect`, `connector.retry`, and the operational +helper `connector.reclaim`. Matching JSON-only CLI hooks include: ```bash tendwire connector poll --name attention --limit 10 --lease-seconds 60 --db-path /path/to/tendwire.db tendwire connector ack --name attention --ref '' --response-json '{"delivered":true}' --db-path /path/to/tendwire.db tendwire connector fail --name attention --ref '' --reason temporary --delay-seconds 60 --db-path /path/to/tendwire.db tendwire connector defer --name attention --ref '' --reason scheduled --available-at 2026-01-01T00:10:00+00:00 --db-path /path/to/tendwire.db +tendwire connector renew --name turn-final --ref '' --lease-seconds 120 --db-path /path/to/tendwire.db +tendwire connector release --name turn-final --ref '' --db-path /path/to/tendwire.db tendwire connector inspect --name turn-final --status dead_letter --limit 100 --db-path /path/to/tendwire.db tendwire connector retry --name turn-final --final-identity 'twfinal1.' --db-path /path/to/tendwire.db ``` @@ -1248,20 +1252,35 @@ effect. `connector.poll` atomically leases due `connector_outbox` rows for one `name` and returns opaque per-attempt refs. A live lease prevents duplicate polling. Expired leases are reclaimed before polling and before ref-mutating operations; -`connector.reclaim` can also be called directly. `connector.ack` validates the -host, name, attempt, lease, and ref before marking the delivery and outbox item -delivered. `connector.fail` records sanitized failure data and schedules retry -availability. `connector.defer` records sanitized defer data and schedules future -availability without treating the item as delivered. Stale, expired, -wrong-host, wrong-name, and superseded refs fail closed with neutral errors. +the daemon also runs this reclaim pass on its periodic tick, and +`connector.reclaim` can be called directly. `connector.renew` extends a live +lease without creating a delivery attempt. `connector.release` records the live +attempt as released and makes the row immediately pollable again. `connector.ack` +validates the host, name, attempt, lease, and ref before marking the delivery and +outbox item delivered. `connector.fail` records sanitized failure data and +schedules retry availability. `connector.defer` records sanitized defer data and +schedules future availability without treating the item as delivered. Stale, +expired, wrong-host, wrong-name, and superseded refs fail closed with neutral +errors. When callers omit `lease_seconds`, the daemon and CLI use `connector_claim_ttl_seconds` from config, defaulting to 60 seconds; an explicit -public `lease_seconds` value still wins. `max_outbox_attempts` prevents +public `lease_seconds` value still wins. `turn-final` poll and renew requests are +capped by `connector_max_claim_ttl_seconds`, defaulting to 300 seconds. +`max_outbox_attempts` prevents unbounded retry loops. Once a failed job reaches the configured cap, Tendwire moves the outbox item to a neutral terminal `dead_letter` state and returns the public status `attempts_exhausted` without exposing private outbox or delivery state. +Final-root FIFO is partitioned by the turn's immutable stable worker key, with +the persisted worker ID as the enqueue-time fallback. A blocked worker therefore +cannot starve another worker, while roots and plan parts for the same worker stay +strictly ordered. `dead_letter`, `superseded`, and `delivered` roots are terminal +for this gate. A committed source in `awaiting_ack` no longer blocks by itself; +its plan jobs carry the ordering obligation. Commit stamps an ACK deadline, and +an expired incomplete plan is failed and its source requeued. Missing or +unrecoverable plan state terminates the source instead of leaving it pending. + ### Delivery-aware final roots and acknowledged history Persisting an authoritative, owner-authenticated complete final atomically diff --git a/src/tendwire/cli.py b/src/tendwire/cli.py index a07619a..67609d6 100644 --- a/src/tendwire/cli.py +++ b/src/tendwire/cli.py @@ -494,15 +494,18 @@ def add_common(action_parser: argparse.ArgumentParser) -> None: reclaim_parser = actions.add_parser("reclaim", help="Expire stale connector leases.") add_common(reclaim_parser) - for action in ("ack", "fail", "defer"): + for action in ("ack", "fail", "defer", "renew", "release"): action_parser = actions.add_parser(action, help=f"Apply connector.{action} to a live ref.") add_common(action_parser) action_parser.add_argument("--ref", required=True) - action_parser.add_argument("--response-json", dest="response_json", default=None) + if action in {"ack", "fail", "defer"}: + action_parser.add_argument("--response-json", dest="response_json", default=None) if action in {"fail", "defer"}: action_parser.add_argument("--reason", default="") action_parser.add_argument("--available-at", dest="available_at", default=None) action_parser.add_argument("--delay-seconds", dest="delay_seconds", type=int, default=None) + if action == "renew": + action_parser.add_argument("--lease-seconds", dest="lease_seconds", type=int, default=None) def _load_worker_bindings(config: Config) -> list[WorkerBinding]: @@ -1424,9 +1427,9 @@ def _connector_params_from_args(args: argparse.Namespace) -> dict[str, Any]: params["limit"] = args.limit if args.lease_seconds is not None: params["lease_seconds"] = args.lease_seconds - if args.connector_action in {"ack", "fail", "defer"}: + if args.connector_action in {"ack", "fail", "defer", "renew", "release"}: params["ref"] = args.ref - if args.response_json: + if getattr(args, "response_json", None): try: parsed = json.loads(args.response_json) except json.JSONDecodeError: @@ -1439,6 +1442,8 @@ def _connector_params_from_args(args: argparse.Namespace) -> dict[str, Any]: params["available_at"] = args.available_at if args.delay_seconds is not None: params["delay_seconds"] = args.delay_seconds + if args.connector_action == "renew" and args.lease_seconds is not None: + params["lease_seconds"] = args.lease_seconds return params @@ -1485,6 +1490,8 @@ def cmd_connector(config: Config, args: argparse.Namespace) -> int: config.db_path, config.host_id, default_lease_seconds=config.connector_claim_ttl_seconds, + max_lease_seconds=config.connector_max_claim_ttl_seconds, + ack_ttl_seconds=config.connector_ack_ttl_seconds, max_attempts=config.max_outbox_attempts, ).dispatch(method, params) print(_connector_payload_json(payload, indent=2)) diff --git a/src/tendwire/config.py b/src/tendwire/config.py index adc8f40..ccbfad9 100644 --- a/src/tendwire/config.py +++ b/src/tendwire/config.py @@ -25,6 +25,8 @@ DEFAULT_PENDING_STALE_GRACE_SECONDS = 30.0 DEFAULT_MAX_OUTBOX_ATTEMPTS = 10 DEFAULT_CONNECTOR_CLAIM_TTL_SECONDS = 60 +DEFAULT_CONNECTOR_MAX_CLAIM_TTL_SECONDS = 300 +DEFAULT_CONNECTOR_ACK_TTL_SECONDS = 300 DEFAULT_ACKNOWLEDGED_FINAL_RETENTION_DAYS = 30 DEFAULT_ACKNOWLEDGED_FINAL_RETENTION_COUNT = 4096 DEFAULT_COMMAND_RETRY_HORIZON_SECONDS = 604_800 @@ -64,6 +66,8 @@ class Config: pending_stale_grace_seconds: float = DEFAULT_PENDING_STALE_GRACE_SECONDS max_outbox_attempts: int = DEFAULT_MAX_OUTBOX_ATTEMPTS connector_claim_ttl_seconds: int = DEFAULT_CONNECTOR_CLAIM_TTL_SECONDS + connector_max_claim_ttl_seconds: int = DEFAULT_CONNECTOR_MAX_CLAIM_TTL_SECONDS + connector_ack_ttl_seconds: int = DEFAULT_CONNECTOR_ACK_TTL_SECONDS acknowledged_final_retention_days: int = DEFAULT_ACKNOWLEDGED_FINAL_RETENTION_DAYS acknowledged_final_retention_count: int = DEFAULT_ACKNOWLEDGED_FINAL_RETENTION_COUNT command_retry_horizon_seconds: int = DEFAULT_COMMAND_RETRY_HORIZON_SECONDS @@ -178,6 +182,24 @@ def __post_init__(self) -> None: minimum=1, ), ) + object.__setattr__( + self, + "connector_max_claim_ttl_seconds", + _positive_int( + self.connector_max_claim_ttl_seconds, + "connector_max_claim_ttl_seconds", + minimum=1, + ), + ) + object.__setattr__( + self, + "connector_ack_ttl_seconds", + _positive_int( + self.connector_ack_ttl_seconds, + "connector_ack_ttl_seconds", + minimum=1, + ), + ) object.__setattr__( self, "acknowledged_final_retention_days", @@ -368,6 +390,8 @@ def load_config( pending_stale_grace_seconds: float | str | None = None, max_outbox_attempts: int | str | None = None, connector_claim_ttl_seconds: int | str | None = None, + connector_max_claim_ttl_seconds: int | str | None = None, + connector_ack_ttl_seconds: int | str | None = None, acknowledged_final_retention_days: int | str | None = None, acknowledged_final_retention_count: int | str | None = None, command_retry_horizon_seconds: int | str | None = None, @@ -496,6 +520,16 @@ def load_config( "TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS", DEFAULT_CONNECTOR_CLAIM_TTL_SECONDS, ), + connector_max_claim_ttl_seconds=_resolve_value( + connector_max_claim_ttl_seconds, + "TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS", + DEFAULT_CONNECTOR_MAX_CLAIM_TTL_SECONDS, + ), + connector_ack_ttl_seconds=_resolve_value( + connector_ack_ttl_seconds, + "TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", + DEFAULT_CONNECTOR_ACK_TTL_SECONDS, + ), acknowledged_final_retention_days=_resolve_value( acknowledged_final_retention_days, "TENDWIRE_ACKNOWLEDGED_FINAL_RETENTION_DAYS", diff --git a/src/tendwire/connectors/outbox.py b/src/tendwire/connectors/outbox.py index b1583f6..43a87d2 100644 --- a/src/tendwire/connectors/outbox.py +++ b/src/tendwire/connectors/outbox.py @@ -22,6 +22,8 @@ prepare_connector_plan_recover, prepare_connector_plan_part, reclaim_expired_connector_leases, + release_connector_delivery, + renew_connector_delivery, retry_final_ready_delivery, ) @@ -198,7 +200,7 @@ def _request_id(value: Any) -> str: class ConnectorOutboxAPI: - """Public-neutral facade for connector.poll/ack/fail/defer.""" + """Public-neutral facade for connector delivery and lease operations.""" def __init__( self, @@ -206,11 +208,15 @@ def __init__( host_id: str, *, default_lease_seconds: int = 60, + max_lease_seconds: int = 300, + ack_ttl_seconds: int = 300, max_attempts: int = 10, ) -> None: self.db_path = Path(db_path) if db_path is not None else None self.host_id = str(host_id) self.default_lease_seconds = max(1, int(default_lease_seconds)) + self.max_lease_seconds = max(1, int(max_lease_seconds)) + self.ack_ttl_seconds = max(1, int(ack_ttl_seconds)) self.max_attempts = max(1, int(max_attempts)) def _require_store(self, name: str = "") -> dict[str, Any] | None: @@ -327,6 +333,7 @@ def prepare(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: name=name, plan_token=token, source_ref=source_ref, + ack_ttl_seconds=self.ack_ttl_seconds, ) if set(data) != { @@ -405,7 +412,11 @@ def poll(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: data.get("lease_seconds"), self.default_lease_seconds, minimum=1, - maximum=86400, + maximum=( + self.max_lease_seconds + if name == _PREPARE_NAME + else 86400 + ), ), max_attempts=self.max_attempts, ) @@ -487,6 +498,51 @@ def fail(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: def defer(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: return self._schedule("defer", params) + def renew(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: + data, name, live_ref = self._mutation_parts(params) + if not name: + return _error("invalid_params", host_id=self.host_id) + if live_ref is None: + return _error("invalid_ref", host_id=self.host_id, name=name) + unavailable = self._require_store(name) + if unavailable is not None: + return unavailable + assert self.db_path is not None + lease_seconds = _int( + data.get("lease_seconds"), + self.default_lease_seconds, + minimum=1, + maximum=( + self.max_lease_seconds + if name == _PREPARE_NAME + else 86400 + ), + ) + return renew_connector_delivery( + self.db_path, + host_id=self.host_id, + name=name, + ref=live_ref, + lease_seconds=lease_seconds, + ) + + def release(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: + _data, name, live_ref = self._mutation_parts(params) + if not name: + return _error("invalid_params", host_id=self.host_id) + if live_ref is None: + return _error("invalid_ref", host_id=self.host_id, name=name) + unavailable = self._require_store(name) + if unavailable is not None: + return unavailable + assert self.db_path is not None + return release_connector_delivery( + self.db_path, + host_id=self.host_id, + name=name, + ref=live_ref, + ) + def _schedule(self, action: str, params: Mapping[str, Any] | None) -> dict[str, Any]: data, name, live_ref = self._mutation_parts(params) if not name: @@ -600,6 +656,10 @@ def dispatch(self, method: str, params: Mapping[str, Any] | None = None) -> dict return self.fail(params) if method == "connector.defer": return self.defer(params) + if method == "connector.renew": + return self.renew(params) + if method == "connector.release": + return self.release(params) if method == "connector.reclaim": return self.reclaim(params) if method == "connector.inspect": diff --git a/src/tendwire/core/models.py b/src/tendwire/core/models.py index 33fdc05..3bb05b5 100644 --- a/src/tendwire/core/models.py +++ b/src/tendwire/core/models.py @@ -443,7 +443,9 @@ "id", "max_outbox_attempts", "origin_command_id", + "outbox_ack_ttl_seconds", "outbox_claim_ttl_seconds", + "outbox_max_claim_ttl_seconds", "output_excerpt_chars", "raw_status", "request_id", diff --git a/src/tendwire/daemon.py b/src/tendwire/daemon.py index 2a90dbe..05448fa 100644 --- a/src/tendwire/daemon.py +++ b/src/tendwire/daemon.py @@ -518,6 +518,7 @@ def start(self) -> None: ), ) self.hooks.init_store(Path(self.config.db_path)) + self._connector_periodic_tick() if self.config.herdr_backend == "socket": self._snapshot = self._start_socket_event_backend() else: @@ -553,6 +554,7 @@ def start(self) -> None: stop_event=self.stop_event, socket_group=self.config.socket_group, prepare_parent=self._prepare_socket_parent, + periodic_callback=self._connector_periodic_tick, ) self._server = server server.start() @@ -949,6 +951,10 @@ def get_health(self) -> dict[str, Any]: "max_workers": self.config.max_workers, "max_outbox_attempts": self.config.max_outbox_attempts, "outbox_claim_ttl_seconds": self.config.connector_claim_ttl_seconds, + "outbox_max_claim_ttl_seconds": ( + self.config.connector_max_claim_ttl_seconds + ), + "outbox_ack_ttl_seconds": self.config.connector_ack_ttl_seconds, "acknowledged_final_retention_days": ( self.config.acknowledged_final_retention_days ), @@ -1068,9 +1074,28 @@ def connector_call(self, method: str, params: Mapping[str, Any]) -> Mapping[str, Path(self.config.db_path), self.config.host_id, default_lease_seconds=self.config.connector_claim_ttl_seconds, + max_lease_seconds=self.config.connector_max_claim_ttl_seconds, + ack_ttl_seconds=self.config.connector_ack_ttl_seconds, max_attempts=self.config.max_outbox_attempts, ).dispatch(method, params) + def _connector_periodic_tick(self) -> None: + """Eagerly reclaim expired connector work without waiting for a poll.""" + if self.config.db_path is None: + return + from .store.sqlite import reclaim_expired_connector_leases + + try: + reclaim_expired_connector_leases( + Path(self.config.db_path), + self.config.host_id, + None, + ) + except Exception: + # Startup and periodic maintenance remain best-effort; store health + # is reported through the normal daemon health surface. + return + def submit_command(self, params: Mapping[str, Any]) -> CommandEnvelope | Mapping[str, Any]: # Preserve the submitted keys exactly so the existing command parser can # reject private/connector fields instead of receiving sanitized input. diff --git a/src/tendwire/daemon_api.py b/src/tendwire/daemon_api.py index c553a4e..a8e1373 100644 --- a/src/tendwire/daemon_api.py +++ b/src/tendwire/daemon_api.py @@ -152,6 +152,8 @@ "connector.ack", "connector.fail", "connector.defer", + "connector.renew", + "connector.release", "connector.reclaim", "connector.retry", "connector.inspect", @@ -935,6 +937,8 @@ def __init__( request_workers: int = 8, max_in_flight_requests: int = 32, shutdown_grace_seconds: float = 6.0, + periodic_callback: Callable[[], Any] | None = None, + periodic_interval_seconds: float = 1.0, ) -> None: if ( not isinstance(request_workers, int) @@ -969,6 +973,12 @@ def __init__( self.request_workers = request_workers self.max_in_flight_requests = max_in_flight_requests self.shutdown_grace_seconds = float(shutdown_grace_seconds) + self.periodic_callback = periodic_callback + self.periodic_interval_seconds = max( + self.accept_timeout_seconds, + float(periodic_interval_seconds), + ) + self._next_periodic_at = time.monotonic() + self.periodic_interval_seconds self._listener: socket.socket | None = None self._identity: EntryIdentity | None = None self._pin_fd: int | None = None @@ -1142,6 +1152,7 @@ def serve_forever(self) -> None: self.start() try: while not self.stop_event.is_set(): + self._run_periodic_callback_if_due() listener = self._listener if listener is None: break @@ -1159,6 +1170,17 @@ def serve_forever(self) -> None: finally: self.close() + def _run_periodic_callback_if_due(self) -> None: + callback = self.periodic_callback + if callback is None or time.monotonic() < self._next_periodic_at: + return + self._next_periodic_at = time.monotonic() + self.periodic_interval_seconds + try: + callback() + except Exception: + # Periodic maintenance is best-effort and must not stop the API loop. + return + def _submit_connection(self, conn: socket.socket) -> None: if not self._admission.acquire(blocking=False): self._reject_connection(conn, "server_busy") diff --git a/src/tendwire/store/sqlite.py b/src/tendwire/store/sqlite.py index 1fe6266..1d71354 100644 --- a/src/tendwire/store/sqlite.py +++ b/src/tendwire/store/sqlite.py @@ -27,6 +27,7 @@ from urllib.parse import parse_qsl, quote, urlsplit from ..config import ( + DEFAULT_CONNECTOR_ACK_TTL_SECONDS, DEFAULT_PENDING_STALE_GRACE_SECONDS, DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS, ) @@ -111,7 +112,8 @@ FINGERPRINT_HEX_LENGTH = FINGERPRINT_HEX_CHARS -STORE_SCHEMA_VERSION = 15 +STORE_SCHEMA_VERSION = 16 +CONNECTOR_ACK_TTL_SECONDS = DEFAULT_CONNECTOR_ACK_TTL_SECONDS TURN_CLAIM_HARD_TTL_SECONDS = DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS = 60.0 TURN_SUBMISSION_OBSERVATION_ADOPTION_WINDOW_SECONDS = 60.0 @@ -170,6 +172,19 @@ def _configured_turn_claim_hard_ttl_seconds() -> int: return configured +def _configured_connector_ack_ttl_seconds() -> int: + raw = os.environ.get("TENDWIRE_CONNECTOR_ACK_TTL_SECONDS") + if raw is None: + return CONNECTOR_ACK_TTL_SECONDS + try: + configured = int(raw) + except (TypeError, ValueError) as exc: + raise StoreSchemaError("connector_ack_ttl_invalid") from exc + if configured <= 0: + raise StoreSchemaError("connector_ack_ttl_invalid") + return configured + + @dataclass(frozen=True) class Migration: """One exact, transaction-external schema transition.""" @@ -745,6 +760,11 @@ def _record_response_size( ), ) +CREATE_CONNECTOR_ORDERING_INDEX = ( + "CREATE INDEX IF NOT EXISTS idx_connector_outbox_final_ordering " + "ON connector_outbox(host_id, connector, ordering_key, delivery_kind, status, id)" +) + CREATE_TURN_PRESENTATION_PLANS_TABLE = """ CREATE TABLE IF NOT EXISTS turn_presentation_plans ( @@ -1000,6 +1020,7 @@ def _record_response_size( delivery_kind TEXT NOT NULL DEFAULT 'generic', turn_id TEXT, content_revision TEXT, + ordering_key TEXT NOT NULL DEFAULT '', status TEXT NOT NULL, payload_json TEXT NOT NULL, private_state_json TEXT NOT NULL DEFAULT '{}', @@ -1899,6 +1920,7 @@ def _connector_response( key: str | None = None, attempt: int | None = None, available_at: str | None = None, + leased_until: str | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = { "schema_version": 1, @@ -1915,6 +1937,8 @@ def _connector_response( payload["attempt"] = int(attempt) if available_at is not None: payload["available_at"] = str(available_at) + if leased_until is not None: + payload["leased_until"] = str(leased_until) return sanitize_public_value(payload) @@ -2019,6 +2043,201 @@ def _connector_reclaim_expired_leases_conn( now=now, ) reclaimed += 1 + reclaimed += _connector_reclaim_expired_awaiting_ack_conn( + conn, + host_id=str(host_id), + name=str(name) if name is not None else None, + now=str(now), + ) + return reclaimed + + +def _connector_reclaim_expired_awaiting_ack_conn( + conn: sqlite3.Connection, + *, + host_id: str, + name: str | None, + now: str, +) -> int: + clauses = ["outbox.host_id = ?", "outbox.status = 'awaiting_ack'"] + params: list[Any] = [str(host_id)] + if name is not None: + clauses.append("outbox.connector = ?") + params.append(str(name)) + rows = conn.execute( + f""" + SELECT outbox.id, outbox.connector, outbox.private_state_json + FROM connector_outbox AS outbox + WHERE {" AND ".join(clauses)} + ORDER BY outbox.id + """, + params, + ).fetchall() + reclaimed = 0 + now_dt = _connector_datetime(now) + for outbox_id, connector, private_state_json in rows: + outbox_state = _json_object(private_state_json) + deadline = str(outbox_state.get("ack_deadline_at") or "") + delivery = conn.execute( + """ + SELECT id, private_state_json + FROM connector_deliveries + WHERE outbox_id = ? AND status = 'awaiting_ack' + ORDER BY id DESC + LIMIT 1 + """, + (int(outbox_id),), + ).fetchone() + if not deadline and delivery is not None: + deadline = str( + _json_object(delivery[1]).get("ack_deadline_at") or "" + ) + if not deadline or _connector_datetime(deadline) > now_dt: + continue + + plan = conn.execute( + """ + SELECT id, state, generation + FROM turn_presentation_plans + WHERE source_outbox_id = ? + ORDER BY id DESC + LIMIT 1 + """, + (int(outbox_id),), + ).fetchone() + if plan is not None and str(plan[1]) == "completed": + if delivery is not None: + conn.execute( + """ + UPDATE connector_deliveries + SET status = 'delivered', delivered_at = COALESCE(delivered_at, ?) + WHERE id = ? AND status = 'awaiting_ack' + """, + (str(now), int(delivery[0])), + ) + conn.execute( + """ + UPDATE connector_outbox + SET status = 'delivered', next_attempt_at = NULL, + updated_at = ?, private_state_json = ? + WHERE id = ? AND status = 'awaiting_ack' + """, + ( + str(now), + _connector_private_clear_current(private_state_json), + int(outbox_id), + ), + ) + reclaimed += 1 + continue + + recoverable = plan is not None and str(plan[1]) in { + "active", + "waiting_predecessor", + "failed", + } + if recoverable: + plan_id = int(plan[0]) + conn.execute( + "UPDATE turn_presentation_plans SET state = 'failed' WHERE id = ?", + (plan_id,), + ) + job_rows = conn.execute( + """ + SELECT outbox.id, outbox.private_state_json + FROM turn_presentation_jobs AS jobs + JOIN connector_outbox AS outbox ON outbox.id = jobs.outbox_id + WHERE jobs.plan_id = ? + AND outbox.status NOT IN ('delivered', 'superseded', 'dead_letter') + """, + (plan_id,), + ).fetchall() + for job_outbox_id, job_private in job_rows: + conn.execute( + """ + UPDATE connector_deliveries + SET status = 'failed', response_json = ?, delivered_at = ? + WHERE outbox_id = ? AND status = 'leased' + """, + ( + _canonical_json( + {"schema_version": 1, "status": "ack_deadline_expired"} + ), + str(now), + int(job_outbox_id), + ), + ) + conn.execute( + """ + UPDATE connector_outbox + SET status = 'dead_letter', next_attempt_at = NULL, + updated_at = ?, private_state_json = ? + WHERE id = ? + """, + ( + str(now), + _connector_private_clear_current(job_private), + int(job_outbox_id), + ), + ) + source_state = _json_object( + _connector_private_clear_current(private_state_json) + ) + source_state["presentation_generation"] = max( + int(source_state.get("presentation_generation") or 1), + int(plan[2] or 1) + 1, + ) + source_status = "queued" + else: + source_state = _json_object( + _connector_private_clear_current(private_state_json) + ) + source_status = _CONNECTOR_EXHAUSTED_OUTBOX_STATUS + + if delivery is not None: + conn.execute( + """ + UPDATE connector_deliveries + SET status = 'failed', response_json = ?, delivered_at = ? + WHERE id = ? AND status = 'awaiting_ack' + """, + ( + _canonical_json( + { + "schema_version": 1, + "status": ( + "ack_deadline_expired" + if recoverable + else "plan_unrecoverable" + ), + } + ), + str(now), + int(delivery[0]), + ), + ) + conn.execute( + """ + UPDATE connector_outbox + SET status = ?, next_attempt_at = ?, updated_at = ?, + private_state_json = ? + WHERE id = ? AND status = 'awaiting_ack' + """, + ( + source_status, + str(now) if source_status == "queued" else None, + str(now), + _canonical_json(source_state), + int(outbox_id), + ), + ) + _activate_waiting_presentation_plans_conn( + conn, + host_id=str(host_id), + name=str(connector), + now=str(now), + ) + reclaimed += 1 return reclaimed @@ -2217,6 +2436,30 @@ def _valid_final_stable_key(value: Any) -> bool: ) +def _turn_ordering_key_conn( + conn: sqlite3.Connection, + *, + host_id: str, + turn_id: str, +) -> str: + row = conn.execute( + "SELECT worker_id, payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (str(host_id), str(turn_id)), + ).fetchone() + if row is None: + return "" + payload = _json_object(row[1]) + meta = _json_object(payload.get("meta")) + stable_key = meta.get("stable_key") + if ( + _valid_final_stable_key(stable_key) + and type(meta.get("stable_key_version")) is int + and meta.get("stable_key_version") == 1 + ): + return str(stable_key) + return str(row[0] or "") + + def _final_content_field_descriptor( *, revision: str, @@ -2883,6 +3126,11 @@ def _ensure_final_ready_anchor_conn( else "queued" ) payload_json = _canonical_json(payload) + ordering_key = _turn_ordering_key_conn( + conn, + host_id=str(host_id), + turn_id=str(turn_id), + ) if existing is not None: existing_state = _json_object(existing[2]) activated_after = ( @@ -2931,13 +3179,14 @@ def _ensure_final_ready_anchor_conn( delivery_kind, turn_id, content_revision, + ordering_key, status, payload_json, private_state_json, created_at, updated_at, next_attempt_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?, NULL) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?, NULL) ON CONFLICT(host_id, connector, delivery_key) DO NOTHING """, ( @@ -2947,6 +3196,7 @@ def _ensure_final_ready_anchor_conn( delivery_kind, str(turn_id), str(content_revision_value), + ordering_key, initial_status, payload_json, str(now), @@ -3668,14 +3918,25 @@ def _materialize_connector_plan_job_conn( ) -> int: plan_identity = conn.execute( """ - SELECT turn_id, content_revision - FROM turn_presentation_plans - WHERE id = ? + SELECT plans.turn_id, plans.content_revision, source.ordering_key + FROM turn_presentation_plans AS plans + LEFT JOIN connector_outbox AS source + ON source.id = plans.source_outbox_id + WHERE plans.id = ? """, (int(plan_id),), ).fetchone() if plan_identity is None: raise StoreSchemaError("presentation_plan_not_found") + ordering_key = ( + str(plan_identity[2]) + if plan_identity[2] is not None + else _turn_ordering_key_conn( + conn, + host_id=str(host_id), + turn_id=str(plan_identity[0]), + ) + ) cursor = conn.execute( """ INSERT INTO connector_outbox ( @@ -3685,13 +3946,14 @@ def _materialize_connector_plan_job_conn( delivery_kind, turn_id, content_revision, + ordering_key, status, payload_json, private_state_json, created_at, updated_at, next_attempt_at - ) VALUES (?, ?, ?, 'final_part', ?, ?, 'queued', ?, '{}', ?, ?, NULL) + ) VALUES (?, ?, ?, 'final_part', ?, ?, ?, 'queued', ?, '{}', ?, ?, NULL) """, ( str(host_id), @@ -3699,6 +3961,7 @@ def _materialize_connector_plan_job_conn( str(delivery_key), str(plan_identity[0]), str(plan_identity[1]), + ordering_key, _canonical_json(dict(payload)), str(created_at), str(created_at), @@ -4254,6 +4517,7 @@ def prepare_connector_plan_commit( name: str, plan_token: str, source_ref: str | None = None, + ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, now: str | None = None, ) -> dict[str, Any]: """Atomically validate exact coverage and materialize one plan's ordered jobs.""" @@ -4278,6 +4542,10 @@ def prepare_connector_plan_commit( name=name, ) current_time = _connector_now(now) + ack_deadline_at = _connector_add_seconds( + current_time, + max(1, int(ack_ttl_seconds)), + ) with _connect(db_path, isolation_level=None) as conn: _ensure_schema(conn) conn.execute("BEGIN IMMEDIATE") @@ -4626,11 +4894,18 @@ def prepare_connector_plan_commit( ).fetchone() if source_private is None: raise StoreSchemaError("presentation_source_not_live") + source_delivery_private = conn.execute( + "SELECT private_state_json FROM connector_deliveries WHERE id = ?", + (int(source_delivery[0]),), + ).fetchone() + if source_delivery_private is None: + raise StoreSchemaError("presentation_source_not_live") delivery_cursor = conn.execute( """ UPDATE connector_deliveries SET status = 'awaiting_ack', response_json = ?, + private_state_json = ?, delivered_at = NULL WHERE id = ? AND outbox_id = ? AND status = 'leased' """, @@ -4641,6 +4916,12 @@ def prepare_connector_plan_commit( "status": "prepared", } ), + _canonical_json( + { + **_json_object(source_delivery_private[0]), + "ack_deadline_at": ack_deadline_at, + } + ), int(source_delivery[0]), source_outbox_id, ), @@ -4649,6 +4930,7 @@ def prepare_connector_plan_commit( _connector_private_clear_current(source_private[0]) ) next_source_state["presentation_generation"] = int(plan[7]) + next_source_state["ack_deadline_at"] = ack_deadline_at next_source_state["presentation_max_part_count"] = max( int( next_source_state.get( @@ -5294,10 +5576,13 @@ def poll_connector_outbox( WHERE earlier_final.host_id = outbox.host_id AND earlier_final.connector = outbox.connector AND earlier_final.delivery_kind = 'final_ready' + AND earlier_final.ordering_key = outbox.ordering_key AND earlier_final.id < outbox.id AND earlier_final.status NOT IN ( 'delivered', - 'superseded' + 'superseded', + 'dead_letter', + 'awaiting_ack' ) ) AND NOT EXISTS ( @@ -5317,6 +5602,52 @@ def poll_connector_outbox( OR active_outbox.status != 'delivered' ) ) + AND NOT EXISTS ( + SELECT 1 + FROM turn_presentation_plans AS earlier_plan + JOIN connector_outbox AS earlier_source + ON earlier_source.id = earlier_plan.source_outbox_id + WHERE earlier_plan.host_id = outbox.host_id + AND earlier_plan.name = outbox.connector + AND earlier_plan.state IN ( + 'active', + 'waiting_predecessor' + ) + AND earlier_source.ordering_key = outbox.ordering_key + AND earlier_source.id < outbox.id + AND EXISTS ( + SELECT 1 + FROM turn_presentation_jobs AS earlier_job + LEFT JOIN connector_outbox AS earlier_job_outbox + ON earlier_job_outbox.id = earlier_job.outbox_id + WHERE earlier_job.plan_id = earlier_plan.id + AND ( + earlier_job_outbox.id IS NULL + OR earlier_job_outbox.status NOT IN ( + 'delivered', + 'superseded', + 'dead_letter' + ) + ) + ) + ) + ) + ) + AND ( + outbox.delivery_kind != 'final_part' + OR NOT EXISTS ( + SELECT 1 + FROM connector_outbox AS earlier_part + WHERE earlier_part.host_id = outbox.host_id + AND earlier_part.connector = outbox.connector + AND earlier_part.delivery_kind = 'final_part' + AND earlier_part.ordering_key = outbox.ordering_key + AND earlier_part.id < outbox.id + AND earlier_part.status NOT IN ( + 'delivered', + 'superseded', + 'dead_letter' + ) ) ) AND ( @@ -5545,6 +5876,188 @@ def _connector_validate_live_ref_conn( return None, "invalid_ref" +def renew_connector_delivery( + db_path: Path, + *, + host_id: str, + name: str, + ref: str, + lease_seconds: int, + now: str | None = None, +) -> dict[str, Any]: + """Extend one live connector lease without creating another attempt.""" + if not _sqlite_store_exists(db_path): + return _connector_error_response( + status="store_unavailable", host_id=host_id, name=name, ref=ref + ) + current_time = _connector_now(now) + with _connect(db_path, isolation_level=None) as conn: + _ensure_schema(conn) + conn.execute("BEGIN IMMEDIATE") + try: + _connector_reclaim_expired_leases_conn( + conn, + host_id=str(host_id), + name=str(name), + now=current_time, + ) + row, error = _connector_validate_live_ref_conn( + conn, + host_id=str(host_id), + name=str(name), + ref=str(ref), + now=current_time, + ) + if error is not None or row is None: + conn.rollback() + return _connector_error_response( + status=error or "invalid_ref", + host_id=host_id, + name=name, + ref=ref, + ) + delivery_state = _json_object(row[7]) + outbox_state = _json_object(row[9]) + current_expiry = str(delivery_state.get("lease_expires_at") or current_time) + requested_expiry = _connector_datetime( + _connector_add_seconds(current_time, max(1, int(lease_seconds))) + ) + leased_until = _connector_iso( + max( + _connector_datetime(current_expiry), + requested_expiry, + ) + ) + delivery_state["lease_expires_at"] = leased_until + outbox_state["lease_expires_at"] = leased_until + conn.execute( + "UPDATE connector_deliveries SET private_state_json = ? WHERE id = ?", + (_canonical_json(delivery_state), int(row[0])), + ) + conn.execute( + """ + UPDATE connector_outbox + SET private_state_json = ?, updated_at = ? + WHERE id = ? AND status = 'leased' + """, + (_canonical_json(outbox_state), current_time, int(row[1])), + ) + conn.commit() + return _connector_response( + ok=True, + status="renewed", + host_id=host_id, + name=name, + ref=ref, + key=str(row[4]), + attempt=int(row[5] or 0), + leased_until=leased_until, + ) + except Exception: + conn.rollback() + raise + + +def release_connector_delivery( + db_path: Path, + *, + host_id: str, + name: str, + ref: str, + now: str | None = None, +) -> dict[str, Any]: + """Release one live lease and make its outbox row immediately available.""" + if not _sqlite_store_exists(db_path): + return _connector_error_response( + status="store_unavailable", host_id=host_id, name=name, ref=ref + ) + current_time = _connector_now(now) + with _connect(db_path, isolation_level=None) as conn: + _ensure_schema(conn) + conn.execute("BEGIN IMMEDIATE") + try: + _connector_reclaim_expired_leases_conn( + conn, + host_id=str(host_id), + name=str(name), + now=current_time, + ) + row, error = _connector_validate_live_ref_conn( + conn, + host_id=str(host_id), + name=str(name), + ref=str(ref), + now=current_time, + ) + if error is not None or row is None: + conn.rollback() + return _connector_error_response( + status=error or "invalid_ref", + host_id=host_id, + name=name, + ref=ref, + ) + terminal_after_lease = bool( + _json_object(row[9]).get("terminal_after_lease") + ) + next_status = ( + _CONNECTOR_SUPERSEDED_OUTBOX_STATUS + if terminal_after_lease + else "queued" + ) + result_status = "superseded" if terminal_after_lease else "released" + conn.execute( + """ + UPDATE connector_deliveries + SET status = ?, response_json = ?, delivered_at = ? + WHERE id = ? AND status = 'leased' + """, + ( + result_status, + _canonical_json( + {"schema_version": 1, "status": result_status} + ), + current_time, + int(row[0]), + ), + ) + conn.execute( + """ + UPDATE connector_outbox + SET status = ?, next_attempt_at = ?, updated_at = ?, + private_state_json = ? + WHERE id = ? AND status = 'leased' + """, + ( + next_status, + None if terminal_after_lease else current_time, + current_time, + _connector_private_clear_current(row[9]), + int(row[1]), + ), + ) + _update_presentation_plan_after_outbox_conn( + conn, + outbox_id=int(row[1]), + outbox_status=next_status, + now=current_time, + ) + conn.commit() + return _connector_response( + ok=True, + status=result_status, + host_id=host_id, + name=name, + ref=ref, + key=str(row[4]), + attempt=int(row[5] or 0), + available_at=None if terminal_after_lease else current_time, + ) + except Exception: + conn.rollback() + raise + + def _connector_update_ref( db_path: Path, *, @@ -10972,6 +11485,116 @@ def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: _increment_turn_list_generation_conn(conn, host_id) +def _legacy_outbox_ordering_key( + *, + outbox_payload: Mapping[str, Any], + worker_id: Any, + turn_payload: Mapping[str, Any], +) -> str: + meta = _json_object(turn_payload.get("meta")) + stable_key = meta.get("stable_key") + if ( + _valid_final_stable_key(stable_key) + and type(meta.get("stable_key_version")) is int + and meta.get("stable_key_version") == 1 + ): + return str(stable_key) + nested_turn = outbox_payload.get("turn") + route = dict(nested_turn) if isinstance(nested_turn, Mapping) else outbox_payload + route_stable_key = route.get("stable_key") + if ( + _valid_final_stable_key(route_stable_key) + and type(route.get("stable_key_version")) is int + and route.get("stable_key_version") == 1 + ): + return str(route_stable_key) + return str(worker_id or route.get("worker_id") or "") + + +def _migrate_v15_to_v16_conn(conn: sqlite3.Connection) -> None: + """Partition final FIFO order and bound legacy awaiting-ack plans.""" + columns = _table_columns(conn, "connector_outbox") + if not columns: + conn.execute(CREATE_CONNECTOR_OUTBOX_TABLE) + elif "ordering_key" not in columns: + conn.execute( + "ALTER TABLE connector_outbox " + "ADD COLUMN ordering_key TEXT NOT NULL DEFAULT ''" + ) + if _table_columns(conn, "turns"): + rows = conn.execute( + """ + SELECT outbox.id, outbox.payload_json, turns.worker_id, turns.payload_json + FROM connector_outbox AS outbox + LEFT JOIN turns + ON turns.host_id = outbox.host_id + AND turns.turn_id = outbox.turn_id + WHERE outbox.turn_id IS NOT NULL + AND outbox.turn_id != '' + AND outbox.ordering_key = '' + ORDER BY outbox.id + """ + ).fetchall() + else: + rows = [ + (row[0], row[1], None, None) + for row in conn.execute( + """ + SELECT id, payload_json + FROM connector_outbox + WHERE turn_id IS NOT NULL AND turn_id != '' AND ordering_key = '' + ORDER BY id + """ + ).fetchall() + ] + for outbox_id, outbox_payload, worker_id, turn_payload in rows: + conn.execute( + "UPDATE connector_outbox SET ordering_key = ? WHERE id = ?", + ( + _legacy_outbox_ordering_key( + outbox_payload=_json_object(outbox_payload), + worker_id=worker_id, + turn_payload=_json_object(turn_payload), + ), + int(outbox_id), + ), + ) + deadline = _connector_add_seconds( + utc_timestamp(), + _configured_connector_ack_ttl_seconds(), + ) + awaiting_rows = conn.execute( + "SELECT id, private_state_json FROM connector_outbox WHERE status = 'awaiting_ack'" + ).fetchall() + for outbox_id, private_state_json in awaiting_rows: + state = _json_object(private_state_json) + state["ack_deadline_at"] = deadline + conn.execute( + "UPDATE connector_outbox SET private_state_json = ? WHERE id = ?", + (_canonical_json(state), int(outbox_id)), + ) + delivery_rows = ( + conn.execute( + """ + SELECT id, private_state_json + FROM connector_deliveries + WHERE outbox_id = ? AND status = 'awaiting_ack' + """, + (int(outbox_id),), + ).fetchall() + if _table_columns(conn, "connector_deliveries") + else [] + ) + for delivery_id, delivery_private in delivery_rows: + delivery_state = _json_object(delivery_private) + delivery_state["ack_deadline_at"] = deadline + conn.execute( + "UPDATE connector_deliveries SET private_state_json = ? WHERE id = ?", + (_canonical_json(delivery_state), int(delivery_id)), + ) + conn.execute(CREATE_CONNECTOR_ORDERING_INDEX) + + MIGRATIONS: tuple[Migration, ...] = ( Migration(0, 1, _migrate_v0_to_v1_conn), Migration(1, 2, _migrate_v1_to_v2_conn), @@ -10988,6 +11611,7 @@ def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: Migration(12, 13, _migrate_v12_to_v13_conn), Migration(13, 14, _migrate_v13_to_v14_conn), Migration(14, 15, _migrate_v14_to_v15_conn), + Migration(15, 16, _migrate_v15_to_v16_conn), ) @@ -11053,6 +11677,7 @@ def _create_current_schema_conn(conn: sqlite3.Connection) -> None: conn.execute(statement) for statement in CREATE_FINAL_DELIVERY_INDEXES: conn.execute(statement) + conn.execute(CREATE_CONNECTOR_ORDERING_INDEX) for statement in CREATE_SNAPSHOT_INDEXES: conn.execute(statement) conn.execute(INSERT_STORE_MAINTENANCE_STATE) diff --git a/tests/test_backend_pending.py b/tests/test_backend_pending.py index c160cb5..d06e3e5 100644 --- a/tests/test_backend_pending.py +++ b/tests/test_backend_pending.py @@ -1449,7 +1449,7 @@ def test_current_schema_creation_has_exact_binding_and_claim_state(tmp_path: Pat db = tmp_path / "current-schema.db" init_store(db) with sqlite3.connect(db) as conn: - assert conn.execute("PRAGMA user_version").fetchone()[0] == STORE_SCHEMA_VERSION == 15 + assert conn.execute("PRAGMA user_version").fetchone()[0] == STORE_SCHEMA_VERSION == 16 columns = { str(row[1]) for row in conn.execute("PRAGMA table_info(backend_pending)").fetchall() diff --git a/tests/test_config.py b/tests/test_config.py index 5ff7cec..ccad0f2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -34,6 +34,8 @@ def test_pr16_runtime_knobs_have_documented_defaults(monkeypatch) -> None: "TENDWIRE_MAX_WORKERS", "TENDWIRE_MAX_OUTBOX_ATTEMPTS", "TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS", + "TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS", + "TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", "TENDWIRE_COMMAND_RETRY_HORIZON_SECONDS", "TENDWIRE_COMMAND_RECEIPT_RETENTION_SECONDS", "TENDWIRE_COMMAND_RECEIPT_RETENTION_COUNT", @@ -49,6 +51,8 @@ def test_pr16_runtime_knobs_have_documented_defaults(monkeypatch) -> None: assert config.max_workers == 512 assert config.max_outbox_attempts == 10 assert config.connector_claim_ttl_seconds == 60 + assert config.connector_max_claim_ttl_seconds == 300 + assert config.connector_ack_ttl_seconds == 300 assert config.command_retry_horizon_seconds == 604_800 assert config.command_receipt_retention_seconds == 2_592_000 assert config.command_receipt_retention_count == 4096 @@ -62,6 +66,8 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: monkeypatch.setenv("TENDWIRE_MAX_WORKERS", "64") monkeypatch.setenv("TENDWIRE_MAX_OUTBOX_ATTEMPTS", "3") monkeypatch.setenv("TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS", "45") + monkeypatch.setenv("TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS", "240") + monkeypatch.setenv("TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", "180") monkeypatch.setenv("TENDWIRE_COMMAND_RETRY_HORIZON_SECONDS", "120") monkeypatch.setenv("TENDWIRE_COMMAND_RECEIPT_RETENTION_SECONDS", "691200") monkeypatch.setenv("TENDWIRE_COMMAND_RECEIPT_RETENTION_COUNT", "99") @@ -75,6 +81,8 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: max_workers="9", max_outbox_attempts="4", connector_claim_ttl_seconds="15", + connector_max_claim_ttl_seconds="120", + connector_ack_ttl_seconds="90", command_retry_horizon_seconds="60", command_receipt_retention_seconds="691200", command_receipt_retention_count="12", @@ -86,6 +94,8 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: assert env_config.max_workers == 64 assert env_config.max_outbox_attempts == 3 assert env_config.connector_claim_ttl_seconds == 45 + assert env_config.connector_max_claim_ttl_seconds == 240 + assert env_config.connector_ack_ttl_seconds == 180 assert env_config.command_retry_horizon_seconds == 120 assert env_config.command_receipt_retention_seconds == 691_200 assert env_config.command_receipt_retention_count == 99 @@ -96,6 +106,8 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: assert explicit.max_workers == 9 assert explicit.max_outbox_attempts == 4 assert explicit.connector_claim_ttl_seconds == 15 + assert explicit.connector_max_claim_ttl_seconds == 120 + assert explicit.connector_ack_ttl_seconds == 90 assert explicit.command_retry_horizon_seconds == 60 assert explicit.command_receipt_retention_seconds == 691_200 assert explicit.command_receipt_retention_count == 12 @@ -111,6 +123,12 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: ("max_workers", 0, "max_workers must be >= 1"), ("max_outbox_attempts", 0, "max_outbox_attempts must be >= 1"), ("connector_claim_ttl_seconds", 0, "connector_claim_ttl_seconds must be >= 1"), + ( + "connector_max_claim_ttl_seconds", + 0, + "connector_max_claim_ttl_seconds must be >= 1", + ), + ("connector_ack_ttl_seconds", 0, "connector_ack_ttl_seconds must be >= 1"), ], ) def test_pr16_runtime_knobs_reject_invalid_values(field: str, value: object, message: str) -> None: diff --git a/tests/test_connector_daemon_cli.py b/tests/test_connector_daemon_cli.py index d30ab6c..f9f8abc 100644 --- a/tests/test_connector_daemon_cli.py +++ b/tests/test_connector_daemon_cli.py @@ -119,6 +119,81 @@ def test_daemon_api_routes_connector_methods_safely(tmp_path: Path) -> None: _assert_json_only_and_safe(ack) +def test_daemon_api_routes_renew_and_release(tmp_path: Path) -> None: + db_path = tmp_path / "daemon-renew-release.db" + _enqueue(db_path, host_id="daemon-host", key="renew-release") + daemon = TendwireDaemon(Config(host_id="daemon-host", db_path=db_path)) + api = TendwireDaemonAPI( + get_snapshot=lambda: Snapshot(host_id="daemon-host"), + get_health=lambda: {"schema_version": 1, "status": "ok", "host_id": "daemon-host"}, + submit_command=daemon.submit_command, + connector_call=daemon.connector_call, + ) + first = api.dispatch( + {"method": "connector.poll", "params": {"name": "attention"}} + )["result"]["items"][0] + renewed = api.dispatch( + { + "method": "connector.renew", + "params": { + "name": "attention", + "ref": first["ref"], + "lease_seconds": 120, + }, + } + ) + released = api.dispatch( + { + "method": "connector.release", + "params": {"name": "attention", "ref": first["ref"]}, + } + ) + second = api.dispatch( + {"method": "connector.poll", "params": {"name": "attention"}} + )["result"]["items"][0] + + assert renewed["result"]["status"] == "renewed" + assert released["result"]["status"] == "released" + assert second["attempt"] == 2 + _assert_json_only_and_safe(renewed) + _assert_json_only_and_safe(released) + + +def test_daemon_periodic_tick_reclaims_without_a_followup_poll(tmp_path: Path) -> None: + db_path = tmp_path / "daemon-periodic-reclaim.db" + _enqueue(db_path, host_id="daemon-host", key="expired") + daemon = TendwireDaemon(Config(host_id="daemon-host", db_path=db_path)) + item = daemon.connector_call( + "connector.poll", + {"name": "attention", "lease_seconds": 60}, + )["items"][0] + assert item["attempt"] == 1 + with sqlite3.connect(str(db_path)) as conn: + for table in ("connector_outbox", "connector_deliveries"): + rows = conn.execute( + f"SELECT id, private_state_json FROM {table}" + ).fetchall() + for row_id, private_state_json in rows: + private = json.loads(private_state_json) + private["lease_expires_at"] = "2000-01-01T00:00:00+00:00" + conn.execute( + f"UPDATE {table} SET private_state_json = ? WHERE id = ?", + (json.dumps(private), row_id), + ) + + daemon._connector_periodic_tick() + + with sqlite3.connect(str(db_path)) as conn: + outbox_status = conn.execute( + "SELECT status FROM connector_outbox" + ).fetchone()[0] + delivery_status = conn.execute( + "SELECT status FROM connector_deliveries" + ).fetchone()[0] + assert outbox_status == "queued" + assert delivery_status == "expired" + + def test_cli_connector_poll_and_ack_print_json_only(tmp_path: Path, capsys) -> None: db_path = tmp_path / "cli-connector.db" _enqueue(db_path) @@ -169,6 +244,72 @@ def test_cli_connector_poll_and_ack_print_json_only(tmp_path: Path, capsys) -> N _assert_json_only_and_safe(ack_payload) +def test_cli_connector_renew_and_release_forward_live_ref_options( + tmp_path: Path, + capsys, +) -> None: + db_path = tmp_path / "cli-renew-release.db" + _enqueue(db_path, key="cli-renew-release") + poll_code = main( + [ + "--host-id", + "host-a", + "connector", + "poll", + "--name", + "attention", + "--db-path", + str(db_path), + ] + ) + poll_payload = json.loads(capsys.readouterr().out) + ref = poll_payload["items"][0]["ref"] + + renew_code = main( + [ + "--host-id", + "host-a", + "connector", + "renew", + "--name", + "attention", + "--ref", + ref, + "--lease-seconds", + "120", + "--db-path", + str(db_path), + ] + ) + renew_capture = capsys.readouterr() + renew_payload = json.loads(renew_capture.out) + release_code = main( + [ + "--host-id", + "host-a", + "connector", + "release", + "--name", + "attention", + "--ref", + ref, + "--db-path", + str(db_path), + ] + ) + release_capture = capsys.readouterr() + release_payload = json.loads(release_capture.out) + + assert poll_code == renew_code == release_code == 0 + assert renew_capture.err == release_capture.err == "" + assert renew_payload["status"] == "renewed" + assert renew_payload["leased_until"] + assert release_payload["status"] == "released" + assert release_payload["available_at"] + _assert_json_only_and_safe(renew_payload) + _assert_json_only_and_safe(release_payload) + + def test_cli_connector_prepare_reads_bounded_action_from_stdin( tmp_path: Path, capsys, diff --git a/tests/test_connector_outbox.py b/tests/test_connector_outbox.py index e9cbf69..e0dc784 100644 --- a/tests/test_connector_outbox.py +++ b/tests/test_connector_outbox.py @@ -106,6 +106,36 @@ def _enqueue(db_path: Path, *, key: str = "job-1", status: str = "queued") -> No ) +def _enqueue_final_root( + db_path: Path, + *, + key_suffix: str, + ordering_key: str, + status: str = "queued", +) -> str: + init_store(db_path) + key = f"turn-final:revision:twfinal1.{key_suffix:0<64}"[:94] + with sqlite3.connect(str(db_path)) as conn: + conn.execute( + """ + INSERT INTO connector_outbox ( + host_id, connector, delivery_key, delivery_kind, + ordering_key, status, payload_json, private_state_json, + created_at, updated_at + ) VALUES (?, 'turn-final', ?, 'final_ready', ?, ?, '{}', '{}', ?, ?) + """, + ( + "host-a", + key, + ordering_key, + status, + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:00:00+00:00", + ), + ) + return key + + def _delivery_rows(db_path: Path) -> list[tuple[Any, ...]]: with sqlite3.connect(str(db_path)) as conn: return conn.execute( @@ -170,6 +200,99 @@ def test_poll_uses_configured_default_lease_and_explicit_lease_wins(tmp_path: Pa assert explicit_delta == 5 +def test_turn_final_lease_request_is_capped_at_server_max(tmp_path: Path) -> None: + db_path = tmp_path / "turn-final-lease-cap.db" + _enqueue_final_root( + db_path, + key_suffix="lease-cap", + ordering_key="worker-a", + ) + api = ConnectorOutboxAPI(db_path, "host-a") + item = api.poll( + {"name": "turn-final", "lease_seconds": 900} + )["items"][0] + delta = ( + datetime.fromisoformat(item["leased_until"]) + - datetime.fromisoformat(item["available_at"]) + ).total_seconds() + assert delta == 300 + renewed = api.renew( + {"name": "turn-final", "ref": item["ref"], "lease_seconds": 900} + ) + renewal_delta = ( + datetime.fromisoformat(renewed["leased_until"]) + - datetime.fromisoformat(item["leased_until"]) + ).total_seconds() + assert 0 <= renewal_delta < 10 + + +def test_renew_extends_live_lease_and_release_requeues_immediately(tmp_path: Path) -> None: + db_path = tmp_path / "renew-release.db" + _enqueue(db_path, key="renew-release") + api = ConnectorOutboxAPI(db_path, "host-a") + first = api.poll({"name": "attention", "lease_seconds": 60})["items"][0] + renewed = api.renew( + {"name": "attention", "ref": first["ref"], "lease_seconds": 120} + ) + released = api.release({"name": "attention", "ref": first["ref"]}) + second = api.poll({"name": "attention"})["items"][0] + + assert renewed["status"] == "renewed" + assert datetime.fromisoformat(renewed["leased_until"]) > datetime.fromisoformat( + first["leased_until"] + ) + assert released["status"] == "released" + assert second["key"] == "renew-release" + assert second["attempt"] == 2 + + +@pytest.mark.parametrize("terminal_status", ["dead_letter", "awaiting_ack"]) +def test_terminal_or_planned_final_head_does_not_block_same_worker( + tmp_path: Path, + terminal_status: str, +) -> None: + db_path = tmp_path / f"nonblocking-{terminal_status}.db" + _enqueue_final_root( + db_path, + key_suffix=f"head-{terminal_status}", + ordering_key="worker-a", + status=terminal_status, + ) + tail_key = _enqueue_final_root( + db_path, + key_suffix=f"tail-{terminal_status}", + ordering_key="worker-a", + ) + items = ConnectorOutboxAPI(db_path, "host-a").poll( + {"name": "turn-final", "limit": 10} + )["items"] + assert [item["key"] for item in items] == [tail_key] + + +def test_final_fifo_is_strict_per_worker_but_isolates_other_workers(tmp_path: Path) -> None: + db_path = tmp_path / "per-worker-fifo.db" + _enqueue_final_root( + db_path, + key_suffix="worker-a-head", + ordering_key="worker-a", + status="leased", + ) + _enqueue_final_root( + db_path, + key_suffix="worker-a-tail", + ordering_key="worker-a", + ) + worker_b_key = _enqueue_final_root( + db_path, + key_suffix="worker-b", + ordering_key="worker-b", + ) + items = ConnectorOutboxAPI(db_path, "host-a").poll( + {"name": "turn-final", "limit": 10} + )["items"] + assert [item["key"] for item in items] == [worker_b_key] + + def test_reclaim_allows_fresh_ref_and_rejects_stale_ref(tmp_path: Path) -> None: db_path = tmp_path / "reclaim.db" _enqueue(db_path) @@ -758,6 +881,7 @@ def _canonical_turn( host_id: str = "host-a", worker_id: str = "worker-prepare", source_turn_id: str = "source-prepare", + stable_key: str = "wsk1_" + ("a" * 64), final_text: str, user_text: str | None = None, ) -> tuple[str, str]: @@ -798,7 +922,7 @@ def _canonical_turn( "source_turn_id": source_turn_id, "complete": True, "meta": { - "stable_key": "wsk1_" + ("a" * 64), + "stable_key": stable_key, "stable_key_version": 1, }, } @@ -1069,7 +1193,7 @@ def test_v6_to_current_plan_migration_is_bounded_atomic_and_preserves_jobs( ).fetchall() } foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall() - assert version == store_sqlite.STORE_SCHEMA_VERSION == 15 + assert version == store_sqlite.STORE_SCHEMA_VERSION == 16 assert plan_row == (plan["plan_token"], 1, None, "active") assert job_count == 2 assert outbox_count == 3 @@ -1624,6 +1748,7 @@ def test_unrelated_plans_poll_concurrently_but_never_colease_siblings( db_path, worker_id="worker-two", source_turn_id="source-two", + stable_key="wsk1_" + ("b" * 64), final_text="ijklmnop", ) api = ConnectorOutboxAPI(db_path, "host-a") @@ -2036,3 +2161,410 @@ def test_lost_commit_response_retries_after_supersede_or_failure_without_duplica assert conn.execute( "SELECT COUNT(*) FROM connector_outbox WHERE connector = 'turn-final'" ).fetchone()[0] == before_retry_count + + +@pytest.mark.parametrize( + ("stable_key", "expected_ordering_key"), + [ + ("wsk1_" + ("b" * 64), "wsk1_" + ("b" * 64)), + ("invalid", "worker-ordering-fallback"), + ], +) +def test_final_ready_anchor_materializes_the_turn_ordering_key( + tmp_path: Path, + stable_key: str, + expected_ordering_key: str, +) -> None: + db_path = tmp_path / f"root-ordering-{expected_ordering_key[:12]}.db" + turn_id, revision = _canonical_turn( + db_path, + worker_id="worker-ordering-fallback", + stable_key=stable_key, + final_text="abcdefgh", + ) + with sqlite3.connect(str(db_path)) as conn: + outbox_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now="2026-01-01T00:00:00+00:00", + ) + ordering_key = conn.execute( + "SELECT ordering_key FROM connector_outbox WHERE id = ?", + (outbox_id,), + ).fetchone()[0] + assert ordering_key == expected_ordering_key + + +def test_final_parts_inherit_the_turn_ordering_key(tmp_path: Path) -> None: + db_path = tmp_path / "part-ordering-key.db" + stable_key = "wsk1_" + ("b" * 64) + turn_id, revision = _canonical_turn( + db_path, + stable_key=stable_key, + final_text="abcdefgh", + ) + committed = _stage_final_plan( + ConnectorOutboxAPI(db_path, "host-a"), + turn_id=turn_id, + revision=revision, + ranges=[(0, 4), (4, 8)], + ) + with sqlite3.connect(str(db_path)) as conn: + keys = conn.execute( + """ + SELECT outbox.ordering_key + FROM turn_presentation_jobs AS jobs + JOIN turn_presentation_plans AS plans ON plans.id = jobs.plan_id + JOIN connector_outbox AS outbox ON outbox.id = jobs.outbox_id + WHERE plans.plan_token = ? + ORDER BY jobs.sequence_index + """, + (committed["plan_token"],), + ).fetchall() + assert keys == [(stable_key,), (stable_key,)] + + +def test_final_parts_preserve_enqueued_worker_fallback_after_adoption( + tmp_path: Path, +) -> None: + db_path = tmp_path / "part-ordering-worker-adoption.db" + turn_id, revision = _canonical_turn( + db_path, + worker_id="worker-before-adoption", + final_text="abcdefgh", + ) + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now="2026-01-01T00:00:00+00:00", + ) + assert source_id is not None + # Model a root whose enqueue-time fallback predates an adoption. The + # source row is the authority after enqueue, even if the turn's current + # worker identity later changes. + conn.execute( + "UPDATE connector_outbox SET ordering_key = ? WHERE id = ?", + ("worker-before-adoption", source_id), + ) + conn.execute( + "UPDATE turns SET worker_id = ? WHERE host_id = ? AND turn_id = ?", + ("worker-after-adoption", "host-a", turn_id), + ) + + api = ConnectorOutboxAPI(db_path, "host-a") + source = api.poll({"name": "turn-final"})["items"][0] + begun = api.prepare( + { + "schema_version": 1, + "action": "begin", + "name": "turn-final", + "turn_id": turn_id, + "content_revision": revision, + "presentation_version": "turn-present-adopted-worker-v1", + "part_count": 2, + "source_ref": source["ref"], + } + ) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=0, start=0, end=4) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=1, start=4, end=8) + committed = api.prepare( + { + "schema_version": 1, + "action": "commit", + "name": "turn-final", + "plan_token": begun["plan_token"], + "source_ref": source["ref"], + } + ) + + assert committed["ok"] is True + with sqlite3.connect(str(db_path)) as conn: + keys = conn.execute( + """ + SELECT outbox.ordering_key + FROM turn_presentation_jobs AS jobs + JOIN turn_presentation_plans AS plans ON plans.id = jobs.plan_id + JOIN connector_outbox AS outbox ON outbox.id = jobs.outbox_id + WHERE plans.plan_token = ? + ORDER BY jobs.sequence_index + """, + (committed["plan_token"],), + ).fetchall() + assert keys == [("worker-before-adoption",), ("worker-before-adoption",)] + + +def test_awaiting_ack_deadline_fails_plan_and_requeues_source(tmp_path: Path) -> None: + db_path = tmp_path / "awaiting-ack-reclaim.db" + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now="2026-01-01T00:00:00+00:00", + ) + conn.commit() + assert source_id is not None + api = ConnectorOutboxAPI(db_path, "host-a", ack_ttl_seconds=30) + source = api.poll({"name": "turn-final"})["items"][0] + begun = api.prepare( + { + "schema_version": 1, + "action": "begin", + "name": "turn-final", + "turn_id": turn_id, + "content_revision": revision, + "presentation_version": "turn-present-ack-deadline-v1", + "part_count": 2, + "source_ref": source["ref"], + } + ) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=0, start=0, end=4) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=1, start=4, end=8) + committed = api.prepare( + { + "schema_version": 1, + "action": "commit", + "name": "turn-final", + "plan_token": begun["plan_token"], + "source_ref": source["ref"], + } + ) + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now="9999-01-01T00:00:00+00:00", + ) + retry = poll_connector_outbox( + db_path, + "host-a", + "turn-final", + now="9999-01-01T00:00:00+00:00", + )["items"][0] + with sqlite3.connect(str(db_path)) as conn: + plan_state = conn.execute( + "SELECT state FROM turn_presentation_plans WHERE plan_token = ?", + (committed["plan_token"],), + ).fetchone()[0] + source_status = conn.execute( + "SELECT status FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone()[0] + job_statuses = conn.execute( + """ + SELECT outbox.status + FROM turn_presentation_jobs AS jobs + JOIN connector_outbox AS outbox ON outbox.id = jobs.outbox_id + JOIN turn_presentation_plans AS plans ON plans.id = jobs.plan_id + WHERE plans.plan_token = ? + ORDER BY jobs.sequence_index + """, + (committed["plan_token"],), + ).fetchall() + assert reclaimed["reclaimed"] == 1 + assert plan_state == "failed" + assert source_status == "leased" + assert job_statuses == [("dead_letter",), ("dead_letter",)] + assert retry["key"] == source["key"] + assert retry["attempt"] == 2 + + +def test_awaiting_ack_deadline_preserves_explicit_suffix_recovery( + tmp_path: Path, +) -> None: + db_path = tmp_path / "awaiting-ack-suffix-recovery.db" + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now="2026-01-01T00:00:00+00:00", + ) + conn.commit() + assert source_id is not None + api = ConnectorOutboxAPI(db_path, "host-a", ack_ttl_seconds=30) + source = api.poll({"name": "turn-final"})["items"][0] + begun = api.prepare( + { + "schema_version": 1, + "action": "begin", + "name": "turn-final", + "turn_id": turn_id, + "content_revision": revision, + "presentation_version": "turn-present-ack-recover-v1", + "part_count": 2, + "source_ref": source["ref"], + } + ) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=0, start=0, end=4) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=1, start=4, end=8) + committed = api.prepare( + { + "schema_version": 1, + "action": "commit", + "name": "turn-final", + "plan_token": begun["plan_token"], + "source_ref": source["ref"], + } + ) + first_part = api.poll({"name": "turn-final", "limit": 10})["items"][0] + assert first_part["payload"]["part_ordinal"] == 0 + assert api.ack({"name": "turn-final", "ref": first_part["ref"]})[ + "status" + ] == "acknowledged" + + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now="9999-01-01T00:00:00+00:00", + ) + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + "SELECT status FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone() == ("queued",) + recovered = api.prepare( + { + "schema_version": 1, + "action": "recover", + "name": "turn-final", + "failed_plan_token": committed["plan_token"], + "request_id": "ack-deadline-recover-1", + } + ) + suffix = api.poll({"name": "turn-final", "limit": 10})["items"] + + assert reclaimed["reclaimed"] == 1 + assert recovered["status"] == "recovered" + assert recovered["acknowledged_prefix_count"] == 1 + assert recovered["executable_job_count"] == 1 + assert len(suffix) == 1 + assert suffix[0]["payload"]["part_ordinal"] == 1 + + +def test_awaiting_ack_without_plan_becomes_terminal_failed(tmp_path: Path) -> None: + db_path = tmp_path / "awaiting-ack-unrecoverable.db" + _enqueue_final_root( + db_path, + key_suffix="orphan-awaiting", + ordering_key="worker-a", + status="awaiting_ack", + ) + with sqlite3.connect(str(db_path)) as conn: + conn.execute( + """ + UPDATE connector_outbox + SET private_state_json = '{"ack_deadline_at":"2026-01-01T00:00:00+00:00"}' + """ + ) + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now="2026-01-01T00:00:01+00:00", + ) + with sqlite3.connect(str(db_path)) as conn: + status = conn.execute("SELECT status FROM connector_outbox").fetchone()[0] + assert reclaimed["reclaimed"] == 1 + assert status == "dead_letter" + + +def test_v16_migration_backfills_ordering_and_awaiting_ack_deadlines( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "connector-v16-migration.db" + stable_key = "wsk1_" + ("c" * 64) + tombstoned_turn, _ = _canonical_turn( + db_path, + worker_id="worker-tombstoned", + source_turn_id="source-tombstoned", + stable_key=stable_key, + final_text="done", + ) + fallback_turn, _ = _canonical_turn( + db_path, + worker_id="worker-fallback", + source_turn_id="source-fallback", + stable_key="invalid", + final_text="done", + ) + with sqlite3.connect(str(db_path)) as conn: + assert store_sqlite._tombstone_turn_conn( + conn, + "host-a", + tombstoned_turn, + superseded_by_turn_id=None, + superseded_at="2026-01-02T00:00:00+00:00", + ) + for turn_id, status in ( + (tombstoned_turn, "awaiting_ack"), + (fallback_turn, "dead_letter"), + (fallback_turn, "superseded"), + ): + cursor = conn.execute( + """ + INSERT INTO connector_outbox ( + host_id, connector, delivery_key, delivery_kind, turn_id, + ordering_key, status, payload_json, private_state_json, + created_at, updated_at + ) VALUES (?, 'turn-final', ?, 'final_ready', ?, '', ?, '{}', '{}', ?, ?) + """, + ( + "host-a", + f"migration-{turn_id}-{status}", + turn_id, + status, + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:00:00+00:00", + ), + ) + if status == "awaiting_ack": + conn.execute( + """ + INSERT INTO connector_deliveries ( + outbox_id, host_id, connector, delivery_key, attempt, + status, response_json, private_state_json, created_at + ) VALUES (?, 'host-a', 'turn-final', 'migration-awaiting', 1, + 'awaiting_ack', '{}', '{}', ?) + """, + (cursor.lastrowid, "2026-01-01T00:00:00+00:00"), + ) + conn.execute("DROP INDEX IF EXISTS idx_connector_outbox_final_ordering") + conn.execute("ALTER TABLE connector_outbox DROP COLUMN ordering_key") + conn.execute("PRAGMA user_version = 15") + migration_now = "2026-01-03T00:00:00+00:00" + monkeypatch.setenv("TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", "123") + monkeypatch.setattr(store_sqlite, "utc_timestamp", lambda: migration_now) + init_store(db_path) + with sqlite3.connect(str(db_path)) as conn: + rows = conn.execute( + """ + SELECT turn_id, status, ordering_key, private_state_json + FROM connector_outbox + WHERE delivery_key LIKE 'migration-%' + ORDER BY id + """ + ).fetchall() + delivery_private = conn.execute( + "SELECT private_state_json FROM connector_deliveries WHERE status = 'awaiting_ack'" + ).fetchone()[0] + assert rows[0][0:3] == (tombstoned_turn, "awaiting_ack", stable_key) + ack_deadline = json.loads(rows[0][3])["ack_deadline_at"] + assert ( + datetime.fromisoformat(ack_deadline) + - datetime.fromisoformat(migration_now) + ).total_seconds() == 123 + assert rows[1][0:3] == (fallback_turn, "dead_letter", "worker-fallback") + assert rows[2][0:3] == (fallback_turn, "superseded", "worker-fallback") + assert json.loads(delivery_private)["ack_deadline_at"] == ack_deadline diff --git a/tests/test_daemon.py b/tests/test_daemon.py index 9f01105..917e178 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -1395,6 +1395,8 @@ def test_daemon_health_exposes_public_operational_status_without_private_values( max_workers=8, max_outbox_attempts=4, connector_claim_ttl_seconds=33, + connector_max_claim_ttl_seconds=222, + connector_ack_ttl_seconds=111, acknowledged_final_retention_days=40, acknowledged_final_retention_count=500, snapshot_retention_days=9, @@ -1525,6 +1527,8 @@ def operational_status(self) -> dict[str, Any]: "max_workers": 8, "max_outbox_attempts": 4, "outbox_claim_ttl_seconds": 33, + "outbox_max_claim_ttl_seconds": 222, + "outbox_ack_ttl_seconds": 111, "acknowledged_final_retention_days": 40, "acknowledged_final_retention_count": 500, "command_retry_horizon_seconds": 120, diff --git a/tests/test_delivery_retention_migration.py b/tests/test_delivery_retention_migration.py index 7d7d134..1b3d2dd 100644 --- a/tests/test_delivery_retention_migration.py +++ b/tests/test_delivery_retention_migration.py @@ -875,7 +875,7 @@ def test_v10_to_v12_migration_retains_finals_without_reposting_or_leaking( finals = _seed_v10_finals(db_path) init_store(db_path) - assert store_sqlite.STORE_SCHEMA_VERSION == 15 + assert store_sqlite.STORE_SCHEMA_VERSION == 16 delivered_key = _final_key(*finals["delivered"]) hold_keys = {_final_key(*finals[label]) for label in ("hold-a", "hold-b")} diff --git a/tests/test_delivery_retention_projection.py b/tests/test_delivery_retention_projection.py index c0e5eee..4e3aa0b 100644 --- a/tests/test_delivery_retention_projection.py +++ b/tests/test_delivery_retention_projection.py @@ -165,7 +165,7 @@ def _turn_rows_snapshot(db_path: Path) -> tuple[tuple[Any, ...], ...]: def _assert_continuity_integrity(db_path: Path) -> None: with sqlite3.connect(str(db_path)) as conn: - assert conn.execute("PRAGMA user_version").fetchone() == (store_sqlite.STORE_SCHEMA_VERSION,) == (15,) + assert conn.execute("PRAGMA user_version").fetchone() == (store_sqlite.STORE_SCHEMA_VERSION,) == (16,) assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] current_counts = conn.execute( """ diff --git a/tests/test_delivery_retention_recovery.py b/tests/test_delivery_retention_recovery.py index 37e2c7e..2e2ad27 100644 --- a/tests/test_delivery_retention_recovery.py +++ b/tests/test_delivery_retention_recovery.py @@ -973,7 +973,7 @@ def test_v12_migration_uses_effective_recovery_lineage_without_repost_or_hold( api = ConnectorOutboxAPI(db_path, HOST_ID) assert api.poll({"name": FINAL_NAME, "limit": 100})["items"] == [] with sqlite3.connect(str(db_path)) as conn: - assert conn.execute("PRAGMA user_version").fetchone()[0] == store_sqlite.STORE_SCHEMA_VERSION == 15 + assert conn.execute("PRAGMA user_version").fetchone()[0] == store_sqlite.STORE_SCHEMA_VERSION == 16 anchor = conn.execute( """ SELECT delivery_kind, status diff --git a/tests/test_store.py b/tests/test_store.py index e0f1123..160726d 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -11191,7 +11191,7 @@ def test_v13_migration_repairs_nonpositive_turn_sequences_and_blocks_recurrence( init_store(db_path) with sqlite3.connect(str(db_path)) as conn: - assert _user_version(conn) == store_sqlite.STORE_SCHEMA_VERSION == 15 + assert _user_version(conn) == store_sqlite.STORE_SCHEMA_VERSION == 16 assert conn.execute( """ SELECT turn_id, list_sequence From 6cbf083227a13bb54f273725bf84c40bd1507f12 Mon Sep 17 00:00:00 2001 From: Jerry the Martian <38183696+jerryfane@users.noreply.github.com> Date: Sun, 19 Jul 2026 12:17:07 +0200 Subject: [PATCH 5/7] Gitmoot merge adhoc-4f4edfa1 Merged by Gitmoot after policy gate passed. --- README.md | 19 +- scripts/turn_ingestion_benchmark.py | 11 +- src/tendwire/cli.py | 10 +- src/tendwire/connectors/outbox.py | 1 + src/tendwire/daemon.py | 34 +- src/tendwire/store/sqlite.py | 415 ++++++++++++++-- tests/test_backend_pending.py | 2 +- tests/test_connector_daemon_cli.py | 21 + tests/test_connector_outbox.py | 513 +++++++++++++++++++- tests/test_delivery_retention_migration.py | 2 +- tests/test_delivery_retention_projection.py | 2 +- tests/test_delivery_retention_recovery.py | 2 +- tests/test_store.py | 2 +- 13 files changed, 961 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index de3baf9..c8302c8 100644 --- a/README.md +++ b/README.md @@ -1252,7 +1252,8 @@ effect. `connector.poll` atomically leases due `connector_outbox` rows for one `name` and returns opaque per-attempt refs. A live lease prevents duplicate polling. Expired leases are reclaimed before polling and before ref-mutating operations; -the daemon also runs this reclaim pass on its periodic tick, and +the daemon first performs a read-only due check on its periodic tick and takes +the reclaim write transaction only when work is actually expired. `connector.reclaim` can be called directly. `connector.renew` extends a live lease without creating a delivery attempt. `connector.release` records the live attempt as released and makes the row immediately pollable again. `connector.ack` @@ -1273,13 +1274,21 @@ public status `attempts_exhausted` without exposing private outbox or delivery state. Final-root FIFO is partitioned by the turn's immutable stable worker key, with -the persisted worker ID as the enqueue-time fallback. A blocked worker therefore +the persisted worker ID as the enqueue-time fallback. Legacy backfill prefers +the enqueue-era stable key carried in the outbox payload; rows that predate that +metadata still fall back to the turn's current persisted worker ID. If neither +identity can be resolved, each row receives its own `orphan:` partition so +unrelated degraded rows cannot block one another. A blocked worker therefore cannot starve another worker, while roots and plan parts for the same worker stay strictly ordered. `dead_letter`, `superseded`, and `delivered` roots are terminal for this gate. A committed source in `awaiting_ack` no longer blocks by itself; -its plan jobs carry the ordering obligation. Commit stamps an ACK deadline, and -an expired incomplete plan is failed and its source requeued. Missing or -unrecoverable plan state terminates the source instead of leaving it pending. +its plan jobs carry the ordering obligation, including source-less recovery +plans. Commit stamps an ACK deadline, each acknowledged plan job extends it, +and a valid part lease prevents deadline reclaim from interrupting in-flight +delivery. ACK-deadline recovery does not consume the connector failure-attempt +budget. An expired incomplete plan is otherwise failed and its source requeued. +Missing or unrecoverable plan state terminates the source instead of leaving it +pending. ### Delivery-aware final roots and acknowledged history diff --git a/scripts/turn_ingestion_benchmark.py b/scripts/turn_ingestion_benchmark.py index 5b9d2bf..13ec651 100755 --- a/scripts/turn_ingestion_benchmark.py +++ b/scripts/turn_ingestion_benchmark.py @@ -1241,7 +1241,16 @@ def submit_command(_config: Config, payload: str) -> Mapping[str, Any]: ) ) - baseline_threads source_calls_final = len(adapter_records) - forbidden_values.extend(str(process_id) for process_id in process_ids) + forbidden_values.extend( + marker + for process_id in process_ids + for marker in ( + f'"process_id":{process_id}', + f'"process_id":"{process_id}"', + f'"pid":{process_id}', + f'"pid":"{process_id}"', + ) + ) overlap_ns = _first_call_overlap_ns(adapter_records, args.blocked_workers) response_bytes_max = max( metric["response_bytes_max"] for metric in latency.values() diff --git a/src/tendwire/cli.py b/src/tendwire/cli.py index 67609d6..e7afc5a 100644 --- a/src/tendwire/cli.py +++ b/src/tendwire/cli.py @@ -1105,7 +1105,10 @@ def cmd_turn_content_get(config: Config, args: argparse.Namespace) -> int: else: from .store.sqlite import get_turn_content, init_store - init_store(config.db_path) + init_store( + config.db_path, + connector_ack_ttl_seconds=config.connector_ack_ttl_seconds, + ) payload = get_turn_content( config.db_path, config.host_id, @@ -1485,7 +1488,10 @@ def cmd_connector(config: Config, args: argparse.Namespace) -> int: from .connectors import ConnectorOutboxAPI from .store.sqlite import init_store - init_store(config.db_path) + init_store( + config.db_path, + connector_ack_ttl_seconds=config.connector_ack_ttl_seconds, + ) payload = ConnectorOutboxAPI( config.db_path, config.host_id, diff --git a/src/tendwire/connectors/outbox.py b/src/tendwire/connectors/outbox.py index 43a87d2..dd94796 100644 --- a/src/tendwire/connectors/outbox.py +++ b/src/tendwire/connectors/outbox.py @@ -490,6 +490,7 @@ def ack(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: name=name, ref=live_ref, response=_clean_mapping(data.get("response")), + ack_ttl_seconds=self.ack_ttl_seconds, ) def fail(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: diff --git a/src/tendwire/daemon.py b/src/tendwire/daemon.py index 05448fa..8d1216b 100644 --- a/src/tendwire/daemon.py +++ b/src/tendwire/daemon.py @@ -431,10 +431,19 @@ def default_socket_path(config: Config) -> Path: return Path(config.data_dir) / "tendwire.sock" -def _default_init_store(db_path: Path) -> None: +def _default_init_store( + db_path: Path, + *, + connector_ack_ttl_seconds: int | None = None, +) -> None: from .store.sqlite import init_store - init_store(db_path) + kwargs = ( + {"connector_ack_ttl_seconds": connector_ack_ttl_seconds} + if connector_ack_ttl_seconds is not None + else {} + ) + init_store(db_path, **kwargs) def _default_observe_initial_snapshot(config: Config) -> Snapshot: @@ -517,7 +526,15 @@ def start(self) -> None: self.config.installation_key_sentinel_path, ), ) - self.hooks.init_store(Path(self.config.db_path)) + if self.hooks.init_store is _default_init_store: + _default_init_store( + Path(self.config.db_path), + connector_ack_ttl_seconds=( + self.config.connector_ack_ttl_seconds + ), + ) + else: + self.hooks.init_store(Path(self.config.db_path)) self._connector_periodic_tick() if self.config.herdr_backend == "socket": self._snapshot = self._start_socket_event_backend() @@ -1083,9 +1100,18 @@ def _connector_periodic_tick(self) -> None: """Eagerly reclaim expired connector work without waiting for a poll.""" if self.config.db_path is None: return - from .store.sqlite import reclaim_expired_connector_leases + from .store.sqlite import ( + connector_reclaim_due, + reclaim_expired_connector_leases, + ) try: + if not connector_reclaim_due( + Path(self.config.db_path), + self.config.host_id, + None, + ): + return reclaim_expired_connector_leases( Path(self.config.db_path), self.config.host_id, diff --git a/src/tendwire/store/sqlite.py b/src/tendwire/store/sqlite.py index 1d71354..1e1b1ae 100644 --- a/src/tendwire/store/sqlite.py +++ b/src/tendwire/store/sqlite.py @@ -112,7 +112,7 @@ FINGERPRINT_HEX_LENGTH = FINGERPRINT_HEX_CHARS -STORE_SCHEMA_VERSION = 16 +STORE_SCHEMA_VERSION = 17 CONNECTOR_ACK_TTL_SECONDS = DEFAULT_CONNECTOR_ACK_TTL_SECONDS TURN_CLAIM_HARD_TTL_SECONDS = DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS = 60.0 @@ -172,19 +172,6 @@ def _configured_turn_claim_hard_ttl_seconds() -> int: return configured -def _configured_connector_ack_ttl_seconds() -> int: - raw = os.environ.get("TENDWIRE_CONNECTOR_ACK_TTL_SECONDS") - if raw is None: - return CONNECTOR_ACK_TTL_SECONDS - try: - configured = int(raw) - except (TypeError, ValueError) as exc: - raise StoreSchemaError("connector_ack_ttl_invalid") from exc - if configured <= 0: - raise StoreSchemaError("connector_ack_ttl_invalid") - return configured - - @dataclass(frozen=True) class Migration: """One exact, transaction-external schema transition.""" @@ -2138,6 +2125,12 @@ def _connector_reclaim_expired_awaiting_ack_conn( } if recoverable: plan_id = int(plan[0]) + if _connector_plan_has_live_job_lease_conn( + conn, + plan_id=plan_id, + now_dt=now_dt, + ): + continue conn.execute( "UPDATE turn_presentation_plans SET state = 'failed' WHERE id = ?", (plan_id,), @@ -2241,6 +2234,37 @@ def _connector_reclaim_expired_awaiting_ack_conn( return reclaimed +def _connector_plan_has_live_job_lease_conn( + conn: sqlite3.Connection, + *, + plan_id: int, + now_dt: datetime, +) -> bool: + live_job_leases = conn.execute( + """ + SELECT deliveries.private_state_json + FROM turn_presentation_jobs AS jobs + JOIN connector_outbox AS outbox + ON outbox.id = jobs.outbox_id + JOIN connector_deliveries AS deliveries + ON deliveries.outbox_id = outbox.id + WHERE jobs.plan_id = ? + AND outbox.status = 'leased' + AND deliveries.status = 'leased' + """, + (int(plan_id),), + ).fetchall() + return any( + ( + lease_expires_at := str( + _json_object(row[0]).get("lease_expires_at") or "" + ) + ) + and _connector_datetime(lease_expires_at) > now_dt + for row in live_job_leases + ) + + def _connector_exhaust_retryable_conn( conn: sqlite3.Connection, *, @@ -2255,7 +2279,22 @@ def _connector_exhaust_retryable_conn( "status IN ('queued', 'deferred', 'retry')", """ ( - SELECT COALESCE(MAX(d.attempt), 0) + SELECT + COALESCE(MAX(d.attempt), 0) + - COALESCE( + SUM( + CASE + WHEN d.status = 'failed' + AND COALESCE( + json_extract(d.response_json, '$.status'), + '' + ) = 'ack_deadline_expired' + THEN 1 + ELSE 0 + END + ), + 0 + ) FROM connector_deliveries d WHERE d.outbox_id = connector_outbox.id ) >= ? @@ -2298,6 +2337,93 @@ def _connector_exhaust_retryable_conn( return int(cursor.rowcount or 0) +def connector_reclaim_due( + db_path: Path, + host_id: str, + name: str | None = None, + *, + now: str | None = None, +) -> bool: + """Return whether connector work is due for reclaim without taking a write lock.""" + if not _sqlite_store_exists(db_path): + return False + current_time = _connector_now(now) + now_dt = _connector_datetime(current_time) + delivery_clauses = ["host_id = ?", "status = 'leased'"] + outbox_clauses = ["host_id = ?", "status = 'awaiting_ack'"] + params: list[Any] = [str(host_id)] + if name is not None: + delivery_clauses.append("connector = ?") + outbox_clauses.append("connector = ?") + params.append(str(name)) + with _connect(db_path) as conn: + _ensure_schema(conn) + lease_rows = conn.execute( + f""" + SELECT private_state_json + FROM connector_deliveries + WHERE {' AND '.join(delivery_clauses)} + """, + params, + ).fetchall() + for (private_state_json,) in lease_rows: + expires_at = str( + _json_object(private_state_json).get("lease_expires_at") or "" + ) + if expires_at and _connector_datetime(expires_at) <= now_dt: + return True + awaiting_rows = conn.execute( + f""" + SELECT + outbox.id, + outbox.private_state_json, + ( + SELECT deliveries.private_state_json + FROM connector_deliveries AS deliveries + WHERE deliveries.outbox_id = outbox.id + AND deliveries.status = 'awaiting_ack' + ORDER BY deliveries.id DESC + LIMIT 1 + ) + FROM connector_outbox AS outbox + WHERE {' AND '.join(outbox_clauses)} + """, + params, + ).fetchall() + for outbox_id, outbox_private, delivery_private in awaiting_rows: + deadline = str( + _json_object(outbox_private).get("ack_deadline_at") or "" + ) + if not deadline: + deadline = str( + _json_object(delivery_private).get("ack_deadline_at") or "" + ) + if not deadline or _connector_datetime(deadline) > now_dt: + continue + plan = conn.execute( + """ + SELECT id, state + FROM turn_presentation_plans + WHERE source_outbox_id = ? + ORDER BY id DESC + LIMIT 1 + """, + (int(outbox_id),), + ).fetchone() + if ( + plan is not None + and str(plan[1]) in {"active", "waiting_predecessor", "failed"} + and _connector_plan_has_live_job_lease_conn( + conn, + plan_id=int(plan[0]), + now_dt=now_dt, + ) + ): + continue + return True + return False + + def reclaim_expired_connector_leases( db_path: Path, host_id: str, @@ -2447,7 +2573,7 @@ def _turn_ordering_key_conn( (str(host_id), str(turn_id)), ).fetchone() if row is None: - return "" + return f"orphan:{turn_id}" payload = _json_object(row[1]) meta = _json_object(payload.get("meta")) stable_key = meta.get("stable_key") @@ -2457,7 +2583,7 @@ def _turn_ordering_key_conn( and meta.get("stable_key_version") == 1 ): return str(stable_key) - return str(row[0] or "") + return str(row[0] or f"orphan:{turn_id}") def _final_content_field_descriptor( @@ -4282,6 +4408,7 @@ def _update_presentation_plan_after_outbox_conn( outbox_id: int, outbox_status: str, now: str, + ack_ttl_seconds: int | None = None, ) -> None: plan = conn.execute( """ @@ -4323,7 +4450,67 @@ def _update_presentation_plan_after_outbox_conn( """, (plan_id,), ).fetchone() - if int(remaining[0] or 0) == 0: + remaining_count = int(remaining[0] or 0) + source_outbox_id = ( + int(plan[4]) if plan[4] is not None else None + ) + if ( + remaining_count > 0 + and source_outbox_id is not None + and ack_ttl_seconds is not None + ): + deadline = _connector_add_seconds( + str(now), + max(1, int(ack_ttl_seconds)), + ) + source_row = conn.execute( + """ + SELECT private_state_json + FROM connector_outbox + WHERE id = ? AND status = 'awaiting_ack' + """, + (source_outbox_id,), + ).fetchone() + if source_row is not None: + source_state = _json_object(source_row[0]) + source_state["ack_deadline_at"] = deadline + conn.execute( + """ + UPDATE connector_outbox + SET private_state_json = ?, updated_at = ? + WHERE id = ? AND status = 'awaiting_ack' + """, + ( + _canonical_json(source_state), + str(now), + source_outbox_id, + ), + ) + delivery_row = conn.execute( + """ + SELECT id, private_state_json + FROM connector_deliveries + WHERE outbox_id = ? AND status = 'awaiting_ack' + ORDER BY id DESC + LIMIT 1 + """, + (source_outbox_id,), + ).fetchone() + if delivery_row is not None: + delivery_state = _json_object(delivery_row[1]) + delivery_state["ack_deadline_at"] = deadline + conn.execute( + """ + UPDATE connector_deliveries + SET private_state_json = ? + WHERE id = ? AND status = 'awaiting_ack' + """, + ( + _canonical_json(delivery_state), + int(delivery_row[0]), + ), + ) + if remaining_count == 0: conn.execute( """ UPDATE turn_presentation_plans @@ -4332,9 +4519,6 @@ def _update_presentation_plan_after_outbox_conn( """, (str(now), plan_id), ) - source_outbox_id = ( - int(plan[4]) if plan[4] is not None else None - ) if source_outbox_id is not None: conn.execute( """ @@ -5605,7 +5789,7 @@ def poll_connector_outbox( AND NOT EXISTS ( SELECT 1 FROM turn_presentation_plans AS earlier_plan - JOIN connector_outbox AS earlier_source + LEFT JOIN connector_outbox AS earlier_source ON earlier_source.id = earlier_plan.source_outbox_id WHERE earlier_plan.host_id = outbox.host_id AND earlier_plan.name = outbox.connector @@ -5613,8 +5797,30 @@ def poll_connector_outbox( 'active', 'waiting_predecessor' ) - AND earlier_source.ordering_key = outbox.ordering_key - AND earlier_source.id < outbox.id + AND ( + ( + earlier_source.ordering_key + = outbox.ordering_key + AND earlier_source.id < outbox.id + ) + OR ( + earlier_plan.source_outbox_id IS NULL + AND EXISTS ( + SELECT 1 + FROM turn_presentation_jobs + AS ordering_job + JOIN connector_outbox + AS ordering_outbox + ON ordering_outbox.id + = ordering_job.outbox_id + WHERE ordering_job.plan_id + = earlier_plan.id + AND ordering_outbox.ordering_key + = outbox.ordering_key + AND ordering_outbox.id < outbox.id + ) + ) + ) AND EXISTS ( SELECT 1 FROM turn_presentation_jobs AS earlier_job @@ -6070,6 +6276,7 @@ def _connector_update_ref( available_at: str | None = None, delay_seconds: int | None = None, max_attempts: int | None = None, + ack_ttl_seconds: int | None = None, now: str | None = None, ) -> dict[str, Any]: if not _sqlite_store_exists(db_path): @@ -6204,6 +6411,7 @@ def _connector_update_ref( outbox_id=outbox_id, outbox_status=_CONNECTOR_TERMINAL_OUTBOX_STATUS, now=current_time, + ack_ttl_seconds=ack_ttl_seconds, ) conn.commit() return _connector_response( @@ -6224,7 +6432,40 @@ def _connector_update_ref( else: available_at = _connector_iso(available_at) attempt_limit = max(1, int(max_attempts)) if max_attempts is not None else None - exhausted = action == "fail" and attempt_limit is not None and attempt >= attempt_limit + attempts_used = int( + conn.execute( + """ + SELECT + COALESCE(MAX(attempt), 0) + - COALESCE( + SUM( + CASE + WHEN status = 'failed' + AND COALESCE( + json_extract( + response_json, + '$.status' + ), + '' + ) = 'ack_deadline_expired' + THEN 1 + ELSE 0 + END + ), + 0 + ) + FROM connector_deliveries + WHERE outbox_id = ? + """, + (outbox_id,), + ).fetchone()[0] + or 0 + ) + exhausted = ( + action == "fail" + and attempt_limit is not None + and attempts_used >= attempt_limit + ) result_status = "attempts_exhausted" if exhausted else ("retry_scheduled" if action == "fail" else "deferred") delivery_status = "failed" if action == "fail" else "deferred" outbox_status = ( @@ -6298,6 +6539,7 @@ def ack_connector_delivery( name: str, ref: str, response: Mapping[str, Any] | None = None, + ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, now: str | None = None, ) -> dict[str, Any]: """Acknowledge a live connector lease and make the outbox item terminal.""" @@ -6308,6 +6550,7 @@ def ack_connector_delivery( name=name, ref=ref, response=response, + ack_ttl_seconds=max(1, int(ack_ttl_seconds)), now=now, ) @@ -11487,10 +11730,26 @@ def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: def _legacy_outbox_ordering_key( *, + outbox_id: int, outbox_payload: Mapping[str, Any], worker_id: Any, turn_payload: Mapping[str, Any], ) -> str: + nested_turn = outbox_payload.get("turn") + route = dict(nested_turn) if isinstance(nested_turn, Mapping) else outbox_payload + route_meta = _json_object(route.get("meta")) + route_stable_key = route.get("stable_key") or route_meta.get("stable_key") + route_stable_key_version = ( + route.get("stable_key_version") + if route.get("stable_key") is not None + else route_meta.get("stable_key_version") + ) + if ( + _valid_final_stable_key(route_stable_key) + and type(route_stable_key_version) is int + and route_stable_key_version == 1 + ): + return str(route_stable_key) meta = _json_object(turn_payload.get("meta")) stable_key = meta.get("stable_key") if ( @@ -11499,19 +11758,14 @@ def _legacy_outbox_ordering_key( and meta.get("stable_key_version") == 1 ): return str(stable_key) - nested_turn = outbox_payload.get("turn") - route = dict(nested_turn) if isinstance(nested_turn, Mapping) else outbox_payload - route_stable_key = route.get("stable_key") - if ( - _valid_final_stable_key(route_stable_key) - and type(route.get("stable_key_version")) is int - and route.get("stable_key_version") == 1 - ): - return str(route_stable_key) - return str(worker_id or route.get("worker_id") or "") + return str(worker_id or route.get("worker_id") or f"orphan:{outbox_id}") -def _migrate_v15_to_v16_conn(conn: sqlite3.Connection) -> None: +def _migrate_v15_to_v16_conn( + conn: sqlite3.Connection, + *, + connector_ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, +) -> None: """Partition final FIFO order and bound legacy awaiting-ack plans.""" columns = _table_columns(conn, "connector_outbox") if not columns: @@ -11529,9 +11783,7 @@ def _migrate_v15_to_v16_conn(conn: sqlite3.Connection) -> None: LEFT JOIN turns ON turns.host_id = outbox.host_id AND turns.turn_id = outbox.turn_id - WHERE outbox.turn_id IS NOT NULL - AND outbox.turn_id != '' - AND outbox.ordering_key = '' + WHERE outbox.ordering_key = '' ORDER BY outbox.id """ ).fetchall() @@ -11542,7 +11794,7 @@ def _migrate_v15_to_v16_conn(conn: sqlite3.Connection) -> None: """ SELECT id, payload_json FROM connector_outbox - WHERE turn_id IS NOT NULL AND turn_id != '' AND ordering_key = '' + WHERE ordering_key = '' ORDER BY id """ ).fetchall() @@ -11552,6 +11804,7 @@ def _migrate_v15_to_v16_conn(conn: sqlite3.Connection) -> None: "UPDATE connector_outbox SET ordering_key = ? WHERE id = ?", ( _legacy_outbox_ordering_key( + outbox_id=int(outbox_id), outbox_payload=_json_object(outbox_payload), worker_id=worker_id, turn_payload=_json_object(turn_payload), @@ -11561,7 +11814,7 @@ def _migrate_v15_to_v16_conn(conn: sqlite3.Connection) -> None: ) deadline = _connector_add_seconds( utc_timestamp(), - _configured_connector_ack_ttl_seconds(), + max(1, int(connector_ack_ttl_seconds)), ) awaiting_rows = conn.execute( "SELECT id, private_state_json FROM connector_outbox WHERE status = 'awaiting_ack'" @@ -11595,6 +11848,48 @@ def _migrate_v15_to_v16_conn(conn: sqlite3.Connection) -> None: conn.execute(CREATE_CONNECTOR_ORDERING_INDEX) +def _migrate_v16_to_v17_conn(conn: sqlite3.Connection) -> None: + """Give unresolved legacy outbox rows independent FIFO partitions.""" + if _table_columns(conn, "turns"): + rows = conn.execute( + """ + SELECT outbox.id, outbox.payload_json, + turns.worker_id, turns.payload_json + FROM connector_outbox AS outbox + LEFT JOIN turns + ON turns.host_id = outbox.host_id + AND turns.turn_id = outbox.turn_id + WHERE outbox.ordering_key = '' + ORDER BY outbox.id + """ + ).fetchall() + else: + rows = [ + (row[0], row[1], None, None) + for row in conn.execute( + """ + SELECT id, payload_json + FROM connector_outbox + WHERE ordering_key = '' + ORDER BY id + """ + ).fetchall() + ] + for outbox_id, outbox_payload, worker_id, turn_payload in rows: + conn.execute( + "UPDATE connector_outbox SET ordering_key = ? WHERE id = ?", + ( + _legacy_outbox_ordering_key( + outbox_id=int(outbox_id), + outbox_payload=_json_object(outbox_payload), + worker_id=worker_id, + turn_payload=_json_object(turn_payload), + ), + int(outbox_id), + ), + ) + + MIGRATIONS: tuple[Migration, ...] = ( Migration(0, 1, _migrate_v0_to_v1_conn), Migration(1, 2, _migrate_v1_to_v2_conn), @@ -11612,6 +11907,7 @@ def _migrate_v15_to_v16_conn(conn: sqlite3.Connection) -> None: Migration(13, 14, _migrate_v13_to_v14_conn), Migration(14, 15, _migrate_v14_to_v15_conn), Migration(15, 16, _migrate_v15_to_v16_conn), + Migration(16, 17, _migrate_v16_to_v17_conn), ) @@ -11708,6 +12004,7 @@ def _run_migrations( conn: sqlite3.Connection, *, target_version: int = STORE_SCHEMA_VERSION, + connector_ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, ) -> None: """Run exact ordered transitions with one transaction per version.""" if conn.in_transaction: @@ -11720,7 +12017,15 @@ def _run_migrations( raise RuntimeError("invalid migration registry dispatch") conn.execute("BEGIN IMMEDIATE") try: - migration.apply(conn) + if migration.apply is _migrate_v15_to_v16_conn: + _migrate_v15_to_v16_conn( + conn, + connector_ack_ttl_seconds=max( + 1, int(connector_ack_ttl_seconds) + ), + ) + else: + migration.apply(conn) conn.execute(f"PRAGMA user_version = {migration.to_version}") conn.commit() except Exception: @@ -11731,7 +12036,11 @@ def _run_migrations( raise StoreSchemaError("schema_version_not_advanced") -def ensure_schema(conn: sqlite3.Connection) -> None: +def ensure_schema( + conn: sqlite3.Connection, + *, + connector_ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, +) -> None: """Gate the current schema cheaply, or initialize/migrate older stores.""" version = int(conn.execute("PRAGMA user_version").fetchone()[0]) if version == STORE_SCHEMA_VERSION: @@ -11757,16 +12066,30 @@ def ensure_schema(conn: sqlite3.Connection) -> None: if version == 0 and not _database_has_application_objects(conn): _create_current_schema_conn(conn) return - _run_migrations(conn) + _run_migrations( + conn, + connector_ack_ttl_seconds=max( + 1, int(connector_ack_ttl_seconds) + ), + ) _ensure_schema = ensure_schema -def init_store(db_path: Path) -> None: +def init_store( + db_path: Path, + *, + connector_ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, +) -> None: """Initialize or migrate the sqlite store to the current schema.""" with _connect(db_path, prepare=True) as conn: - ensure_schema(conn) + ensure_schema( + conn, + connector_ack_ttl_seconds=max( + 1, int(connector_ack_ttl_seconds) + ), + ) def _normalized_command_request_policy( diff --git a/tests/test_backend_pending.py b/tests/test_backend_pending.py index d06e3e5..4cfd409 100644 --- a/tests/test_backend_pending.py +++ b/tests/test_backend_pending.py @@ -1449,7 +1449,7 @@ def test_current_schema_creation_has_exact_binding_and_claim_state(tmp_path: Pat db = tmp_path / "current-schema.db" init_store(db) with sqlite3.connect(db) as conn: - assert conn.execute("PRAGMA user_version").fetchone()[0] == STORE_SCHEMA_VERSION == 16 + assert conn.execute("PRAGMA user_version").fetchone()[0] == STORE_SCHEMA_VERSION == 17 columns = { str(row[1]) for row in conn.execute("PRAGMA table_info(backend_pending)").fetchall() diff --git a/tests/test_connector_daemon_cli.py b/tests/test_connector_daemon_cli.py index f9f8abc..96b1c65 100644 --- a/tests/test_connector_daemon_cli.py +++ b/tests/test_connector_daemon_cli.py @@ -17,6 +17,7 @@ from tendwire.core.models import Snapshot from tendwire.daemon import TendwireDaemon from tendwire.daemon_api import TendwireDaemonAPI +from tendwire.store import sqlite as store_sqlite from tendwire.store.sqlite import init_store @@ -194,6 +195,26 @@ def test_daemon_periodic_tick_reclaims_without_a_followup_poll(tmp_path: Path) - assert delivery_status == "expired" +def test_daemon_periodic_tick_skips_write_reclaim_when_nothing_is_due( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "daemon-no-reclaim-due.db" + _enqueue(db_path, host_id="daemon-host", key="not-leased") + daemon = TendwireDaemon(Config(host_id="daemon-host", db_path=db_path)) + + def reject_write(*_args: Any, **_kwargs: Any) -> None: + raise AssertionError("write reclaim should not run without due work") + + monkeypatch.setattr( + store_sqlite, + "reclaim_expired_connector_leases", + reject_write, + ) + + daemon._connector_periodic_tick() + + def test_cli_connector_poll_and_ack_print_json_only(tmp_path: Path, capsys) -> None: db_path = tmp_path / "cli-connector.db" _enqueue(db_path) diff --git a/tests/test_connector_outbox.py b/tests/test_connector_outbox.py index e0dc784..2d8845d 100644 --- a/tests/test_connector_outbox.py +++ b/tests/test_connector_outbox.py @@ -5,7 +5,7 @@ import json import sqlite3 from concurrent.futures import ThreadPoolExecutor -from datetime import datetime +from datetime import datetime, timedelta, timezone from pathlib import Path from threading import Barrier from typing import Any @@ -18,6 +18,7 @@ from tendwire.store.sqlite import ( SnapshotObservationContext, ack_connector_delivery, + connector_reclaim_due, defer_connector_delivery, fail_connector_delivery, init_store, @@ -246,6 +247,36 @@ def test_renew_extends_live_lease_and_release_requeues_immediately(tmp_path: Pat assert second["attempt"] == 2 +@pytest.mark.parametrize("action", ["renew", "release"]) +def test_renew_and_release_reject_stale_live_refs( + tmp_path: Path, + action: str, +) -> None: + db_path = tmp_path / f"stale-{action}.db" + _enqueue(db_path, key=f"stale-{action}") + api = ConnectorOutboxAPI(db_path, "host-a") + item = api.poll({"name": "attention"})["items"][0] + with sqlite3.connect(str(db_path)) as conn: + row_id, private_state_json = conn.execute( + "SELECT id, private_state_json FROM connector_outbox" + ).fetchone() + private_state = json.loads(private_state_json) + private_state["current_delivery_id"] = -1 + conn.execute( + "UPDATE connector_outbox SET private_state_json = ? WHERE id = ?", + (json.dumps(private_state), row_id), + ) + + params = {"name": "attention", "ref": item["ref"]} + if action == "renew": + params["lease_seconds"] = 120 + result = getattr(api, action)(params) + + assert result["ok"] is False + assert result["status"] == "stale_ref" + _assert_no_forbidden(result) + + @pytest.mark.parametrize("terminal_status", ["dead_letter", "awaiting_ack"]) def test_terminal_or_planned_final_head_does_not_block_same_worker( tmp_path: Path, @@ -1067,6 +1098,49 @@ def _stage_final_plan( return _commit_plan(api, token) +def _stage_source_bound_plan( + api: ConnectorOutboxAPI, + *, + turn_id: str, + revision: str, + ranges: list[tuple[int, int]], + version: str, +) -> tuple[dict[str, Any], dict[str, Any]]: + source = api.poll({"name": "turn-final"})["items"][0] + begun = api.prepare( + { + "schema_version": 1, + "action": "begin", + "name": "turn-final", + "turn_id": turn_id, + "content_revision": revision, + "presentation_version": version, + "part_count": len(ranges), + "source_ref": source["ref"], + } + ) + assert begun["ok"] is True + for ordinal, (start, end) in enumerate(ranges): + assert _put_final_part( + api, + plan_token=begun["plan_token"], + ordinal=ordinal, + start=start, + end=end, + )["ok"] is True + committed = api.prepare( + { + "schema_version": 1, + "action": "commit", + "name": "turn-final", + "plan_token": begun["plan_token"], + "source_ref": source["ref"], + } + ) + assert committed["ok"] is True + return source, committed + + def _drain_turn_final(api: ConnectorOutboxAPI) -> list[str]: keys: list[str] = [] while True: @@ -1193,7 +1267,7 @@ def test_v6_to_current_plan_migration_is_bounded_atomic_and_preserves_jobs( ).fetchall() } foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall() - assert version == store_sqlite.STORE_SCHEMA_VERSION == 16 + assert version == store_sqlite.STORE_SCHEMA_VERSION == 17 assert plan_row == (plan["plan_token"], 1, None, "active") assert job_count == 2 assert outbox_count == 3 @@ -2298,6 +2372,357 @@ def test_final_parts_preserve_enqueued_worker_fallback_after_adoption( assert keys == [("worker-before-adoption",), ("worker-before-adoption",)] +def test_unresolvable_final_rows_receive_independent_ordering_partitions( + tmp_path: Path, +) -> None: + db_path = tmp_path / "orphan-ordering-partitions.db" + init_store(db_path) + with sqlite3.connect(str(db_path)) as conn: + first_key = store_sqlite._turn_ordering_key_conn( + conn, + host_id="host-a", + turn_id="turn-deleted-a", + ) + second_key = store_sqlite._turn_ordering_key_conn( + conn, + host_id="host-a", + turn_id="turn-deleted-b", + ) + _enqueue_final_root( + db_path, + key_suffix="deleted-a", + ordering_key=first_key, + ) + _enqueue_final_root( + db_path, + key_suffix="deleted-b", + ordering_key=second_key, + ) + + items = ConnectorOutboxAPI(db_path, "host-a").poll( + {"name": "turn-final", "limit": 10} + )["items"] + + assert first_key == "orphan:turn-deleted-a" + assert second_key == "orphan:turn-deleted-b" + assert len(items) == 2 + + +def test_source_less_recovery_plan_blocks_later_same_worker_final( + tmp_path: Path, +) -> None: + db_path = tmp_path / "source-less-recovery-plan-ordering.db" + stable_key = "wsk1_" + ("e" * 64) + first_turn, first_revision = _canonical_turn( + db_path, + worker_id="worker-shared", + source_turn_id="source-first", + stable_key=stable_key, + final_text="abcdefgh", + ) + api = ConnectorOutboxAPI(db_path, "host-a", max_attempts=1) + failed_plan = _stage_final_plan( + api, + turn_id=first_turn, + revision=first_revision, + ranges=[(0, 4), (4, 8)], + ) + failed_job = api.poll({"name": "turn-final"})["items"][0] + assert api.fail( + { + "name": "turn-final", + "ref": failed_job["ref"], + "delay_seconds": 0, + } + )["status"] == "attempts_exhausted" + recovery_plan = api.prepare( + { + "schema_version": 1, + "action": "recover", + "name": "turn-final", + "failed_plan_token": failed_plan["plan_token"], + "request_id": "source-less-ordering-recovery-1", + } + ) + assert recovery_plan["status"] == "recovered" + second_turn, second_revision = _canonical_turn( + db_path, + worker_id="worker-shared", + source_turn_id="source-second", + stable_key=stable_key, + final_text="ijklmnop", + ) + with sqlite3.connect(str(db_path)) as conn: + later_source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=second_turn, + content_revision_value=second_revision, + now="2026-01-01T00:01:00+00:00", + ) + assert later_source_id is not None + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + """ + SELECT source_outbox_id, recovers_plan_token + FROM turn_presentation_plans + WHERE plan_token = ? + """, + (recovery_plan["plan_token"],), + ).fetchone() == (None, failed_plan["plan_token"]) + + items = api.poll({"name": "turn-final", "limit": 10})["items"] + + assert len(items) == 1 + assert items[0]["payload"]["plan_token"] == recovery_plan["plan_token"] + + +def test_part_ack_progress_extends_source_deadline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "ack-progress-deadline.db" + current = [datetime(2026, 1, 1, tzinfo=timezone.utc)] + monkeypatch.setattr( + store_sqlite, + "utc_timestamp", + lambda: current[0].isoformat(), + ) + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now=current[0].isoformat(), + ) + assert source_id is not None + api = ConnectorOutboxAPI(db_path, "host-a", ack_ttl_seconds=30) + _source, plan = _stage_source_bound_plan( + api, + turn_id=turn_id, + revision=revision, + ranges=[(0, 4), (4, 8)], + version="turn-present-progress-v1", + ) + first_part = api.poll({"name": "turn-final"})["items"][0] + current[0] += timedelta(seconds=20) + + acknowledged = api.ack( + {"name": "turn-final", "ref": first_part["ref"]} + ) + with sqlite3.connect(str(db_path)) as conn: + source_private = json.loads( + conn.execute( + "SELECT private_state_json FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone()[0] + ) + current[0] += timedelta(seconds=11) + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + ) + + assert acknowledged["status"] == "acknowledged" + assert datetime.fromisoformat(source_private["ack_deadline_at"]) == ( + datetime(2026, 1, 1, tzinfo=timezone.utc) + timedelta(seconds=50) + ) + assert reclaimed["reclaimed"] == 0 + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + "SELECT state FROM turn_presentation_plans WHERE plan_token = ?", + (plan["plan_token"],), + ).fetchone() == ("active",) + + +def test_ack_deadline_reclaim_preserves_validly_leased_part( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "ack-deadline-live-part.db" + current = [datetime(2026, 1, 1, tzinfo=timezone.utc)] + monkeypatch.setattr( + store_sqlite, + "utc_timestamp", + lambda: current[0].isoformat(), + ) + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now=current[0].isoformat(), + ) + assert source_id is not None + api = ConnectorOutboxAPI(db_path, "host-a", ack_ttl_seconds=30) + _source, plan = _stage_source_bound_plan( + api, + turn_id=turn_id, + revision=revision, + ranges=[(0, 4), (4, 8)], + version="turn-present-live-lease-v1", + ) + leased_part = api.poll( + {"name": "turn-final", "lease_seconds": 60} + )["items"][0] + current[0] += timedelta(seconds=31) + + assert connector_reclaim_due( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + ) is False + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + ) + acknowledged = api.ack( + {"name": "turn-final", "ref": leased_part["ref"]} + ) + + assert reclaimed["reclaimed"] == 0 + assert acknowledged["status"] == "acknowledged" + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + "SELECT state FROM turn_presentation_plans WHERE plan_token = ?", + (plan["plan_token"],), + ).fetchone() == ("active",) + assert conn.execute( + "SELECT status FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone() == ("awaiting_ack",) + + +def test_repeated_ack_deadline_reclaims_do_not_exhaust_healthy_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "repeated-ack-deadline-reclaim.db" + current = [datetime(2026, 1, 1, tzinfo=timezone.utc)] + monkeypatch.setattr( + store_sqlite, + "utc_timestamp", + lambda: current[0].isoformat(), + ) + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now=current[0].isoformat(), + ) + assert source_id is not None + api = ConnectorOutboxAPI( + db_path, + "host-a", + ack_ttl_seconds=30, + max_attempts=3, + ) + + for cycle in range(3): + source, _plan = _stage_source_bound_plan( + api, + turn_id=turn_id, + revision=revision, + ranges=[(0, 8)], + version=f"turn-present-slow-ack-v{cycle + 1}", + ) + assert source["attempt"] == cycle + 1 + current[0] += timedelta(seconds=31) + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + ) + assert reclaimed["reclaimed"] == 1 + + fourth = api.poll({"name": "turn-final"})["items"][0] + + assert fourth["attempt"] == 4 + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + "SELECT status FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone() == ("leased",) + + +def test_ack_deadline_reclaims_do_not_inflate_later_failure_budget( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "ack-deadline-failure-budget.db" + current = [datetime(2026, 1, 1, tzinfo=timezone.utc)] + monkeypatch.setattr( + store_sqlite, + "utc_timestamp", + lambda: current[0].isoformat(), + ) + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + assert store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now=current[0].isoformat(), + ) is not None + api = ConnectorOutboxAPI( + db_path, + "host-a", + ack_ttl_seconds=30, + max_attempts=2, + ) + + _stage_source_bound_plan( + api, + turn_id=turn_id, + revision=revision, + ranges=[(0, 8)], + version="turn-present-timeout-before-failure-v1", + ) + current[0] += timedelta(seconds=31) + assert reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + )["reclaimed"] == 1 + + first_real_attempt = api.poll({"name": "turn-final"})["items"][0] + first_failure = api.fail( + { + "name": "turn-final", + "ref": first_real_attempt["ref"], + "delay_seconds": 0, + } + ) + second_real_attempt = api.poll({"name": "turn-final"})["items"][0] + second_failure = api.fail( + { + "name": "turn-final", + "ref": second_real_attempt["ref"], + "delay_seconds": 0, + } + ) + + assert first_real_attempt["attempt"] == 2 + assert first_failure["status"] == "retry_scheduled" + assert second_real_attempt["attempt"] == 3 + assert second_failure["status"] == "attempts_exhausted" + + def test_awaiting_ack_deadline_fails_plan_and_requeues_source(tmp_path: Path) -> None: db_path = tmp_path / "awaiting-ack-reclaim.db" turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") @@ -2485,6 +2910,7 @@ def test_v16_migration_backfills_ordering_and_awaiting_ack_deadlines( ) -> None: db_path = tmp_path / "connector-v16-migration.db" stable_key = "wsk1_" + ("c" * 64) + enqueue_stable_key = "wsk1_" + ("d" * 64) tombstoned_turn, _ = _canonical_turn( db_path, worker_id="worker-tombstoned", @@ -2499,6 +2925,15 @@ def test_v16_migration_backfills_ordering_and_awaiting_ack_deadlines( stable_key="invalid", final_text="done", ) + deleted_turns = [ + _canonical_turn( + db_path, + worker_id=f"worker-deleted-{suffix}", + source_turn_id=f"source-deleted-{suffix}", + final_text="done", + )[0] + for suffix in ("a", "b") + ] with sqlite3.connect(str(db_path)) as conn: assert store_sqlite._tombstone_turn_conn( conn, @@ -2512,19 +2947,28 @@ def test_v16_migration_backfills_ordering_and_awaiting_ack_deadlines( (fallback_turn, "dead_letter"), (fallback_turn, "superseded"), ): + payload = ( + { + "stable_key": enqueue_stable_key, + "stable_key_version": 1, + } + if status == "dead_letter" + else {} + ) cursor = conn.execute( """ INSERT INTO connector_outbox ( host_id, connector, delivery_key, delivery_kind, turn_id, ordering_key, status, payload_json, private_state_json, created_at, updated_at - ) VALUES (?, 'turn-final', ?, 'final_ready', ?, '', ?, '{}', '{}', ?, ?) + ) VALUES (?, 'turn-final', ?, 'final_ready', ?, '', ?, ?, '{}', ?, ?) """, ( "host-a", f"migration-{turn_id}-{status}", turn_id, status, + json.dumps(payload), "2026-01-01T00:00:00+00:00", "2026-01-01T00:00:00+00:00", ), @@ -2540,17 +2984,47 @@ def test_v16_migration_backfills_ordering_and_awaiting_ack_deadlines( """, (cursor.lastrowid, "2026-01-01T00:00:00+00:00"), ) + for deleted_turn in deleted_turns: + conn.execute( + "DELETE FROM turn_content_revisions WHERE host_id = ? AND turn_id = ?", + ("host-a", deleted_turn), + ) + conn.execute( + "DELETE FROM turns WHERE host_id = ? AND turn_id = ?", + ("host-a", deleted_turn), + ) + orphan_ids: list[int] = [] + for suffix, deleted_turn in zip(("a", "b"), deleted_turns, strict=True): + cursor = conn.execute( + """ + INSERT INTO connector_outbox ( + host_id, connector, delivery_key, delivery_kind, turn_id, + ordering_key, status, payload_json, private_state_json, + created_at, updated_at + ) VALUES ( + 'host-a', 'turn-final', ?, 'final_ready', ?, '', + 'dead_letter', '{}', '{}', ?, ? + ) + """, + ( + f"migration-deleted-{suffix}", + deleted_turn, + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:00:00+00:00", + ), + ) + orphan_ids.append(int(cursor.lastrowid)) conn.execute("DROP INDEX IF EXISTS idx_connector_outbox_final_ordering") conn.execute("ALTER TABLE connector_outbox DROP COLUMN ordering_key") conn.execute("PRAGMA user_version = 15") migration_now = "2026-01-03T00:00:00+00:00" - monkeypatch.setenv("TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", "123") + monkeypatch.setenv("TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", "999") monkeypatch.setattr(store_sqlite, "utc_timestamp", lambda: migration_now) - init_store(db_path) + init_store(db_path, connector_ack_ttl_seconds=123) with sqlite3.connect(str(db_path)) as conn: rows = conn.execute( """ - SELECT turn_id, status, ordering_key, private_state_json + SELECT delivery_key, turn_id, status, ordering_key, private_state_json FROM connector_outbox WHERE delivery_key LIKE 'migration-%' ORDER BY id @@ -2559,12 +3033,31 @@ def test_v16_migration_backfills_ordering_and_awaiting_ack_deadlines( delivery_private = conn.execute( "SELECT private_state_json FROM connector_deliveries WHERE status = 'awaiting_ack'" ).fetchone()[0] - assert rows[0][0:3] == (tombstoned_turn, "awaiting_ack", stable_key) - ack_deadline = json.loads(rows[0][3])["ack_deadline_at"] + by_key = {row[0]: row[1:] for row in rows} + tombstoned_key = f"migration-{tombstoned_turn}-awaiting_ack" + dead_letter_key = f"migration-{fallback_turn}-dead_letter" + superseded_key = f"migration-{fallback_turn}-superseded" + assert by_key[tombstoned_key][0:3] == ( + tombstoned_turn, + "awaiting_ack", + stable_key, + ) + ack_deadline = json.loads(by_key[tombstoned_key][3])["ack_deadline_at"] assert ( datetime.fromisoformat(ack_deadline) - datetime.fromisoformat(migration_now) ).total_seconds() == 123 - assert rows[1][0:3] == (fallback_turn, "dead_letter", "worker-fallback") - assert rows[2][0:3] == (fallback_turn, "superseded", "worker-fallback") + assert by_key[dead_letter_key][0:3] == ( + fallback_turn, + "dead_letter", + enqueue_stable_key, + ) + assert by_key[superseded_key][0:3] == ( + fallback_turn, + "superseded", + "worker-fallback", + ) + assert by_key["migration-deleted-a"][2] == f"orphan:{orphan_ids[0]}" + assert by_key["migration-deleted-b"][2] == f"orphan:{orphan_ids[1]}" + assert by_key["migration-deleted-a"][2] != by_key["migration-deleted-b"][2] assert json.loads(delivery_private)["ack_deadline_at"] == ack_deadline diff --git a/tests/test_delivery_retention_migration.py b/tests/test_delivery_retention_migration.py index 1b3d2dd..0e01fa9 100644 --- a/tests/test_delivery_retention_migration.py +++ b/tests/test_delivery_retention_migration.py @@ -875,7 +875,7 @@ def test_v10_to_v12_migration_retains_finals_without_reposting_or_leaking( finals = _seed_v10_finals(db_path) init_store(db_path) - assert store_sqlite.STORE_SCHEMA_VERSION == 16 + assert store_sqlite.STORE_SCHEMA_VERSION == 17 delivered_key = _final_key(*finals["delivered"]) hold_keys = {_final_key(*finals[label]) for label in ("hold-a", "hold-b")} diff --git a/tests/test_delivery_retention_projection.py b/tests/test_delivery_retention_projection.py index 4e3aa0b..0e8f150 100644 --- a/tests/test_delivery_retention_projection.py +++ b/tests/test_delivery_retention_projection.py @@ -165,7 +165,7 @@ def _turn_rows_snapshot(db_path: Path) -> tuple[tuple[Any, ...], ...]: def _assert_continuity_integrity(db_path: Path) -> None: with sqlite3.connect(str(db_path)) as conn: - assert conn.execute("PRAGMA user_version").fetchone() == (store_sqlite.STORE_SCHEMA_VERSION,) == (16,) + assert conn.execute("PRAGMA user_version").fetchone() == (store_sqlite.STORE_SCHEMA_VERSION,) == (17,) assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] current_counts = conn.execute( """ diff --git a/tests/test_delivery_retention_recovery.py b/tests/test_delivery_retention_recovery.py index 2e2ad27..f91f1d0 100644 --- a/tests/test_delivery_retention_recovery.py +++ b/tests/test_delivery_retention_recovery.py @@ -973,7 +973,7 @@ def test_v12_migration_uses_effective_recovery_lineage_without_repost_or_hold( api = ConnectorOutboxAPI(db_path, HOST_ID) assert api.poll({"name": FINAL_NAME, "limit": 100})["items"] == [] with sqlite3.connect(str(db_path)) as conn: - assert conn.execute("PRAGMA user_version").fetchone()[0] == store_sqlite.STORE_SCHEMA_VERSION == 16 + assert conn.execute("PRAGMA user_version").fetchone()[0] == store_sqlite.STORE_SCHEMA_VERSION == 17 anchor = conn.execute( """ SELECT delivery_kind, status diff --git a/tests/test_store.py b/tests/test_store.py index 160726d..7f1691b 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -11191,7 +11191,7 @@ def test_v13_migration_repairs_nonpositive_turn_sequences_and_blocks_recurrence( init_store(db_path) with sqlite3.connect(str(db_path)) as conn: - assert _user_version(conn) == store_sqlite.STORE_SCHEMA_VERSION == 16 + assert _user_version(conn) == store_sqlite.STORE_SCHEMA_VERSION == 17 assert conn.execute( """ SELECT turn_id, list_sequence From b470885487470c1645bf811fce380e319c041aa5 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Sun, 19 Jul 2026 13:11:54 +0200 Subject: [PATCH 6/7] merge: unify per-owner final ordering with ordering_key partitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream 9919e76 scoped final_ready head-of-line blocking by payload stable_key; the ordering_key column introduced here materializes the same owner identity at enqueue time (with orphan fallbacks), so the payload-JSON clauses are dropped as subsumed. dead_letter stays terminal for ordering per the incident verdict — the owner-isolation test now asserts a dead_letter final blocks no later finals at all. Co-Authored-By: Claude Fable 5 --- src/tendwire/store/sqlite.py | 14 -------------- tests/test_delivery_retention.py | 13 +++++++++---- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/tendwire/store/sqlite.py b/src/tendwire/store/sqlite.py index a7fc14d..7d437d6 100644 --- a/src/tendwire/store/sqlite.py +++ b/src/tendwire/store/sqlite.py @@ -5770,20 +5770,6 @@ def poll_connector_outbox( AND earlier_final.delivery_kind = 'final_ready' AND earlier_final.ordering_key = outbox.ordering_key AND earlier_final.id < outbox.id - AND json_extract( - earlier_final.payload_json, - '$.stable_key' - ) = json_extract( - outbox.payload_json, - '$.stable_key' - ) - AND json_extract( - earlier_final.payload_json, - '$.stable_key_version' - ) = json_extract( - outbox.payload_json, - '$.stable_key_version' - ) AND earlier_final.status NOT IN ( 'delivered', 'superseded', diff --git a/tests/test_delivery_retention.py b/tests/test_delivery_retention.py index d42965b..9302a24 100644 --- a/tests/test_delivery_retention.py +++ b/tests/test_delivery_retention.py @@ -321,7 +321,11 @@ def _aggressive_cleanup(db_path: Path, *, batch_size: int = 100) -> dict[str, An ) -def test_dead_letter_final_blocks_only_its_stable_owner(tmp_path: Path) -> None: +def test_dead_letter_final_blocks_no_later_finals(tmp_path: Path) -> None: + # dead_letter is terminal for FIFO ordering: it must not block ANY later + # final — not other owners (the original intent of this test) and not its + # own owner either (a poisoned reply must never wedge its worker; recovery + # goes through retry_final_ready_delivery/supersede instead). db_path = tmp_path / "dead-letter-owner-isolation.db" second_worker_id = "worker-2" second_stable_key = "wsk1_" + ("b" * 64) @@ -394,8 +398,9 @@ def test_dead_letter_final_blocks_only_its_stable_owner(tmp_path: Path) -> None: {"name": FINAL_NAME, "limit": 100, "lease_seconds": 60} ) assert unrelated["ok"] is True - assert len(unrelated["items"]) == 1 - assert unrelated["items"][0]["payload"]["stable_key"] == second_stable_key + assert len(unrelated["items"]) == 2 + polled_keys = {item["payload"]["stable_key"] for item in unrelated["items"]} + assert polled_keys == {STABLE_KEY, second_stable_key} with sqlite3.connect(str(db_path)) as conn: first_owner_states = conn.execute( @@ -409,7 +414,7 @@ def test_dead_letter_final_blocks_only_its_stable_owner(tmp_path: Path) -> None: """, (FINAL_NAME, STABLE_KEY), ).fetchall() - assert first_owner_states == [("dead_letter",), ("queued",)] + assert first_owner_states == [("dead_letter",), ("leased",)] def test_acknowledged_prefix_is_cleaned_but_failed_suffix_and_exact_final_survive( From c8aceec0a88090c8472a5be18002b08d59907c7e Mon Sep 17 00:00:00 2001 From: jerryfane Date: Sun, 19 Jul 2026 13:23:46 +0200 Subject: [PATCH 7/7] merge: working-predecessor pointer is omitted under adopt-in-place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 46abd8c stamps working_predecessor_turn_id only when the final's turn id differs from the command predecessor; adoption preserves the command row's id, so the pointer is correctly absent — assert that instead of the split-id flow. Co-Authored-By: Claude Fable 5 --- tests/test_command_submission.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_command_submission.py b/tests/test_command_submission.py index bed13f1..171aceb 100644 --- a/tests/test_command_submission.py +++ b/tests/test_command_submission.py @@ -3735,10 +3735,10 @@ def test_stable_owner_pending_command_survives_worker_churn_and_source_wins( """, (config.host_id,), ).fetchone() - assert ( - final_root["working_predecessor_turn_id"] - == command_before["id"] - ) + # Adopt-in-place preserves the command row's turn id, so the final's own + # turn IS the working predecessor; the payload pointer is correctly + # omitted (it is only stamped when the ids differ). + assert "working_predecessor_turn_id" not in final_root source_wins = upsert_command_pending_turn( config.db_path,